mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 05:56:19 +01:00
Merge master into sleep blocker inventory
Preserve the explicit external database fixture and owned-client cleanup while adopting the upstream PostgreSQL shutdown timeout in both sleep suites. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
4a9974ea03
314 files changed
+10887
-2969
No files matched your search
@@ -4,7 +4,7 @@ import request from "supertest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
|
||||
import { actorMiddleware } from "../middleware/auth.js";
|
||||
@@ -53,7 +53,7 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
it("bounds actual middleware responses and preserves the complete stored run", async () => {
|
||||
const fixture = await seed();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
|
||||
@@ -61,7 +61,7 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId,
|
||||
identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" });
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384);
|
||||
|
||||
@@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js";
|
||||
import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
|
||||
import { agents, companies, issues, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js";
|
||||
import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "../services/agent-instruction-working-copies.js";
|
||||
@@ -24,6 +24,7 @@ import { resolveManagedInstructionsRoot } from "../services/agent-instructions.j
|
||||
import { buildNativeRuntimeContext } from "../services/native-runtime/runtime-context.js";
|
||||
import type { EnvironmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js";
|
||||
import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js";
|
||||
import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
import { heartbeatRunEvents } from "@paperclipai/db";
|
||||
@@ -123,6 +124,44 @@ describe("persistent agent directories", () => {
|
||||
for (const copy of [active, completed, transferred]) expect((await fs.stat(copy.localRoot)).isDirectory()).toBe(true);
|
||||
} finally { await worker.stop(); }
|
||||
});
|
||||
it("retires a stale retry on an already cleaned receipt after restart without filesystem work", async () => {
|
||||
const copy = await run();
|
||||
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
|
||||
nextAttemptAt: new Date(), receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
clock(); const worker = coordinator(), removes = vi.spyOn(fs, "rm");
|
||||
try {
|
||||
await agentInstructionWorkingCopyService(db).registerDirectoryCleanupWork(worker, () => true, async () => true).ready;
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect((await copies.get(companyId, copy.runId))?.nextAttemptAt).toBeNull();
|
||||
});
|
||||
await expect.poll(() => idleWorkSnapshot().active).toBe(0);
|
||||
const reads = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
expect(reads).not.toHaveBeenCalled(); expect(removes).not.toHaveBeenCalled();
|
||||
expect(worker.nextWakeAt()).toBeNull();
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
} finally { await worker.stop(); }
|
||||
});
|
||||
it("honors a cleanup obligation published before stale-deadline settlement", async () => {
|
||||
const copy = await run(), dueAt = new Date(Date.now() + 30_000);
|
||||
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
|
||||
nextAttemptAt: dueAt, receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
const transact = db.transaction.bind(db);
|
||||
const race = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => {
|
||||
await db.update(agentInstructionWorkingCopies).set({ receipt: { ...copy.receipt, cleanupPending: true } })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
return transact(callback, config);
|
||||
});
|
||||
try {
|
||||
await copies.release(companyId, copy.runId);
|
||||
expect(await copies.get(companyId, copy.runId)).toMatchObject({ receipt: { cleanupPending: false }, nextAttemptAt: null });
|
||||
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
} finally { race.mockRestore(); }
|
||||
});
|
||||
it("reactivates from real collection, preserves saved files, retries failed cleanup, and becomes quiet", async () => {
|
||||
const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "memory.txt"), "saved memory");
|
||||
clock(); const worker = coordinator();
|
||||
@@ -1461,6 +1500,53 @@ describe("persistent agent directories", () => {
|
||||
return { copy: (await copies.get(companyId, copy.runId))!, lease, environmentId };
|
||||
}
|
||||
|
||||
it.each(["destroyed", "stopped", "broken_handoff", "extra_owner", "foreign_company"] as const)(
|
||||
"recovers an earlier remote copy only from the exact destroyed successor chain (%s)", async scenario => {
|
||||
const { copy, lease } = await unavailableRemote();
|
||||
const issueId = randomUUID(), successorRunId = randomUUID(), successorId = randomUUID();
|
||||
const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000);
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Completed retained workspace", status: "done" });
|
||||
await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });
|
||||
const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment",
|
||||
issueId, acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: { agentId } })
|
||||
.where(eq(environmentLeases.id, lease.id)).returning();
|
||||
const successor = { ...previous!, id: successorId, heartbeatRunId: successorRunId, acquiredAt: handoffAt,
|
||||
releasedAt: new Date(), metadata: { agentId } };
|
||||
if (scenario === "foreign_company") {
|
||||
const foreignCompanyId = randomUUID();
|
||||
await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated owner", issuePrefix: randomUUID().slice(0, 8) });
|
||||
successor.companyId = foreignCompanyId;
|
||||
}
|
||||
const handoff = createReusableLeaseHandoff(previous!, successor);
|
||||
const termination = remoteTerminationReceipt(successor, { providerLeaseId: lease.providerLeaseId,
|
||||
state: scenario === "stopped" ? "stopped" : "destroyed" });
|
||||
await db.update(environmentLeases).set({ metadata: { agentId, reusableLeaseSuccessor: handoff } })
|
||||
.where(eq(environmentLeases.id, lease.id));
|
||||
await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, remoteExecutionTermination: termination,
|
||||
reusableLeasePredecessor: scenario === "broken_handoff" ? { ...handoff, handoffId: randomUUID() } : handoff } });
|
||||
if (scenario === "extra_owner") await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), status: "active" });
|
||||
await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", attempts: 0, candidateHash: null, candidateBase64: null })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockRejectedValue(new Error("must not execute"));
|
||||
try {
|
||||
await copies.recoverStopped();
|
||||
const recovered = (await copies.get(companyId, copy.runId))!;
|
||||
if (scenario === "destroyed") {
|
||||
expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE",
|
||||
receipt: { cleanupPending: false }, nextAttemptAt: null });
|
||||
expect(recovered.processStoppedAt).toBeInstanceOf(Date);
|
||||
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} else {
|
||||
expect(recovered).toMatchObject({ state: "pending_collection", processStoppedAt: null });
|
||||
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
|
||||
}
|
||||
expect(shell).not.toHaveBeenCalled();
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
const [historical] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
|
||||
expect(historical!.metadata).not.toHaveProperty("remoteExecutionTermination");
|
||||
} finally { shell.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each([undefined, "stopped", "companyId", "runId", "leaseId", "provider", "providerLeaseId"])(
|
||||
"defers unavailable cleanup without exact destruction proof (%s)", async proof => {
|
||||
const { copy } = await unavailableRemote({ state: proof === undefined ? undefined : proof === "stopped" ? "stopped" : "destroyed",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { agents, companies, costEvents, createDb } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
@@ -61,7 +61,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => {
|
||||
|
||||
async function observe<T>(run: (observed: ReturnType<typeof createDb>) => Promise<T>) {
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agents, costEvents }, capture);
|
||||
try {
|
||||
const result = await run(observed);
|
||||
|
||||
@@ -1496,6 +1496,46 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
|
||||
} finally { fetchSpy.mockRestore(); }
|
||||
});
|
||||
|
||||
it("reads provider identity only for an authorized credential and preserves connected state when profile access is denied", async () => {
|
||||
const owner = "identity-owner";
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const account = await service.save(companyId, owner, { provider: "anthropic", method: "subscription", ownership: "personal", name: "Identity account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "identity-selected-secret");
|
||||
const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ account: { email: "avery@example.test", uuid: "private-account-id" }, organization: { name: "Example team", uuid: "private-org-id" } }));
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = req.header("x-test-anonymous") ? { type: "none" }
|
||||
: req.header("x-agent") ? { type: "agent", agentId, companyId, companyIds: [companyId], onBehalfOfUserId: owner }
|
||||
: { type: "board", source: "session", userId: req.header("x-test-user") ?? owner, companyIds: [companyId] };
|
||||
next();
|
||||
});
|
||||
app.use("/api", aiConnectionRoutes(db));
|
||||
app.use((error: { status?: number; message: string }, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(error.status ?? 500).json({ error: error.message }); });
|
||||
const url = `/api/companies/${companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
|
||||
try {
|
||||
const result = await request(app).get(url);
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.headers["cache-control"]).toBe("no-store");
|
||||
expect(result.body).toMatchObject({ ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" });
|
||||
expect(fetchSpy).toHaveBeenCalledWith("https://api.anthropic.com/api/oauth/profile", expect.objectContaining({ headers: expect.objectContaining({ Authorization: "Bearer identity-selected-secret" }) }));
|
||||
for (const value of ["identity-selected-secret", "private-account-id", "private-org-id"]) expect(JSON.stringify(result.body)).not.toContain(value);
|
||||
fetchSpy.mockReset();
|
||||
expect((await request(app).get(url).set("x-test-user", "bob")).status).toBe(404);
|
||||
expect((await request(app).get(url.replace(companyId, otherCompanyId))).status).toBe(403);
|
||||
expect((await request(app).get(url.replace(account.grantId, randomUUID()))).status).toBe(404);
|
||||
expect((await request(app).get(url).set("x-test-anonymous", "yes")).status).toBe(401);
|
||||
expect((await request(app).get(url).set("x-agent", "yes")).status).toBe(403);
|
||||
expect((await request(app).get(url.replace(account.connectionId, "invalid"))).status).toBe(404);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
fetchSpy.mockResolvedValue(Response.json({ error: "private-provider-error" }, { status: 403 }));
|
||||
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "permission_denied", accountLabel: null });
|
||||
expect((await service.list(companyId, owner)).find(item => item.id === account.connectionId)?.status).toBe("connected");
|
||||
fetchSpy.mockClear();
|
||||
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
|
||||
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "connection_unavailable" });
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
} finally { fetchSpy.mockRestore(); }
|
||||
});
|
||||
|
||||
it("uses the same usage probe for shared accounts across legacy and native runner selections", async () => {
|
||||
const account = await service.save(companyId, "alice", { provider: "anthropic", method: "subscription", ownership: "shared", name: "Shared usage account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "shared-usage-secret");
|
||||
const selected = { provider: "anthropic", method: "subscription", mode: "shared", ...account } as const;
|
||||
|
||||
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
@@ -1164,13 +1164,13 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
})));
|
||||
}
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
|
||||
expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]);
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64);
|
||||
@@ -1209,7 +1209,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
});
|
||||
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const feed = await attentionService(db).list(companyId, {
|
||||
@@ -1218,7 +1218,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort())
|
||||
.toEqual([...keptIds].sort());
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4);
|
||||
@@ -1407,7 +1407,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
|
||||
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
|
||||
const summaryEvents: Array<Record<string, unknown>> = [], captureNow = Date.now();
|
||||
const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
|
||||
const summaryCapture = createFullDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
|
||||
installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
|
||||
try {
|
||||
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
|
||||
@@ -1418,7 +1418,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
expect(index.totalCount).toBe(feed.totalCount);
|
||||
|
||||
} finally { summaryCapture.flush(); }
|
||||
const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const cardReads = summaryEvents.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& ["documents", "assets", "issue_attachments"].includes(String(event.table)));
|
||||
expect(cardReads).toEqual([]);
|
||||
|
||||
|
||||
@@ -33,11 +33,13 @@ import { budgetService } from "../services/budgets.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js";
|
||||
import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js";
|
||||
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
|
||||
import { withQueuedCommentIdsInWakePayload } from "../services/issue-queued-comment-queue.js";
|
||||
import { createSystemRecovery } from "../services/system-recovery-state.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js";
|
||||
import * as activityLogModule from "../services/activity-log.js";
|
||||
import * as connectionCredentialBindings from "../services/connection-credential-bindings.js";
|
||||
@@ -112,6 +114,7 @@ import {
|
||||
isNotNull,
|
||||
isNull,
|
||||
like,
|
||||
ne,
|
||||
notInArray,
|
||||
or,
|
||||
sql,
|
||||
@@ -2887,7 +2890,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk };
|
||||
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations,
|
||||
chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks,
|
||||
companyMemberships, chatActions, heartbeatRuns }, capture);
|
||||
@@ -2916,7 +2919,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
} };
|
||||
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatDeliveries }, capture);
|
||||
let projected: { event: Record<string, unknown> };
|
||||
try {
|
||||
@@ -7152,6 +7155,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
expect(await readChatConversationsByEndpointId(f.endpoint.id)).toHaveLength(0);
|
||||
expect(await readChatGitHubReviewsByEndpointId(f.endpoint.id)).toHaveLength(0);
|
||||
});
|
||||
it("offers Retry for a saved automatic PR event after a successful older review", async () => {
|
||||
const f = await reviewBotFixture();
|
||||
const payload = {
|
||||
action: "opened", installation: { id: 2468 },
|
||||
repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" } },
|
||||
sender: { id: 77, login: "maintainer" },
|
||||
pull_request: { number: 81, title: "Review successive heads", body: "", draft: false,
|
||||
base: { sha: "a".repeat(40), ref: "master" }, head: { sha: "b".repeat(40) },
|
||||
user: { id: 42, login: "octocat", type: "User" }, labels: [] },
|
||||
};
|
||||
const send = async (event: typeof payload) => {
|
||||
await f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({
|
||||
event: "pull_request", delivery: randomUUID(), payload: event, webhookSecret: f.webhookSecret,
|
||||
}));
|
||||
await f.service.processPendingDeliveries();
|
||||
};
|
||||
await send(payload);
|
||||
await expect.poll(async () => {
|
||||
await f.service.processPendingDeliveries();
|
||||
return (await readChatConversationsByEndpointId(f.endpoint.id)).length;
|
||||
}, { timeout: 10_000 }).toBe(1);
|
||||
const [conversation] = await readChatConversationsByEndpointId(f.endpoint.id);
|
||||
const [task] = await readIssuesById(conversation.issueId!);
|
||||
const [original] = await db.select().from(chatActions).where(and(
|
||||
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")));
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(original.id)).length).toBe(1);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId,
|
||||
issueId: task.id, status: "succeeded", finishedAt: new Date(), wakeupRequestId: original.id,
|
||||
contextSnapshot: { issueId: task.id, taskKey: task.identifier, source: "chat:github", wakeCommentIds: [original.payload.commentId] },
|
||||
}).returning();
|
||||
await db.update(agentWakeupRequests).set({ status: "completed", runId: run.id }).where(eq(agentWakeupRequests.id, original.id));
|
||||
await send({ ...payload, action: "synchronize", pull_request: { ...payload.pull_request, head: { sha: "c".repeat(40) } } });
|
||||
await expect.poll(async () => {
|
||||
await f.service.processPendingDeliveries();
|
||||
return (await db.select().from(chatActions).where(and(
|
||||
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")))).length;
|
||||
}).toBe(2);
|
||||
const [pending] = await db.select().from(chatActions).where(and(
|
||||
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"), ne(chatActions.id, original.id)));
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(pending.id)).length).toBe(1);
|
||||
const [receipt] = await readAgentWakeupRequestsById(pending.id);
|
||||
await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null,
|
||||
payload: withQueuedCommentIdsInWakePayload(receipt.payload, [String(pending.payload.commentId)]),
|
||||
requestedAt: new Date(Date.now() - 60_000) }).where(eq(agentWakeupRequests.id, pending.id));
|
||||
expect(receipt.requestedByActorType).toBe("user");
|
||||
await db.update(issues).set({ status: "blocked", executionRunId: null }).where(eq(issues.id, task.id));
|
||||
expect(await taskInterruptionPresentation(db, { ...task, status: "blocked" }))
|
||||
.toMatchObject({ state: "failed", canRetry: true, intentId: pending.id, observedRunId: run.id });
|
||||
await db.update(agents).set({ status: "running", runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } })
|
||||
.where(eq(agents.id, f.assignedAgentId));
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "running", startedAt: new Date() });
|
||||
f.wakeup.mockImplementation(heartbeatService(db).wakeup);
|
||||
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: task.id, actorId: "owner-user",
|
||||
actorSource: "cloud_tenant", observedRunId: run.id, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry });
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId: f.companyId });
|
||||
const [resumed] = await readAgentWakeupRequestsById(pending.id);
|
||||
expect(resumed.runId).toBeTruthy();
|
||||
expect((await readHeartbeatRunsById(resumed.runId!))[0]).toMatchObject({ status: "queued", retryOfRunId: null,
|
||||
contextSnapshot: { wakeCommentIds: [pending.payload.commentId] } });
|
||||
expect((await readAgentWakeupRequestsById(retry.requestId!))[0]).toMatchObject({ status: "coalesced", runId: resumed.runId });
|
||||
});
|
||||
|
||||
it("admits signed PR events into an ordinary assigned task and deduplicates delivery", async () => {
|
||||
const f = await reviewBotFixture();
|
||||
const { buildSkillMentionHref } = await import("@paperclipai/shared");
|
||||
@@ -62866,7 +62931,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
const commentsBeforeDisabledLifecycle = await db
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.issueId, conversation.issueId));
|
||||
.where(eq(issueComments.issueId, conversation.issueId))
|
||||
.orderBy(issueComments.id);
|
||||
const priorLifecycleIds = new Set(
|
||||
(
|
||||
await db
|
||||
@@ -62946,7 +63012,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
db
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.issueId, conversation.issueId)),
|
||||
.where(eq(issueComments.issueId, conversation.issueId))
|
||||
.orderBy(issueComments.id),
|
||||
).resolves.toEqual(commentsBeforeDisabledLifecycle);
|
||||
await service.shutdown();
|
||||
});
|
||||
@@ -64140,6 +64207,163 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
return { ...f, pending, originalDelivery };
|
||||
}
|
||||
|
||||
async function successfulGitHubQueuedInputFixture() {
|
||||
const f = await currentGitHubTaskRetryFixture(true);
|
||||
const companyId = f.fixture.companyId, issueId = f.issue.id;
|
||||
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(saved.payload,
|
||||
[String(f.pending!.payload.commentId)]) }).where(eq(agentWakeupRequests.id, saved.id));
|
||||
const contractId = randomUUID(), resultId = randomUUID();
|
||||
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
|
||||
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {},
|
||||
canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, nativePhase: "committed",
|
||||
completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const [result] = await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: f.runId,
|
||||
completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId,
|
||||
resultJson: { terminal: { runTerminalState: "succeeded" }, summary: "Earlier review remains saved" } }).returning();
|
||||
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.receipt.id));
|
||||
const input = { db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant",
|
||||
observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry };
|
||||
return { ...f, input, result };
|
||||
}
|
||||
|
||||
it("retries saved GitHub input after a successful run without replaying that run", async () => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const { input } = f;
|
||||
const companyId = f.fixture.companyId;
|
||||
const before = (await readHeartbeatRunsById(f.runId))[0];
|
||||
const [request, duplicate] = await Promise.all([requestTaskRetry(input),
|
||||
requestTaskRetry({ ...input, idempotencyKey: randomUUID() })]);
|
||||
expect(request.accepted).toBe(true);
|
||||
expect(duplicate.requestId).toBe(request.requestId);
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
const [source] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
expect(source.runId).toBeTruthy();
|
||||
const [successor] = await readHeartbeatRunsById(source.runId!);
|
||||
expect(successor).toMatchObject({ status: "queued", retryOfRunId: null,
|
||||
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
expect((await readAgentWakeupRequestsById(request.requestId!))[0])
|
||||
.toMatchObject({ status: "coalesced", runId: successor.id });
|
||||
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
|
||||
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
|
||||
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it.each(["connection_revoked", "source_rebound", "task_completed", "agent_paused", "cleanup_pending"] as const)(
|
||||
"keeps saved-input Retry behind current admission gates: %s", async gate => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const companyId = f.fixture.companyId;
|
||||
const request = await requestTaskRetry(f.input);
|
||||
const before = (await readHeartbeatRunsById(f.runId))[0];
|
||||
if (gate === "connection_revoked") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id));
|
||||
if (gate === "source_rebound") await db.update(chatActions).set({ payload: { ...f.pending!.payload, commentId: randomUUID() } })
|
||||
.where(eq(chatActions.id, f.pending!.id));
|
||||
if (gate === "task_completed") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issue.id));
|
||||
if (gate === "agent_paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId));
|
||||
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
|
||||
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
|
||||
if (gate === "cleanup_pending") {
|
||||
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
|
||||
await db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
|
||||
}
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
|
||||
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(0);
|
||||
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
|
||||
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
|
||||
if (gate === "cleanup_pending") {
|
||||
expect((await readAgentWakeupRequestsById(request.requestId!))[0].status).toBe("deferred_issue_execution");
|
||||
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
|
||||
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
|
||||
}).where(eq(environmentLeases.id, lease.id));
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
|
||||
}
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it.each(["issue.comment.reopen", "boardMessageRetry"] as const)(
|
||||
"resumes saved GitHub input after a successful board turn: %s", async origin => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const [previous] = await readHeartbeatRunsById(f.runId);
|
||||
await db.update(heartbeatRuns).set({ responsibleUserId: f.input.actorId, contextSnapshot: {
|
||||
...previous.contextSnapshot, ...(origin === "boardMessageRetry" ? { boardMessageRetry: true } : { source: origin }),
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const before = (await readHeartbeatRunsById(f.runId))[0];
|
||||
const retry = await requestTaskRetry(f.input);
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId: f.fixture.companyId, requestId: retry.requestId! });
|
||||
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
expect(saved.runId).toBeTruthy();
|
||||
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ retryOfRunId: null,
|
||||
contextSnapshot: { source: "chat:github", wakeCommentIds: [f.pending!.payload.commentId] } });
|
||||
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it("keeps a board message queued separately when saved GitHub Retry resumes after cleanup", async () => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const companyId = f.fixture.companyId;
|
||||
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
|
||||
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
|
||||
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
|
||||
await db.update(issues).set({ status: "in_review", executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
|
||||
const retry = await requestTaskRetry(f.input);
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
|
||||
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
|
||||
const [comment] = await db.insert(issueComments).values({ companyId, issueId: f.issue.id,
|
||||
authorType: "user", authorUserId: f.input.actorId, body: "A separate board follow-up" }).returning();
|
||||
const [board] = await db.insert(agentWakeupRequests).values({ companyId, agentId: f.fixture.assignedAgentId,
|
||||
source: "automation", triggerDetail: "system", reason: "issue_commented", status: "deferred_issue_execution",
|
||||
requestedByActorType: "user", requestedByActorId: f.input.actorId,
|
||||
payload: withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, commentId: comment.id,
|
||||
_paperclipWakeContext: { issueId: f.issue.id, source: "issue.comment.reopen", wakeReason: "issue_commented" },
|
||||
}, [comment.id]),
|
||||
}).returning();
|
||||
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
|
||||
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
|
||||
}).where(eq(environmentLeases.id, lease.id));
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
|
||||
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
expect(saved.runId).toBeTruthy();
|
||||
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ status: "queued",
|
||||
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
|
||||
expect((await readAgentWakeupRequestsById(board.id))[0]).toEqual(board);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it("dispatches saved input through the task Retry HTTP endpoint and survives a repeated click", async () => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
const { issueRoutes } = await import("../routes/issues.js");
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => { req.actor = boardActor(f.fixture.companyId); next(); });
|
||||
app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service }));
|
||||
app.use(errorHandler);
|
||||
try {
|
||||
const body = { observedRunId: f.runId, idempotencyKey: randomUUID() };
|
||||
const first = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
|
||||
expect(first.status, JSON.stringify(first.body)).toBe(202);
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
|
||||
const second = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
|
||||
expect(second.status).toBe(202);
|
||||
expect(second.body.requestId).toBe(first.body.requestId);
|
||||
expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
|
||||
const detail = await request(app).get(`/api/issues/${f.issue.id}`);
|
||||
expect(detail.status).toBe(200);
|
||||
expect(detail.body.interruption).toMatchObject({ state: "working", canRetry: false });
|
||||
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)(
|
||||
"admits current GitHub task Retry without replaying obsolete input: %s", async kind => {
|
||||
const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input"));
|
||||
@@ -64269,7 +64493,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
status: "active", membershipRole: "operator" });
|
||||
}
|
||||
const queryEvents: Record<string, unknown>[] = [];
|
||||
const capture = mode === "backlog_51" ? createDatabaseResponseCapture({ start: Date.now() - 1,
|
||||
const capture = mode === "backlog_51" ? createFullDatabaseResponseCapture({ start: Date.now() - 1,
|
||||
end: Date.now() + 59_000 }, event => queryEvents.push(event)) : undefined;
|
||||
if (capture) installDatabaseResponseCapture(db, { chatActions, chatDeliveries, chatMessageLinks, issueComments,
|
||||
agentWakeupRequests, chatEndpoints, chatConversations }, capture);
|
||||
@@ -64289,7 +64513,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
idempotencyKey: mode === "concurrent_clicks" ? randomUUID() : input.idempotencyKey })]);
|
||||
if (capture) {
|
||||
capture.flush();
|
||||
expect(Number(queryEvents.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.startedQueries))
|
||||
expect(Number(queryEvents.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.startedQueries))
|
||||
.toBeLessThan(180);
|
||||
}
|
||||
expect(two.requestId).toBe(one.requestId);
|
||||
@@ -67489,7 +67713,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
})));
|
||||
const scope = { companyId: context.fixture.companyId, issueId: context.issue.id };
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
|
||||
let scopes: Array<{ companyId: string; issueId: string }>;
|
||||
try {
|
||||
@@ -67506,7 +67730,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
}
|
||||
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
|
||||
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
|
||||
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
|
||||
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
|
||||
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
|
||||
} finally {
|
||||
await context.service.shutdown(); await other.service.shutdown();
|
||||
@@ -77214,7 +77438,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
})));
|
||||
const scope = { companyId: context.fixture.companyId, issueId: context.conversation.issueId };
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
|
||||
let scopes: Array<{ companyId: string; issueId: string }>;
|
||||
try {
|
||||
@@ -77231,7 +77455,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
}
|
||||
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
|
||||
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
|
||||
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
|
||||
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
|
||||
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
|
||||
} finally {
|
||||
await context.service.shutdown(); await other.service.shutdown();
|
||||
|
||||
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { and, asc, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs,
|
||||
companies, createDb, documents, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
|
||||
companies, createDb, documents, environmentLeases, executionWorkspaces, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues, workspaceOperations } from "@paperclipai/db";
|
||||
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
|
||||
import { getExecutionBlocker } from "../services/execution-blocker.js";
|
||||
@@ -15,6 +15,8 @@ import { documentService } from "../services/documents.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { chatCompletionDeliveryService, isCompletedOnboardingHandoffWake, prepareChatCompletionTurn, recordChatCompletion, recordChatHandoff } from "../services/chat-completion-delivery.js";
|
||||
import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat.js";
|
||||
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
|
||||
(support.supported ? describe : describe.skip)("chat completion delivery", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
@@ -68,6 +70,75 @@ import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../ser
|
||||
})]);
|
||||
expect(JSON.stringify(run.contextSnapshot?.chatCompletionUpdates)).not.toMatch(/Injected|credentials|Plan accepted|Agent approved/);
|
||||
});
|
||||
it("retains a distinct completion receipt behind a real dependency gate", async () => {
|
||||
const f = await seed();
|
||||
const blocker = await f.create();
|
||||
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [blocker.id] });
|
||||
await f.finish();
|
||||
const heartbeat = heartbeatService(db);
|
||||
// The generic dependency signal is replaceable. The outbox's exact wake
|
||||
// receipt is not: it needs that identity to reconcile a refused admission.
|
||||
await heartbeat.wakeup(f.agentId, {
|
||||
source: "automation", triggerDetail: "system", reason: "issue_continuation_needed",
|
||||
requestedByActorType: "system", requestedByActorId: "dependency-test",
|
||||
payload: { issueId: f.sourceId }, contextSnapshot: { issueId: f.sourceId },
|
||||
});
|
||||
const [delivery] = await f.rows();
|
||||
const service = chatCompletionDeliveryService(db, heartbeat);
|
||||
await service.deliver(delivery.id);
|
||||
const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId));
|
||||
expect(receipts).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ idempotencyKey: `chat-completion:${delivery.id}:0`,
|
||||
reason: "issue_dependencies_blocked", status: "skipped", runId: null,
|
||||
payload: expect.objectContaining({ chatCompletionDeliveryIds: [delivery.id] }) }),
|
||||
]));
|
||||
expect(receipts).toHaveLength(2);
|
||||
expect(receipts.find(receipt => receipt.idempotencyKey?.startsWith("execution-wait:")))
|
||||
.toMatchObject({ coalescedCount: 0 });
|
||||
expect((await f.rows())[0]).toMatchObject({ attempts: 0, status: "pending", targetRunId: null, error: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
|
||||
expect(await issueService(db).getDependencyReadiness(f.sourceId)).toMatchObject({
|
||||
isDependencyReady: false, unresolvedBlockerIssueIds: [blocker.id],
|
||||
});
|
||||
});
|
||||
it("admits a saved worker completion after its owning finalize encloses the snapshot", async () => {
|
||||
const f = await seed(), workspaceId = randomUUID(), workerRunId = randomUUID();
|
||||
await db.update(issues).set({ responsibleUserId: "operator" }).where(eq(issues.id, f.sourceId));
|
||||
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId,
|
||||
mode: "isolated_workspace", strategyType: "managed_remote", name: "Saved worker", providerType: "daytona" });
|
||||
await db.insert(heartbeatRuns).values({ id: workerRunId, companyId: f.companyId, agentId: f.agentId,
|
||||
status: "succeeded", contextSnapshot: { issueId: f.task.id } });
|
||||
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.task.id));
|
||||
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [f.task.id] });
|
||||
await db.insert(workspaceOperations).values([
|
||||
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
|
||||
phase: "workspace_finalize", status: "succeeded", startedAt: new Date(10_000), finishedAt: new Date(12_000) },
|
||||
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
|
||||
phase: "remote_workspace_snapshot", status: "succeeded", startedAt: new Date(10_100), finishedAt: new Date(11_900) },
|
||||
]);
|
||||
await f.finish();
|
||||
const deps = createHeartbeatQueueTestDependencies(db), issueSvc = issueService(db);
|
||||
deps.getAgent.mockImplementation(id => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null));
|
||||
deps.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(db).getExperimental());
|
||||
deps.issuesSvc.getDependencyReadiness.mockImplementation(issueSvc.getDependencyReadiness);
|
||||
deps.issuesSvc.listDependencyReadiness.mockImplementation(issueSvc.listDependencyReadiness);
|
||||
// Admit the real durable run, then hold provider dispatch at the normal drain boundary.
|
||||
deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" });
|
||||
const queue = createHeartbeatQueue(db, deps);
|
||||
const service = chatCompletionDeliveryService(db, { wakeup: queue.trackWakeup });
|
||||
const [delivery] = await f.rows();
|
||||
await service.deliver(delivery.id);
|
||||
const [queued] = await f.rows();
|
||||
expect(queued).toMatchObject({ status: "queued", attempts: 0, error: null, targetRunId: expect.any(String) });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.targetRunId!));
|
||||
expect(run).toMatchObject({ status: "queued", contextSnapshot: { issueId: f.sourceId,
|
||||
wakeReason: "chat_task_completed", chatCompletionDeliveryIds: [delivery.id] } });
|
||||
const prepared = await prepareChatCompletionTurn(db, run);
|
||||
expect(prepared.contextSnapshot?.chatCompletionUpdates).toEqual([
|
||||
expect.objectContaining({ id: f.task.id, status: "done", completedAt: expect.any(Date), url: `/issues/${f.task.identifier ?? f.task.id}` }),
|
||||
]);
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
});
|
||||
it("dispatches a committed completion without activity publication and recovers after restart", async () => {
|
||||
const f = await seed();
|
||||
const run = vi.fn(() => f.service.sweepPending());
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
projectWorkspaces,
|
||||
} from "@paperclipai/db";
|
||||
import { parseFrontmatterMarkdown } from "@paperclipai/shared";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
@@ -651,17 +651,17 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
]);
|
||||
await svc.list(companyId);
|
||||
const events: Record<string, unknown>[] = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { agents, costEvents }, capture);
|
||||
let listed: Awaited<ReturnType<typeof svc.list>>;
|
||||
try { listed = await svc.list(companyId); } finally { capture.flush(); }
|
||||
expect(listed.find(item => item.id === first.id)?.attachedAgentCount).toBe(2);
|
||||
expect(listed.find(item => item.id === second.id)?.attachedAgentCount).toBe(1);
|
||||
expect(listed.find(item => item.id === ambiguous.id)?.attachedAgentCount).toBe(0);
|
||||
const agentReads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "agents");
|
||||
const agentReads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "agents");
|
||||
expect(agentReads).toHaveLength(1);
|
||||
expect(Number(agentReads[0]?.responseValueBytes)).toBeLessThan(2_000);
|
||||
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
|
||||
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
|
||||
await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [second.key] } } }).where(eq(agents.id, readerId));
|
||||
const refreshed = await svc.list(companyId, { q: "First" });
|
||||
expect(refreshed.find(item => item.id === first.id)?.attachedAgentCount).toBe(1);
|
||||
@@ -680,12 +680,12 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
const before = await svc.list(companyId);
|
||||
expect(before.find(item => item.id === inserted[0]!.id)?.folderPath).toBe("bundled/paperclip-core");
|
||||
const events: Record<string, unknown>[] = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkills }, capture);
|
||||
let after: Awaited<ReturnType<typeof svc.list>>;
|
||||
try { after = await svc.list(companyId); } finally { capture.flush(); }
|
||||
expect(after).toEqual(before);
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "company_skills");
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skills");
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(950_000);
|
||||
const stored = await svc.getById(companyId, inserted[0]!.id);
|
||||
expect(stored?.markdown).toBe(savedMarkdown);
|
||||
@@ -762,11 +762,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
.orderBy(companySkillVersions.id);
|
||||
expect(before.length).toBeGreaterThan(0);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
|
||||
try { expect(await companySkillService(db).list(companyId)).toEqual(expected); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "company_skill_versions" && event.operation === "select");
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
@@ -783,11 +783,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
? { type, companyId: otherCompanyId, agentId: randomUUID() }
|
||||
: { type, companyIds: [otherCompanyId] };
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
|
||||
try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); }
|
||||
finally { capture.flush(); }
|
||||
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0);
|
||||
expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.completedQueries).toBe(0);
|
||||
});
|
||||
|
||||
it("restores a deleted release after warm validation", async () => {
|
||||
|
||||
@@ -6,9 +6,10 @@ import { join } from "node:path";
|
||||
import { createServer } from "node:http";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
|
||||
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
|
||||
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, aiConnectionPools, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
|
||||
import { findSatisfiedToolConnection, satisfiedConnectionIntentService } from "../services/satisfied-connection-intents.js";
|
||||
import { connectionManagementService, continuingConnectionManagement } from "../services/connection-management.js";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import { redactRunnerApiPayloads } from "../services/native-runtime/runner-api-log-redaction.js";
|
||||
import * as remoteEndpointGuard from "../services/remote-http-endpoint-guard.js";
|
||||
import * as remoteHttpFetch from "../services/remote-http-fetch.js";
|
||||
@@ -87,13 +88,14 @@ describe("connection management with production default agents", () => {
|
||||
authority: new PaperclipRunnerToolAuthority(server.db, { companyId: base.companyId, agentId: agent.id, issueId: base.issueId, runId: base.runId, apiUrl: server.apiUrl, storage: server.storage }) };
|
||||
}
|
||||
|
||||
it("matches ordinary install capabilities on real PostgreSQL for scoped, retained and fallback resources", async () => {
|
||||
it.each(["tool", "ai"] as const)("matches ordinary %s install capabilities on real PostgreSQL for scoped, retained and fallback resources", async (connectionPurpose) => {
|
||||
const f = await server.fixture({ connectionScenario: "fresh" });
|
||||
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
|
||||
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Capability target ${index}`, adapterType: "process" })));
|
||||
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Capability differential", type: "mcp_http" }).returning();
|
||||
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Capability differential", transport: "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
|
||||
uid: randomUUID(), name: "Capability differential", connectionPurpose,
|
||||
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
|
||||
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
|
||||
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds.slice(0, 2) } });
|
||||
const service = connectionManagementService(server.db);
|
||||
@@ -177,7 +179,7 @@ describe("connection management with production default agents", () => {
|
||||
expect(await faulting.describeAgentInstallCapabilities(localActor, resource, [candidateIds[1]])).toEqual([denied]);
|
||||
});
|
||||
|
||||
it("preserves the complete grants GET across human roles and denies a real install mutation after revocation", async () => {
|
||||
it.each(["tool", "ai"] as const)("preserves the complete %s grants GET across human roles and denies a real install mutation after revocation", async (connectionPurpose) => {
|
||||
const f = await server.fixture({ connectionScenario: "fresh" });
|
||||
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
|
||||
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Grant target ${index}`, adapterType: "process" })));
|
||||
@@ -198,7 +200,8 @@ describe("connection management with production default agents", () => {
|
||||
scope: permissionKey === "tools:manage_profiles" ? { targetAgentIds: candidateIds.slice(0, 2) } : null })));
|
||||
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Grants response parity", type: "mcp_http" }).returning();
|
||||
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Grants response parity", transport: "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
|
||||
uid: randomUUID(), name: "Grants response parity", connectionPurpose,
|
||||
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
|
||||
const grants = await server.db.insert(connectionGrants).values([
|
||||
{ companyId: f.companyId, connectionId: connection.id, kind: "organization", createdByUserId: users.creator, isDefault: true },
|
||||
...[users.creator, users.manager, users.member].map((subjectUserId, index) => ({ companyId: f.companyId, connectionId: connection.id, kind: "user" as const,
|
||||
@@ -253,6 +256,65 @@ describe("connection management with production default agents", () => {
|
||||
expect(savedInstalls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("checks company-wide personal AI installs once per target and preserves pool member authorization", async () => {
|
||||
const f = await server.fixture({ connectionScenario: "fresh" });
|
||||
const candidateIds = [f.agentId, ...Array.from({ length: 19 }, () => randomUUID())];
|
||||
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `AI target ${index}`, adapterType: "process" })));
|
||||
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "AI capability checks", type: "mcp_http" }).returning();
|
||||
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Personal AI account", transport: "runtime_auth", connectionPurpose: "ai", authKind: "api_key",
|
||||
credentialPolicy: "per_user", createdByUserId: f.responsibleUserId }).returning();
|
||||
const [grant] = await server.db.insert(connectionGrants).values({ companyId: f.companyId, connectionId: connection.id,
|
||||
kind: "user", subjectUserId: f.responsibleUserId, createdByUserId: f.responsibleUserId }).returning();
|
||||
await server.db.insert(toolConnectionInstalls).values({ companyId: f.companyId, connectionId: connection.id, targetType: "company", targetId: f.companyId });
|
||||
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
|
||||
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds } });
|
||||
let targetReads = 0;
|
||||
const observedDb = new Proxy(server.db, { get(target, key, receiver) {
|
||||
if (key !== "select") return Reflect.get(target, key, receiver);
|
||||
return (...fields: Parameters<typeof target.select>) => {
|
||||
const builder = target.select(...fields), from = builder.from.bind(builder);
|
||||
builder.from = ((...args: Parameters<typeof from>) => {
|
||||
const query = from(...args), execute = query.execute.bind(query);
|
||||
query.execute = (async (...executeArgs: Parameters<typeof execute>) => {
|
||||
if (query.toSQL().sql.includes('"agents"."id" =')) targetReads++;
|
||||
return execute(...executeArgs);
|
||||
}) as typeof execute;
|
||||
return query;
|
||||
}) as typeof builder.from;
|
||||
return builder;
|
||||
};
|
||||
} });
|
||||
const service = connectionManagementService(observedDb);
|
||||
const actor: AuthorizationActor = { type: "board", userId: f.responsibleUserId, source: "cloud_tenant" };
|
||||
const resource = { type: "connection_management" as const, companyId: f.companyId, connectionId: connection.id, operation: "configure", readOnly: false };
|
||||
const baseline = [];
|
||||
for (const targetAgentId of candidateIds) baseline.push(await service.capability(actor, "connections:profiles", { ...resource, targetAgentId }));
|
||||
expect(baseline.every(result => result.allowed)).toBe(true);
|
||||
expect(targetReads).toBe(candidateIds.length ** 2);
|
||||
targetReads = 0;
|
||||
expect(await service.describeAgentInstallCapabilities(actor, resource, candidateIds)).toEqual(baseline);
|
||||
expect(targetReads).toBe(candidateIds.length);
|
||||
|
||||
const [pool] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Pool with private member", transport: "runtime_auth", connectionPurpose: "ai", authKind: "none" }).returning();
|
||||
await server.db.insert(aiConnectionPools).values({ id: pool.id, companyId: f.companyId, pluginKey: "fixture-pool", config: {
|
||||
name: "Pool", enabled: true, mode: "round_robin", thresholdPercent: 90,
|
||||
members: [{ id: randomUUID(), binding: { mode: "delegated", provider: "openai", method: "api_key", connectionId: connection.id, grantId: grant.id }, profile: { provider: "codex", model: "fixture-model" } }],
|
||||
} });
|
||||
await server.db.delete(principalPermissionGrants).where(and(eq(principalPermissionGrants.companyId, f.companyId), eq(principalPermissionGrants.principalId, f.responsibleUserId!)));
|
||||
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
|
||||
permissionKey: "tools:manage_profiles", scope: { connectionIds: [pool.id] } });
|
||||
const poolResource = { ...resource, connectionId: pool.id };
|
||||
expect((await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).every(result => result.allowed)).toBe(true);
|
||||
// The pool grant cannot replace the member account's own authority.
|
||||
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, grant.id));
|
||||
await server.db.update(toolConnections).set({ createdByUserId: "different-owner" }).where(eq(toolConnections.id, connection.id));
|
||||
const ordinary = await service.capability(actor, "connections:profiles", { ...poolResource, targetAgentId: candidateIds[0] });
|
||||
expect(ordinary.allowed).toBe(false);
|
||||
expect(await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).toEqual([ordinary, ordinary]);
|
||||
});
|
||||
|
||||
it("creates and configures tool connections and profiles with fresh grants and actual key/JWT authentication", async () => {
|
||||
const f = await fixture();
|
||||
const grants = await server.db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, f.agentId));
|
||||
@@ -1642,6 +1704,36 @@ describe("connection management with production default agents", () => {
|
||||
} finally { provider = null; unregisterServerAdapter("paperclip_runner"); }
|
||||
});
|
||||
|
||||
it("reads only the authorized Anthropic account identity through actual agent authentication", async () => {
|
||||
const f = await fixture();
|
||||
const account = await aiConnectionService(server.db).save(f.companyId, f.userId, {
|
||||
provider: "anthropic", method: "subscription", ownership: "personal", name: "Account identity fixture",
|
||||
loginSessionId: "fixture", agentIds: [f.agentId], allAgents: false,
|
||||
}, "selected-identity-credential");
|
||||
let calls = 0;
|
||||
provider = async (url, init) => {
|
||||
calls++;
|
||||
expect(url).toBe("https://api.anthropic.com/api/oauth/profile");
|
||||
expect(new Headers(init?.headers).get("authorization")).toBe("Bearer selected-identity-credential");
|
||||
return Response.json({ account: { uuid: "private-seat", email: "avery@example.test" }, organization: { uuid: "private-org", name: "Example team" } });
|
||||
};
|
||||
const path = `/companies/${f.companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
|
||||
try {
|
||||
for (const token of [f.key.token, f.jwt]) {
|
||||
const response = await f.request("GET", path, undefined, token);
|
||||
expect(response).toMatchObject({ status: 200, body: { ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" } });
|
||||
for (const secret of ["selected-identity-credential", "private-seat", "private-org"]) expect(JSON.stringify(response)).not.toContain(secret);
|
||||
}
|
||||
expect(calls).toBe(2);
|
||||
expect((await f.request("GET", path.replace(f.companyId, f.foreignCompanyId))).status).toBe(403);
|
||||
expect((await f.request("GET", path.replace(account.grantId, randomUUID()))).status).toBe(404);
|
||||
expect(calls).toBe(2);
|
||||
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
|
||||
expect(await f.request("GET", path)).toMatchObject({ status: 200, body: { status: "unavailable", errorCode: "connection_unavailable", accountLabel: null } });
|
||||
expect(calls).toBe(2);
|
||||
} finally { provider = null; }
|
||||
});
|
||||
|
||||
it("filters AI account and pool listings by the agent's resolved connection scope", async () => {
|
||||
const f = await fixture();
|
||||
provider = async url => {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { settleConversationTurnForRun } from "../services/agent-conversations.js";
|
||||
|
||||
@@ -52,11 +52,11 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => {
|
||||
const input = await seed(reply), before = await sourceDigests(input);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture);
|
||||
try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table))
|
||||
&& (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4);
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { QueryPromise } from "drizzle-orm/query-promise";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { withDatabaseOperation } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals";
|
||||
import { databaseOperationMiddleware } from "../middleware/database-operation.js";
|
||||
import { withBudgetSpan } from "../services/budget-tracing.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js";
|
||||
import {
|
||||
createRunPreparationTiming,
|
||||
measureRunPhase,
|
||||
withRunExecutionTiming,
|
||||
withRunResponsePresentationTiming,
|
||||
withRunWorkspaceFinalizationTiming,
|
||||
} from "../services/execution-preparation-timing.js";
|
||||
|
||||
const { owners } = await vi.hoisted(async () => {
|
||||
const { AsyncLocalStorage } = await import("node:async_hooks");
|
||||
return { owners: new AsyncLocalStorage<string>() };
|
||||
});
|
||||
vi.mock("@paperclipai/db", async importOriginal => ({
|
||||
...await importOriginal<typeof import("@paperclipai/db")>(),
|
||||
withDatabaseOperation: <T>(owner: string, action: () => T): T => owners.run(owner, action),
|
||||
}));
|
||||
|
||||
class OwnerQuery extends QueryPromise<string | undefined> {
|
||||
async execute() { return owners.getStore(); }
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it.each(["", "http://collector.test"])("attributes lazy queries to execution, preparation, and budget owners with endpoint %s", async endpoint => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
|
||||
await withDatabaseOperation("http.request", async () => {
|
||||
const query = new OwnerQuery();
|
||||
await expect(withRunExecutionTiming("private-run", () => query)).resolves.toBe("task.execution_attempt");
|
||||
await expect(withRunWorkspaceFinalizationTiming("private-run", () => query)).resolves.toBe("task.workspace_finalization");
|
||||
await expect(withRunResponsePresentationTiming("private-run", () => query)).resolves.toBe("task.response_presentation");
|
||||
await expect(createRunPreparationTiming("private-run")("agent_read", () => query)).resolves.toBe("heartbeat.prepare.agent_read");
|
||||
await expect(createRunPreparationTiming("private-run", "native.controller")("backend_setup", () => query)).resolves.toBe("native.controller.backend_setup");
|
||||
await expect(measureRunPhase("skills.inventory", () => query)).resolves.toBe("skills.inventory");
|
||||
await expect(withBudgetSpan("budget.accounting.transaction", () => query)).resolves.toBe("budget.accounting.transaction");
|
||||
await expect(query).resolves.toBe("http.request");
|
||||
});
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps owner attribution after the span detail limit and restores nested scopes after failure", async () => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
|
||||
const failure = new Error("private accounting failure");
|
||||
await withRunExecutionTiming("private-run", async () => {
|
||||
for (let i = 0; i < 18; i++) {
|
||||
await expect(measureRunPhase("skills.inventory", () => new OwnerQuery())).resolves.toBe("skills.inventory");
|
||||
}
|
||||
await expect(measureRunPhase("execution.budget_reservation", async () => {
|
||||
await expect(withBudgetSpan("budget.admission.reserve", async () => {
|
||||
await Promise.resolve();
|
||||
expect(owners.getStore()).toBe("budget.admission.reserve");
|
||||
throw failure;
|
||||
})).rejects.toBe(failure);
|
||||
return new OwnerQuery();
|
||||
})).resolves.toBe("execution.budget_reservation");
|
||||
await expect(new OwnerQuery()).resolves.toBe("task.execution_attempt");
|
||||
});
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps concurrent HTTP requests and nested work in fixed scopes without using private URLs", async () => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "");
|
||||
const app = express();
|
||||
app.use(databaseOperationMiddleware);
|
||||
app.use(express.json());
|
||||
let arrived = 0;
|
||||
let release!: () => void;
|
||||
const both = new Promise<void>(resolve => { release = resolve; });
|
||||
app.all("/:privateId", async (req, res) => {
|
||||
const before = await new OwnerQuery();
|
||||
const work = async () => {
|
||||
if (++arrived === 2) release();
|
||||
await both;
|
||||
return new OwnerQuery();
|
||||
};
|
||||
const during = req.params.privateId === "first"
|
||||
? await withBudgetSpan("budget.admission.reserve", work)
|
||||
: await measureRunPhase("skills.inventory", work);
|
||||
res.json({ before, during, after: await new OwnerQuery() });
|
||||
});
|
||||
const [first, second] = await Promise.all([
|
||||
request(app).get("/first?secret=private-first").expect(200),
|
||||
request(app).post("/second?secret=private-second").send({ private: "request-body" }).expect(200),
|
||||
]);
|
||||
expect(first.body).toEqual({ before: "http.request", during: "budget.admission.reserve", after: "http.request" });
|
||||
expect(second.body).toEqual({ before: "http.request", during: "skills.inventory", after: "http.request" });
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("replaces producer ownership on each delivery dispatch, including retry and pending checks", async () => {
|
||||
vi.useFakeTimers();
|
||||
const execute = vi.fn(async () => [{ xid: "42" }]);
|
||||
const database = installDatabaseWorkSignals({ execute, async transaction<T>(action: (tx: object) => Promise<T>): Promise<T> {
|
||||
return action({ execute });
|
||||
} });
|
||||
const seen: Array<{ queue: string; phase: string; owner: string | undefined }> = [];
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: database, canRun: () => true,
|
||||
onError: (_error, queue) => { seen.push({ queue, phase: "failure", owner: owners.getStore() }); },
|
||||
});
|
||||
let fail = true;
|
||||
try {
|
||||
const workers = withDatabaseOperation("http.request", () => [DELIVERY_QUEUES.feedback, DELIVERY_QUEUES.email].map(queue =>
|
||||
coordinator.register(queue, { retryMs: 5000,
|
||||
run: async () => {
|
||||
await Promise.resolve();
|
||||
seen.push({ queue, phase: "run", owner: await new OwnerQuery() });
|
||||
if (queue === DELIVERY_QUEUES.feedback && fail) { fail = false; throw new Error("retry"); }
|
||||
},
|
||||
hasPending: async () => {
|
||||
seen.push({ queue, phase: "pending", owner: await new OwnerQuery() });
|
||||
return false;
|
||||
},
|
||||
})));
|
||||
await Promise.all(workers.map(worker => worker.ready));
|
||||
await withDatabaseOperation("budget.accounting.transaction", () => database.transaction(tx =>
|
||||
notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback)));
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(seen.map(({ phase }) => phase).sort()).toEqual(["failure", "pending", "pending", "run", "run", "run"]);
|
||||
for (const entry of seen) expect(entry.owner).toBe(`delivery.${entry.queue}`);
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
expect(coordinator.nextWakeAt()).toBeNull();
|
||||
} finally {
|
||||
await coordinator.stop();
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
|
||||
import { createOwnedSandbox } from "../services/sandbox-creation-operations.js";
|
||||
import { createOwnedSandbox, recordSandboxCreationResource } from "../services/sandbox-creation-operations.js";
|
||||
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -116,15 +116,18 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
|
||||
]));
|
||||
await quiet();
|
||||
});
|
||||
it("settles a deleted-company creation as manual history across restart without idle polling", async () => {
|
||||
it.each([false, true])("settles a deleted-company creation as manual history across restart without idle polling (contextual=%s)", async contextual => {
|
||||
await start(); await quiet();
|
||||
await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
|
||||
const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {},
|
||||
metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} },
|
||||
async () => ({ providerLeaseId: "fixture-deleted-company" }));
|
||||
await db.delete(companies).where(eq(companies.id, companyId));
|
||||
await settle(31 * 60_000);
|
||||
await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
|
||||
expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
|
||||
]));
|
||||
if (contextual) await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId,
|
||||
providerLeaseId: created.providerLeaseId });
|
||||
await quiet();
|
||||
worker!.stop(); await coordinator!.stop();
|
||||
await start(); await quiet();
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db";
|
||||
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db";
|
||||
import { resolvePluginTaskResume } from "../services/environment-task-runtime.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
@@ -20,6 +21,108 @@ databaseSuite("environment task lease authority", () => {
|
||||
});
|
||||
afterAll(async () => { await stopDb?.(); });
|
||||
|
||||
it("replaces a stopped warm candidate within its tracked startup context", async () => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
|
||||
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
|
||||
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
|
||||
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
|
||||
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
|
||||
const calls: string[] = [];
|
||||
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
|
||||
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void) => {
|
||||
calls.push(method === "environmentTask" ? params.operation.kind : method);
|
||||
try {
|
||||
if (method === "environmentAcquireLease") return { providerLeaseId: "replacement-task" };
|
||||
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
|
||||
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
|
||||
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
|
||||
return { kind: "accepted", taskId: params.taskId };
|
||||
} finally { settled?.(); }
|
||||
},
|
||||
} as unknown as PluginWorkerManager;
|
||||
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
|
||||
try {
|
||||
await expect(environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
|
||||
operation: { kind: "describe" }, executionContext: context })).resolves.toMatchObject({ kind: "description", taskId: "replacement-task" });
|
||||
expect(calls).toEqual(["status", "status", "complete", "environmentAcquireLease", "describe"]);
|
||||
expect(context.pending).toBe(false);
|
||||
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
|
||||
expect(lease.providerLeaseId).toBe("replacement-task");
|
||||
expect(lease.metadata?.runnerTaskReused).toBe(false);
|
||||
expect(JSON.stringify(lease.metadata)).not.toContain(context.id);
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
|
||||
it.each(["late", "post-result", "retiring"])("queues a replacement instead of attaching after %s startup cancellation", async mode => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
|
||||
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
|
||||
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
|
||||
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
|
||||
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
|
||||
const calls: string[] = [];
|
||||
let reply!: () => Promise<void>, acquiring!: () => void;
|
||||
const entered = new Promise<void>(resolve => { acquiring = resolve; });
|
||||
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
|
||||
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void, _contextId?: string, observe?: (result: any) => Promise<void>) => {
|
||||
calls.push(method === "environmentTask" ? params.operation.kind : method);
|
||||
try {
|
||||
if (method === "environmentAcquireLease") return await new Promise(resolve => {
|
||||
reply = async () => { const result = { providerLeaseId: "replacement-task", metadata: { binding: "fresh" } };
|
||||
await observe?.(result);
|
||||
if (mode === "retiring") await db.update(environmentLeases).set({ metadata: { driver: "plugin", pluginId, driverKey: "tasks", runnerTaskRetiring: true } }).where(eq(environmentLeases.id, leaseId));
|
||||
resolve(result); };
|
||||
acquiring();
|
||||
});
|
||||
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
|
||||
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
|
||||
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
|
||||
return { kind: "accepted", taskId: params.taskId };
|
||||
} finally { settled?.(); }
|
||||
},
|
||||
} as unknown as PluginWorkerManager;
|
||||
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
|
||||
if (mode === "post-result") {
|
||||
const wait = context.wait.bind(context);
|
||||
vi.spyOn(context, "wait").mockImplementation(operation => wait(async () => {
|
||||
const value = await operation(); context.abort(new Error("Cancelled after recording")); return value;
|
||||
}));
|
||||
}
|
||||
try {
|
||||
const request = environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
|
||||
operation: { kind: "describe" }, executionContext: context });
|
||||
const rejected = expect(request).rejects.toThrow("operation unavailable");
|
||||
await entered;
|
||||
if (mode === "late") {
|
||||
context.abort(new Error("Startup cancelled"));
|
||||
await rejected;
|
||||
expect(context.pending).toBe(true);
|
||||
}
|
||||
await reply(); await rejected;
|
||||
expect(context.pending).toBe(false);
|
||||
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
|
||||
expect(lease.providerLeaseId).toBe("stopped-task");
|
||||
const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.runId, runId));
|
||||
expect(creation.state).toBe("cleanup_pending");
|
||||
expect(creation.metadata.runnerTaskCreationContextId).toBe(context.id);
|
||||
const [cleanup] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, creation.leaseId!));
|
||||
expect(cleanup).toMatchObject({ heartbeatRunId: runId, providerLeaseId: "replacement-task", status: "pending_cleanup" });
|
||||
expect(cleanup.metadata).toMatchObject({ runnerTaskLease: true, providerMetadata: { binding: "fresh" } });
|
||||
expect(calls).not.toContain("describe");
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
|
||||
it("authorizes a new task lease only after scoped prior execution has stopped", async () => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID();
|
||||
const oldRunId = randomUUID(), nextRunId = randomUUID(), oldLeaseId = randomUUID(), nextLeaseId = randomUUID();
|
||||
|
||||
@@ -2,14 +2,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { executionWorkspaces, environmentLeases, type Db } from "@paperclipai/db";
|
||||
import { environmentTaskOperationSchema, parseEnvironmentTaskResult } from "@paperclipai/plugin-sdk";
|
||||
import { executeEnvironmentTask } from "../services/environment-task-runtime.js";
|
||||
import { executeEnvironmentTask, settleDispatchedEnvironmentTask } from "../services/environment-task-runtime.js";
|
||||
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
|
||||
|
||||
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn() }));
|
||||
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn(), identity: vi.fn(), config: vi.fn() }));
|
||||
vi.mock("../services/provider-fresh-workspace.js", () => ({
|
||||
stageProviderFreshWorkspace: async (input: { existing: unknown }) => input.existing,
|
||||
releaseProviderRecoveryTransfers: state.release,
|
||||
}));
|
||||
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin) }) }));
|
||||
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin), getConfig: state.config }) }));
|
||||
vi.mock("../services/plugin-agent-runtime-state.js", () => ({ pluginAgentRuntimeIdentity: state.identity }));
|
||||
const leaseId = "10000000-0000-4000-8000-000000000001";
|
||||
const projectId = "10000000-0000-4000-8000-000000000002";
|
||||
const secondProjectId = "10000000-0000-4000-8000-000000000003";
|
||||
@@ -25,7 +27,7 @@ function database(value: ReturnType<typeof row> | null = row(), projectRows = [{
|
||||
const query: any = { from: (table: unknown) => { selectedRows = table === executionWorkspaces ? workspaceRows : table === environmentLeases ? ownerRows : projectRows; return query; }, leftJoin: () => query, where: vi.fn(() => query), limit: () => Promise.resolve(results.shift()), then: (resolve: (rows: typeof projectRows) => unknown) => Promise.resolve(selectedRows).then(resolve as (rows: unknown[]) => unknown) };
|
||||
const sets: Record<string, unknown>[] = [];
|
||||
const update: any = { set: (value: Record<string, unknown>) => { sets.push(value); return update; }, where: () => update,
|
||||
returning: async () => [{ id: leaseId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
|
||||
returning: async () => [{ id: leaseId, metadata: value?.lease.metadata, heartbeatRunId: value?.lease.heartbeatRunId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
|
||||
const db = { select: () => query, update: () => update,
|
||||
transaction: vi.fn(async (write: (tx: Db) => Promise<unknown>) => write(db as unknown as Db)) };
|
||||
return { db: db as unknown as Db, query, sets, transaction: db.transaction };
|
||||
@@ -37,9 +39,65 @@ const submit = { kind: "submit" as const, preparedLaunchId: "configuration", pro
|
||||
|
||||
beforeEach(() => {
|
||||
state.release.mockReset();
|
||||
state.identity.mockReset().mockResolvedValue({ instanceSecrets: { launcher: "synthetic-transient-binding" } });
|
||||
state.config.mockReset().mockResolvedValue({ configJson: { endpoint: "synthetic" } });
|
||||
state.plugin = { id: "original", pluginKey: "test.provider", status: "ready", manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } };
|
||||
});
|
||||
describe("plugin-provided Runner execution", () => {
|
||||
it.each([false, true])("retires an undispatched warm task only with required preparation proof (%s)", async preparationStopped => {
|
||||
const value = row();
|
||||
Object.assign(value.lease.metadata, { runnerTaskDispatched: false,
|
||||
runnerTaskStartupCleanup: { contextId: "startup", requiresPreparationStop: true } });
|
||||
const fixture = database(value), workers = worker();
|
||||
fixture.db.select = ((...args: unknown[]) => (database(value).db.select as (...values: unknown[]) => unknown)(...args)) as Db["select"];
|
||||
let release!: () => void;
|
||||
const drain = new Promise<void>(resolve => { release = resolve; });
|
||||
const drainStartup = vi.fn(() => drain);
|
||||
Object.assign(workers, { drainStartup });
|
||||
const operations: string[] = [];
|
||||
workers.call.mockImplementation(async (...args: any[]) => {
|
||||
const operation = args[2].operation.kind; operations.push(operation);
|
||||
return operation === "status" ? { kind: "status", taskId: "attempt-1", phase: "cancelled", executionStopped: true, preparationStopped }
|
||||
: { kind: "accepted", taskId: "attempt-1" };
|
||||
});
|
||||
const retiring = settleDispatchedEnvironmentTask(fixture.db, workers as never, { companyId: "company", id: leaseId });
|
||||
const checked = retiring.then(value => ({ value, error: null }), error => ({ value: false, error }));
|
||||
await vi.waitFor(() => expect(operations).toContain("stop"));
|
||||
await vi.waitFor(() => expect(operations).toContain("status"));
|
||||
expect(drainStartup).toHaveBeenCalledWith("startup");
|
||||
expect(operations).not.toContain("complete");
|
||||
release();
|
||||
const result = await checked;
|
||||
if (preparationStopped) { expect(result).toEqual({ value: true, error: null }); expect(operations).toEqual(["stop", "status", "complete"]); }
|
||||
else { expect(result.error?.message).toContain("termination is unconfirmed"); expect(fixture.sets).toContainEqual(expect.objectContaining({ status: "pending_cleanup", cleanupStatus: "failed" })); }
|
||||
});
|
||||
it("keeps legacy payloads passive without capability or transport activation", async () => {
|
||||
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: "run" } } as never, 1_000);
|
||||
const workers = worker();
|
||||
workers.call.mockImplementation(async (...args: any[]) => { args[5]?.(); return { kind: "description", taskId: "attempt-1", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; });
|
||||
try {
|
||||
for (const executionContext of [undefined, context]) {
|
||||
await executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId,
|
||||
operation: { kind: "describe" }, ...(executionContext ? { executionContext } : {}) });
|
||||
const sent = workers.call.mock.calls.at(-1) as unknown as any[];
|
||||
expect(sent[2]).not.toHaveProperty("executionContextSupported");
|
||||
expect(sent[2]).not.toHaveProperty("executionContext");
|
||||
if (!executionContext) expect(sent).toHaveLength(4);
|
||||
}
|
||||
expect(state.identity).not.toHaveBeenCalled();
|
||||
expect(state.config).not.toHaveBeenCalled();
|
||||
expect(context.pending).toBe(false);
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
it.each(["json", "wrong-run"])("rejects a %s context before the worker", async mode => {
|
||||
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: mode === "wrong-run" ? "other" : "run" } } as never, 1_000);
|
||||
const workers = worker();
|
||||
try {
|
||||
await expect(executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId, operation: { kind: "describe" },
|
||||
executionContext: mode === "json" ? JSON.parse(JSON.stringify(context)) : context })).rejects.toThrow("native_execution_context");
|
||||
expect(workers.call).not.toHaveBeenCalled();
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
it("forwards a retained fresh directory intent on later launch calls and checks provider support", async () => {
|
||||
const value = row();
|
||||
value.lease.metadata.runnerTaskProjectAuthValidation.projectIds = [];
|
||||
|
||||
@@ -8,7 +8,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
@@ -87,7 +87,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
});
|
||||
await db.insert(toolConnections).values(irrelevant);
|
||||
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { toolConnections }, capture);
|
||||
try {
|
||||
expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" }))
|
||||
@@ -119,7 +119,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
}).where(eq(heartbeatRuns.id, input.runId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({
|
||||
|
||||
@@ -689,14 +689,9 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
||||
});
|
||||
expect(interactionWake).not.toBeNull();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, interactionWake!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
// This checks dependency interaction semantics, not provider startup speed.
|
||||
// Join the admitted execution before inspecting its persisted outcome.
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const interactionRun = await db
|
||||
.select({
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { NativeStartupCleanupPendingError, readNativeStartupCleanup } from "../services/native-runtime/native-startup-cleanup.js";
|
||||
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
|
||||
|
||||
const execute = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../adapters/index.js", async () => {
|
||||
@@ -70,4 +75,108 @@ suite("native terminal-run cleanup admission", () => {
|
||||
expect((await heartbeat.getRun(nextId))?.status).toBe("succeeded");
|
||||
expect(execute).toHaveBeenCalledTimes(3);
|
||||
}, 30_000);
|
||||
|
||||
it("retries a failed startup intent through timer failure and drain before verified cancellation", async () => {
|
||||
const home = createIsolatedTestHome(), originalHome = process.env.PAPERCLIP_HOME;
|
||||
process.env.PAPERCLIP_HOME = home.home;
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID();
|
||||
const contextId = randomUUID(), leaseOwner = randomUUID(), taskId = randomUUID(), deadlineAtMs = Date.now() - 1;
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
const retries: Array<{ timer: ReturnType<typeof setTimeout>; fire(): void }> = [];
|
||||
const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => {
|
||||
if (typeof delay === "number" && delay <= 30_000 && new Error().stack?.includes("scheduleNativeSessionResumeDispatch")) {
|
||||
const timer = originalSetTimeout(() => {}, 3_600_000);
|
||||
retries.push({ timer, fire() { clearTimeout(timer); callback(...args); } });
|
||||
return timer;
|
||||
}
|
||||
return originalSetTimeout(callback, delay, ...args);
|
||||
}) as typeof setTimeout);
|
||||
let failSuppression = false, runId = "";
|
||||
const runtimeEnv = new Proxy<Record<string, string | undefined>>({}, { get(_target, key) {
|
||||
if (key === "PAPERCLIP_DATABASE_RESTORE_IN_PROGRESS" && failSuppression) {
|
||||
failSuppression = false;
|
||||
throw new Error("Scheduling dependency unavailable");
|
||||
}
|
||||
return undefined;
|
||||
} });
|
||||
const heartbeat = heartbeatService(db, { runtimeEnv });
|
||||
try {
|
||||
await db.insert(companies).values({ id: companyId, name: "Intent retry", issuePrefix: "CIR", defaultResponsibleUserId: "owner" });
|
||||
await db.insert(authUsers).values({ id: "owner", name: "Fixture user", email: "owner@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "owner", status: "active", membershipRole: "member" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", adapterType: "process",
|
||||
adapterConfig: {}, permissions: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Recover startup", status: "todo", assigneeAgentId: agentId, responsibleUserId: "owner" });
|
||||
const [environment] = await db.insert(environments).values({ companyId, name: "Controlled provider", driver: "plugin", config: {} }).returning();
|
||||
await db.execute(sql`create table test_cleanup_intent_fault (enabled boolean not null)`);
|
||||
await db.execute(sql`insert into test_cleanup_intent_fault values (true)`);
|
||||
await db.execute(sql`create sequence test_cleanup_intent_attempts`);
|
||||
await db.execute(sql.raw(`create function test_reject_cleanup_intent() returns trigger language plpgsql as $$ begin
|
||||
if new.company_id = '${companyId}'::uuid and new.error_code = 'native_startup_cleanup_pending'
|
||||
and exists (select 1 from test_cleanup_intent_fault where enabled) then
|
||||
perform nextval('test_cleanup_intent_attempts'); raise exception 'Cleanup intent write unavailable';
|
||||
end if; return new; end $$`));
|
||||
await db.execute(sql`create trigger test_cleanup_intent_fault before update on heartbeat_runs for each row execute function test_reject_cleanup_intent()`);
|
||||
execute.mockReset();
|
||||
execute.mockImplementation(async (context: { runId: string }) => {
|
||||
runId = context.runId;
|
||||
const [run] = await db.update(heartbeatRuns).set({ runtimeMode: "native", runtimeModeResolvedAt: null }).where(eq(heartbeatRuns.id, runId)).returning();
|
||||
await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Recover startup", description: null }, environmentLeaseId: leaseId });
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } }).where(eq(agents.id, agentId));
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1,
|
||||
controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() - 1_000) });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId,
|
||||
issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment", metadata: { runnerTaskLease: true } });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({
|
||||
nativeStartupContext: { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId,
|
||||
contextId, deadlineAtMs, leaseId, leaseOwner, controllerGeneration: 1, attempt: 1 },
|
||||
})}::jsonb` }).where(eq(heartbeatRuns.id, runId));
|
||||
throw new NativeStartupCleanupPendingError(contextId, deadlineAtMs, leaseOwner, false, leaseId);
|
||||
});
|
||||
await heartbeat.invoke(agentId, "on_demand", { issueId }, "manual");
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!);
|
||||
const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!);
|
||||
expect((await db.execute<{ last_value: string }>(sql`select last_value from test_cleanup_intent_attempts`))[0].last_value).toBe("2");
|
||||
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "running", errorCode: null });
|
||||
expect((await owner()).leaseOwner).toBe(leaseOwner);
|
||||
expect(await lease()).toMatchObject({ status: "active", cleanupStatus: null });
|
||||
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)).toBeNull();
|
||||
const drainedRetries = retries.length;
|
||||
expect(drainedRetries).toBeGreaterThan(0);
|
||||
await heartbeat.resumeQueuedRunsAfterDrain();
|
||||
expect(retries.length).toBeGreaterThan(drainedRetries);
|
||||
const resumedRetries = retries.length;
|
||||
failSuppression = true;
|
||||
retries.at(-1)!.fire();
|
||||
await vi.waitFor(() => expect(retries.length).toBeGreaterThan(resumedRetries));
|
||||
expect((await owner()).leaseOwner).toBe(leaseOwner);
|
||||
expect((await heartbeat.getRun(runId))?.status).toBe("running");
|
||||
await db.execute(sql`update test_cleanup_intent_fault set enabled = false`);
|
||||
retries.at(-1)!.fire();
|
||||
await vi.waitFor(async () => expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("pending"));
|
||||
await vi.waitFor(async () => expect((await heartbeat.listEvents(runId)).some(event => event.eventType === "native.recovery.transition")).toBe(true));
|
||||
const current = await lease();
|
||||
await environmentService(db).releaseLease(leaseId, "released", { cleanupStatus: "success",
|
||||
remoteExecutionTermination: remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" }) });
|
||||
await heartbeat.cancelRun(runId, "Startup cleanup retry completed");
|
||||
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("complete");
|
||||
expect(await owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null });
|
||||
expect(execute).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
timeout.mockRestore();
|
||||
for (const retry of retries) clearTimeout(retry.timer);
|
||||
await db.execute(sql`drop trigger if exists test_cleanup_intent_fault on heartbeat_runs`);
|
||||
await db.execute(sql`drop function if exists test_reject_cleanup_intent()`);
|
||||
await db.execute(sql`drop table if exists test_cleanup_intent_fault`);
|
||||
await db.execute(sql`drop sequence if exists test_cleanup_intent_attempts`);
|
||||
if (originalHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = originalHome;
|
||||
home.cleanup();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
@@ -63,11 +63,11 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => {
|
||||
protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } },
|
||||
});
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture);
|
||||
try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16);
|
||||
|
||||
@@ -41,3 +41,21 @@ describe("compactRunLogChunk", () => {
|
||||
expect(compacted).not.toContain("paperclip-flag-secret");
|
||||
});
|
||||
});
|
||||
|
||||
describe("compactRunLogChunk with long base64 commands", () => {
|
||||
it("redacts a tool call that writes base64 of source text without stalling", () => {
|
||||
const typescriptSource = "export const limit = 65536;\nexport function keep(value: string) {\n return value.slice(0, limit);\n}\n";
|
||||
const blob = Buffer.from(typescriptSource.repeat(1_000)).toString("base64").slice(0, 48 * 1024);
|
||||
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
|
||||
const command = `printf '%s' '${blob}' | base64 -d > src/keep.ts && echo '{"GITHUB_TOKEN":"tool-call-secret"}'`;
|
||||
const chunk = `${JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", input: { command } }] } })}\n`;
|
||||
|
||||
const startedAt = performance.now();
|
||||
const compacted = compactRunLogChunk(chunk);
|
||||
|
||||
expect(performance.now() - startedAt).toBeLessThan(1_000);
|
||||
expect(compacted).toContain(blob);
|
||||
expect(compacted).not.toContain("tool-call-secret");
|
||||
expect(compacted).toContain("***REDACTED***");
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
@@ -268,7 +268,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
}).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const app = createApp(f.companyId, f.grantedAgentId, observed);
|
||||
|
||||
@@ -791,19 +791,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
`));
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await waitForCondition(async () => {
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db.select({ status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
return run?.status === "cancelled" && wakeup?.status === "skipped";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const [run, wakeup, issue] = await Promise.all([
|
||||
db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
@@ -970,16 +958,10 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(ordering).toEqual(["validated", "parked"]);
|
||||
};
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await parkPromise;
|
||||
|
||||
expect((await heartbeat.getRun(runId))?.status).toBe("cancelled");
|
||||
const issue = await db
|
||||
.select({ status: issues.status })
|
||||
.from(issues)
|
||||
@@ -1108,7 +1090,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
});
|
||||
|
||||
expect(run).not.toBeNull();
|
||||
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(countExecuteCallsForRun(run!.id)).toBe(1);
|
||||
});
|
||||
|
||||
@@ -1126,7 +1108,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
});
|
||||
|
||||
expect(run).not.toBeNull();
|
||||
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(countExecuteCallsForRun(run!.id)).toBe(1);
|
||||
});
|
||||
|
||||
@@ -1718,13 +1700,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId));
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await waitForCondition(async () => {
|
||||
const [deferred] = await db
|
||||
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, deferredWakeupId));
|
||||
return Boolean(deferred?.runId) && deferred?.status !== "deferred_issue_execution";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const [deferred] = await db
|
||||
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
|
||||
@@ -1788,15 +1764,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const [run, wakeup, issue] = await Promise.all([
|
||||
db
|
||||
|
||||
@@ -1,35 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
applyPendingMigrations,
|
||||
createDb,
|
||||
getEmbeddedPostgresTestSupport as embeddedSupport,
|
||||
startEmbeddedPostgresTestDatabase as startEmbedded,
|
||||
export {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
|
||||
type EmbeddedPostgresTestDatabase,
|
||||
type EmbeddedPostgresTestSupport,
|
||||
} from "@paperclipai/db";
|
||||
|
||||
export type { EmbeddedPostgresTestDatabase, EmbeddedPostgresTestSupport };
|
||||
|
||||
export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgresTestSupport> {
|
||||
return process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? { supported: true } : embeddedSupport();
|
||||
}
|
||||
|
||||
export async function startEmbeddedPostgresTestDatabase(prefix: string): Promise<EmbeddedPostgresTestDatabase> {
|
||||
const adminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
|
||||
if (!adminUrl) return startEmbedded(prefix);
|
||||
const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`;
|
||||
const template = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
|
||||
if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) throw new Error("Invalid disposable test template name");
|
||||
const admin = createDb(adminUrl, { maxConnections: 1 });
|
||||
const url = new URL(adminUrl);
|
||||
url.pathname = `/${name}`;
|
||||
const cleanup = async () => {
|
||||
try { await admin.$client.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); }
|
||||
finally { await admin.$client.end({ timeout: 1 }); }
|
||||
};
|
||||
try {
|
||||
await admin.$client.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`);
|
||||
if (!template) await applyPendingMigrations(url.toString());
|
||||
return { connectionString: url.toString(), cleanup };
|
||||
} catch (error) { await cleanup(); throw error; }
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { createFullDatabaseResponseCapture } from "./full-database-response-capture.js";
|
||||
|
||||
const metadata = { operation: "select", table: "heartbeat_runs", callers: [] };
|
||||
|
||||
it("measures every oversized string and complex value without sampling or truncation", () => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
|
||||
const text = "private-payload🙂".repeat(30_000);
|
||||
const nested = { instructions: text, additional: [{ prompt: text }] };
|
||||
const rows = Object.assign([[text, nested]], { columns: [{ name: "stdout_excerpt" }, { name: "context_snapshot" }] });
|
||||
const bytes = Buffer.byteLength(text) + Buffer.byteLength(JSON.stringify(nested));
|
||||
try {
|
||||
for (let index = 0; index < 17; index++) expect(capture.execute(() => rows, metadata)).toBe(rows);
|
||||
capture.flush();
|
||||
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.entry.v1",
|
||||
calls: 17, rows: 17, responseValueBytes: 17 * bytes, maxResponseValueBytes: bytes,
|
||||
estimatedValues: 17, serializationErrors: 0,
|
||||
});
|
||||
expect(events[1]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
|
||||
startedQueries: 17, completedQueries: 17, inFlightAtEnd: 0, overflowCalls: 0, responseValueBytes: 17 * bytes,
|
||||
});
|
||||
expect(JSON.stringify(events)).not.toContain("private-payload");
|
||||
expect(nested.instructions).toBe(text);
|
||||
} finally { capture.flush(); }
|
||||
});
|
||||
|
||||
it("preserves asynchronous results and original query errors", async () => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
|
||||
const rows = Object.assign([{ value: { nested: "result" } }], { columns: [{ name: "value" }] });
|
||||
const failure = new Error("private query failure");
|
||||
try {
|
||||
await expect(capture.execute(async () => rows, metadata)).resolves.toBe(rows);
|
||||
await expect(capture.execute(async () => { throw failure; }, metadata)).rejects.toBe(failure);
|
||||
expect(() => capture.execute(() => { throw failure; }, metadata)).toThrow(failure);
|
||||
capture.flush();
|
||||
expect(events[0]).toMatchObject({ calls: 3, rows: 1, queryErrors: 2,
|
||||
responseValueBytes: Buffer.byteLength(JSON.stringify(rows[0].value)), serializationErrors: 0,
|
||||
});
|
||||
expect(events[1]).toMatchObject({ startedQueries: 3, completedQueries: 3, inFlightAtEnd: 0 });
|
||||
expect(JSON.stringify(events)).not.toContain(failure.message);
|
||||
} finally { capture.flush(); }
|
||||
});
|
||||
|
||||
it("reports unfinished work at the deadline without cancelling it or counting a late result", async () => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
let now = 0;
|
||||
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => now);
|
||||
let release!: (value: string[]) => void;
|
||||
const original = new Promise<string[]>(resolve => { release = resolve; });
|
||||
const pending = capture.execute(() => original, metadata);
|
||||
now = 1000;
|
||||
capture.flush();
|
||||
release(["result"]);
|
||||
await expect(pending).resolves.toEqual(["result"]);
|
||||
capture.flush();
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
|
||||
startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, type ResponseCaptureWindow } from "@paperclipai/db/response-capture";
|
||||
|
||||
export { installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
|
||||
type Capture = ReturnType<typeof createDatabaseResponseCapture>;
|
||||
type Metadata = Parameters<Capture["execute"]>[1];
|
||||
type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number;
|
||||
queryErrors: number; serializationErrors: number; estimatedValues: number };
|
||||
|
||||
function fullResponseValueSize(value: unknown): { bytes: number; estimatedValues: number } {
|
||||
const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined;
|
||||
if (!Array.isArray(value) || !Array.isArray(columns)) {
|
||||
const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
|
||||
return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 };
|
||||
}
|
||||
let bytes = 0, estimatedValues = 0;
|
||||
for (const row of value) {
|
||||
const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]);
|
||||
for (const cell of cells) {
|
||||
if (cell === null || cell === undefined) continue;
|
||||
if (typeof cell === "string") bytes += Buffer.byteLength(cell);
|
||||
else if (typeof cell === "boolean") bytes += 1;
|
||||
else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell));
|
||||
else {
|
||||
estimatedValues += 1;
|
||||
const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
|
||||
if (serialized !== undefined) bytes += Buffer.byteLength(serialized);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { bytes, estimatedValues };
|
||||
}
|
||||
|
||||
export function createFullDatabaseResponseCapture(
|
||||
window: ResponseCaptureWindow,
|
||||
emit: (event: Record<string, unknown>) => void,
|
||||
now: () => number = Date.now,
|
||||
): Capture {
|
||||
const capture = createDatabaseResponseCapture(window, () => {}, now);
|
||||
const id = randomUUID(), entries = new Map<string, Entry>();
|
||||
let closed = false, inFlight = 0, started = 0, completed = 0, overflowCalls = 0;
|
||||
const send = (event: Record<string, unknown>) => { try { emit(event); } catch {} };
|
||||
function flush() {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
clearTimeout(timer);
|
||||
capture.flush();
|
||||
const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes);
|
||||
for (const [groupIndex, entry] of ordered.entries()) send({
|
||||
event: "paperclip.test_db_response_capture.entry.v1", captureId: id,
|
||||
groupIndex, groupPage: Math.floor(groupIndex / 128),
|
||||
operation: entry.operation, table: entry.table, callers: entry.callers,
|
||||
calls: entry.calls, rows: entry.rows, responseValueBytes: entry.bytes,
|
||||
maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues,
|
||||
queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors,
|
||||
});
|
||||
send({ event: "paperclip.test_db_response_capture.summary.v1", captureId: id,
|
||||
windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(),
|
||||
closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed,
|
||||
inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: 128, groupPages: Math.ceil(ordered.length / 128),
|
||||
overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0),
|
||||
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false,
|
||||
measurementBoundary: "postgres-js results before Drizzle column decoding",
|
||||
relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true,
|
||||
taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false,
|
||||
});
|
||||
entries.clear();
|
||||
}
|
||||
const timer = setTimeout(flush, Math.max(0, window.end - now()));
|
||||
timer.unref();
|
||||
function finish(metadata: Metadata, value: unknown, failed: boolean) {
|
||||
if (!capture.active()) return;
|
||||
inFlight -= 1;
|
||||
completed += 1;
|
||||
let key = JSON.stringify(metadata);
|
||||
if (!entries.has(key) && entries.size >= 2047) {
|
||||
overflowCalls += 1;
|
||||
metadata = { operation: "other", table: "other", callers: [] };
|
||||
key = "overflow";
|
||||
}
|
||||
let entry = entries.get(key);
|
||||
if (!entry) {
|
||||
entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0,
|
||||
queryErrors: 0, serializationErrors: 0, estimatedValues: 0 };
|
||||
entries.set(key, entry);
|
||||
}
|
||||
entry.calls += 1;
|
||||
if (failed) { entry.queryErrors += 1; return; }
|
||||
entry.rows += Array.isArray(value) ? value.length : 0;
|
||||
try {
|
||||
const size = fullResponseValueSize(value);
|
||||
entry.bytes += size.bytes;
|
||||
entry.maxBytes = Math.max(entry.maxBytes, size.bytes);
|
||||
entry.estimatedValues += size.estimatedValues;
|
||||
} catch { entry.serializationErrors += 1; }
|
||||
}
|
||||
function execute(action: () => unknown, metadata: Metadata) {
|
||||
if (!capture.active()) return action();
|
||||
inFlight += 1;
|
||||
started += 1;
|
||||
let result: unknown;
|
||||
try { result = action(); }
|
||||
catch (error) { finish(metadata, undefined, true); throw error; }
|
||||
let asynchronous = false;
|
||||
try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {}
|
||||
if (asynchronous) return Promise.resolve(result).then(
|
||||
value => { finish(metadata, value, false); return value; },
|
||||
error => { finish(metadata, undefined, true); throw error; },
|
||||
);
|
||||
finish(metadata, result, false);
|
||||
return result;
|
||||
}
|
||||
return { ...capture, execute, flush };
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db";
|
||||
|
||||
export async function createToolProfileMatchingFixture(db: Db) {
|
||||
const [company] = await db.insert(companies).values({ name: "Synthetic profile matching", issuePrefix: `BM${randomUUID().slice(0, 6)}` }).returning();
|
||||
const companyAgents = await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` }))).returning();
|
||||
const agent = companyAgents[0]!;
|
||||
const [app] = await db.insert(toolApplications).values({ companyId: company.id, name: "Synthetic app", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: app.id, name: "Synthetic connection", uid: randomUUID(), transport: "mcp_remote" }).returning();
|
||||
const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: 1890 }, (_, i) => ({ companyId: company.id, applicationId: app.id, connectionId: connection.id, name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const }))).returning();
|
||||
const [profile] = await db.insert(toolProfiles).values({ companyId: company.id, profileKey: "selected", name: "Selected tools", defaultAction: "deny" }).returning();
|
||||
await db.insert(toolProfileBindings).values({ companyId: company.id, profileId: profile.id, targetType: "agent", targetId: agent.id });
|
||||
await db.insert(toolProfileEntries).values(catalog.slice(0, 1575).map((entry, i) => ({ companyId: company.id, profileId: profile.id, selectorType: "catalog_entry" as const, catalogEntryId: entry.id, effect: i % 5 === 0 ? "exclude" as const : "include" as const })));
|
||||
return { company, agent };
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db";
|
||||
|
||||
export async function createProfileProjectionFixture(db: Db, options: { profiles?: number; tools?: number; rules?: number } = {}) {
|
||||
const profileCount = options.profiles ?? 200, toolCount = options.tools ?? 1900, ruleCount = options.rules ?? 380, maxRuleCount = options.rules ?? 695;
|
||||
const [company] = await db.insert(companies).values({ name: "Profile projection fixture", issuePrefix: `PP${randomUUID().slice(0, 6)}` }).returning();
|
||||
await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` })));
|
||||
const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "Projection application", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: application.id, name: "Projection connection", uid: randomUUID(), transport: "mcp_remote", status: "active", enabled: true, healthStatus: "ok" }).returning();
|
||||
const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: toolCount + maxRuleCount }, (_, i) => ({
|
||||
companyId: company.id, applicationId: application.id, connectionId: connection.id,
|
||||
name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const,
|
||||
status: i < toolCount ? "active" as const : "removed" as const,
|
||||
firstSeenAt: new Date(i < toolCount ? "2026-01-03T00:00:00Z" : "2026-01-01T00:00:00Z"),
|
||||
}))).returning();
|
||||
const profiles = await db.insert(toolProfiles).values(Array.from({ length: profileCount }, (_, i) => ({
|
||||
companyId: company.id, profileKey: `selected-${i}`, name: `Selected profile ${i}`, defaultAction: "deny" as const,
|
||||
createdAt: new Date("2026-01-02T00:00:00Z"),
|
||||
}))).returning();
|
||||
for (let i = 0; i < profiles.length; i++) {
|
||||
const count = i % 4 < 2 ? maxRuleCount : 2 * ruleCount - maxRuleCount;
|
||||
const rules = (i % 2 ? catalog.slice(toolCount, toolCount + count) : catalog.slice(0, count)).map(entry => ({
|
||||
companyId: company.id, profileId: profiles[i]!.id, selectorType: "catalog_entry" as const,
|
||||
effect: "include" as const, catalogEntryId: entry.id,
|
||||
}));
|
||||
for (let offset = 0; offset < rules.length; offset += 500) await db.insert(toolProfileEntries).values(rules.slice(offset, offset + 500));
|
||||
}
|
||||
return { company, application, connection, catalog: catalog.slice(0, toolCount), profiles };
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import { agentApiKeys, agents, budgetPolicies, chatDeliveries, chatEndpoints, co
|
||||
companyTransferRuns, connectionEventDeliveries, createDb, heartbeatRuns, issues,
|
||||
routineTriggers, routines, toolApplications, toolConnections, type Db } from "@paperclipai/db";
|
||||
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const now = Date.UTC(2026, 10, 2);
|
||||
@@ -25,7 +25,7 @@ if (!support.supported) console.warn(`Skipping idle sleep diagnostics Postgres t
|
||||
db = createDb(externalDatabaseUrl ?? temporary!.connectionString, { maxConnections: 4 });
|
||||
}, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
const read = (scheduled = false, monitors = false) => readIdleSleepSafety(db, hold, () => now, ownerId,
|
||||
localNone, undefined, undefined, scheduled, monitors, { collectDiagnostics: true });
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
heartbeatRuns, issues, issueWatchdogs, issueThreadInteractions, issueQuestionResponseDeliveries,
|
||||
projects, projectWorkspaces, routines, routineTriggers, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
|
||||
|
||||
@@ -175,7 +175,7 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
|
||||
db = createDb(externalDatabaseUrl ?? database.connectionString, { maxConnections: 4 });
|
||||
}, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal);
|
||||
const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true);
|
||||
@@ -206,6 +206,19 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps a failed startup intent awake after its local worker counter settles", async () => {
|
||||
const f = await seed();
|
||||
expect(await read()).toEqual(none);
|
||||
const finish = beginIdleTrackedWork();
|
||||
await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativePhase: "observed",
|
||||
runnerProfileJson: { nativeStartupContext: { contextId: randomUUID(), deadlineAtMs: now - 1 } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
finish();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
expect(await read()).toEqual(present);
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await read()).toEqual(none);
|
||||
});
|
||||
|
||||
it("retains committed budget stop intent until its exact generation is delivered", async () => {
|
||||
const f = await seed();
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { activityLog, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
@@ -55,7 +55,7 @@ describe.skipIf(!support.supported)("creation source run projections", () => {
|
||||
expect(run.issueId).toBe(binding === "scoped" ? sourceId : null);
|
||||
expect(run.nativeIssueId).toBeNull();
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns, issues }, capture);
|
||||
try {
|
||||
const result = await resolveIssueCreationSource(observed, child);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { expect, it } from "vitest";
|
||||
import request from "supertest";
|
||||
import { agents, heartbeatRuns, issueRecoveryActions, issueRelations, issues, statusCards } from "@paperclipai/db";
|
||||
import { companySearchQuerySchema, createIssueSchema, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
|
||||
import { issueDisplayStatusSql } from "../services/issue-display-status.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { companySearchService } from "../services/company-search.js";
|
||||
import { statusCardService } from "../services/status-cards.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import {
|
||||
describeEmbeddedPostgres, routeApp,
|
||||
seedCompanyWithBoardAccess, useEmbeddedPostgres,
|
||||
} from "./helpers/route-test-harness.js";
|
||||
|
||||
describeEmbeddedPostgres("Monitoring task presentation", () => {
|
||||
const ctx = useEmbeddedPostgres("paperclip-monitoring-status-", {
|
||||
resetEach: async (db) => { await db.execute(sql`SET client_min_messages = warning`); await db.execute(sql`TRUNCATE TABLE companies CASCADE`); },
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const company = await seedCompanyWithBoardAccess(ctx.db, "Monitoring");
|
||||
const agentId = randomUUID();
|
||||
await ctx.db.insert(agents).values({ id: agentId, companyId: company.companyId, name: "Monitor owner", role: "engineer", status: "paused", adapterType: "process", adapterConfig: {} });
|
||||
const nextCheckAt = new Date(Date.now() + 60 * 60_000);
|
||||
const issueId = randomUUID();
|
||||
await ctx.db.insert(issues).values({
|
||||
id: issueId, companyId: company.companyId, title: "Check deployment", status: "in_progress",
|
||||
assigneeAgentId: agentId, createdByUserId: company.actor.userId,
|
||||
monitorNextCheckAt: nextCheckAt,
|
||||
executionPolicy: { mode: "normal", stages: [], commentRequired: false, monitor: { nextCheckAt: nextCheckAt.toISOString(), notes: "Check deployment" } },
|
||||
});
|
||||
return { ...company, agentId, issueId };
|
||||
}
|
||||
|
||||
async function displayStatus(id: string) {
|
||||
const [row] = await ctx.db.select({ status: issues.status, displayStatus: issueDisplayStatusSql() }).from(issues).where(eq(issues.id, id));
|
||||
return row;
|
||||
}
|
||||
|
||||
it("projects a scheduled wait consistently across detail, compact rows, summaries, search and counts", async () => {
|
||||
const f = await fixture();
|
||||
const svc = issueService(ctx.db);
|
||||
expect(await svc.getById(f.issueId)).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
|
||||
expect(await svc.list(f.companyId, { displayStatus: "monitoring", listProjection: "index" })).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
|
||||
expect(await svc.count(f.companyId, { displayStatus: "monitoring" })).toBe(1);
|
||||
const app = routeApp(ctx.db, f.actor, issueRoutes);
|
||||
const summary = await request(app).get(`/api/companies/${f.companyId}/issues/summaries?ids=${f.issueId}`).expect(200);
|
||||
expect(summary.body[0]).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
|
||||
const result = await companySearchService(ctx.db).search(f.companyId, companySearchQuerySchema.parse({ q: "deployment", displayStatus: "monitoring" }));
|
||||
expect(result.results[0]?.issue).toMatchObject({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" });
|
||||
expect(result.filterOptionCounts.displayStatus?.monitoring).toBe(1);
|
||||
expect(result.filterOptionCounts.status.in_progress).toBe(1);
|
||||
const [card] = await ctx.db.insert(statusCards).values({ companyId: f.companyId, interestPrompt: "Watch deployment", mentionedIssueIds: [f.issueId], refreshPolicy: { mode: "manual" } }).returning();
|
||||
expect(await statusCardService(ctx.db).listMentionedIssues(card!)).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
|
||||
// Existing agent/API lifecycle filters keep their original meaning.
|
||||
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(1);
|
||||
expect(createIssueSchema.safeParse({ title: "Manual monitoring", status: "monitoring" }).success).toBe(false);
|
||||
});
|
||||
|
||||
it("filters before pagination and keeps In Progress separate from Monitoring", async () => {
|
||||
const f = await fixture();
|
||||
await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ordinary work", status: "in_progress", assigneeAgentId: f.agentId });
|
||||
const svc = issueService(ctx.db);
|
||||
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 0 })).toEqual([expect.objectContaining({ id: f.issueId })]);
|
||||
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 1 })).toEqual([]);
|
||||
expect(await svc.count(f.companyId, { displayStatus: "in_progress" })).toBe(1);
|
||||
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(2);
|
||||
});
|
||||
|
||||
it("shows the same wait indicator in list blocker summaries and task detail relations", async () => {
|
||||
const f = await fixture();
|
||||
const [dependent] = await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ship after deployment", status: "blocked" }).returning();
|
||||
await ctx.db.insert(issueRelations).values({ companyId: f.companyId, issueId: f.issueId, relatedIssueId: dependent!.id, type: "blocks" });
|
||||
const svc = issueService(ctx.db);
|
||||
const list = await svc.list(f.companyId, { includeBlockedBy: true });
|
||||
expect(list.find(row => row.id === dependent!.id)?.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
|
||||
const detail = await request(routeApp(ctx.db, f.actor, issueRoutes)).get(`/api/issues/${dependent!.id}`).expect(200);
|
||||
expect(detail.body.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
|
||||
});
|
||||
|
||||
it("returns to In Progress during the check and Monitoring after another yield, then Done", async () => {
|
||||
const f = await fixture();
|
||||
const runId = randomUUID();
|
||||
await ctx.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "queued" });
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId));
|
||||
await ctx.db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId));
|
||||
await ctx.db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
|
||||
await ctx.db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId));
|
||||
expect(await displayStatus(f.issueId)).toEqual({ status: "done", displayStatus: "done" });
|
||||
});
|
||||
|
||||
it.each(["blocked", "in_review", "todo", "cancelled"])("preserves %s even with a retained monitor", async (status) => {
|
||||
const f = await fixture();
|
||||
await ctx.db.update(issues).set({ status }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe(status);
|
||||
});
|
||||
|
||||
it("does not label cleared, consumed, orphaned or quota-recovery schedules Monitoring", async () => {
|
||||
const f = await fixture();
|
||||
await ctx.db.update(issues).set({ executionState: { monitor: { status: "cleared" } } }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(issues).set({ executionState: null, monitorNextCheckAt: null }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(issues).set({ monitorNextCheckAt: new Date(), executionPolicy: { monitor: { serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } } }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(issues).set({ executionPolicy: null }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
});
|
||||
|
||||
it("keeps recovery and review holds visible and never writes a new lifecycle state", async () => {
|
||||
const f = await fixture();
|
||||
await ctx.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "source", ownerType: "board", cause: "process_lost", fingerprint: randomUUID(), nextAction: "Inspect stopped run" });
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'escalated' WHERE source_issue_id = ${f.issueId}`);
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'resolved' WHERE source_issue_id = ${f.issueId}`);
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
|
||||
await ctx.db.update(issues).set({ executionState: { currentParticipant: { type: "user", userId: "reviewer" } } }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agentRuntimeState,
|
||||
@@ -202,7 +202,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
if (fallback) await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed))
|
||||
@@ -238,7 +238,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
}).where(eq(heartbeatRuns.companyId, f.companyId));
|
||||
const [before] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agentWakeupRequests, heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
|
||||
@@ -275,7 +275,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
})),
|
||||
]);
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agentWakeupRequests }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
|
||||
|
||||
@@ -50,7 +50,7 @@ describeEmbeddedPostgres("bounded issue summaries", () => {
|
||||
expect(res.headers["cache-control"]).toBe("private, no-store");
|
||||
expect(res.body.map((row: { id: string }) => row.id)).toEqual([f.rows[1]!.id, f.rows[0]!.id]);
|
||||
expect(Object.keys(res.body[0]).sort()).toEqual([
|
||||
"companyId", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
|
||||
"companyId", "displayStatus", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
|
||||
]);
|
||||
expect(res.body[0].externalConversationState).toBeNull();
|
||||
expect(forbiddenRead).not.toHaveBeenCalled();
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import request from "supertest";
|
||||
import { expect, it } from "vitest";
|
||||
import { activityLog, agents, authUsers, issues } from "@paperclipai/db";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { deriveIssueUserContext, issueService } from "../services/issues.js";
|
||||
import { describeEmbeddedPostgres, resetCompanyIssueFixtures, routeApp, seedCompanyWithBoardAccess, useEmbeddedPostgres } from "./helpers/route-test-harness.js";
|
||||
|
||||
describeEmbeddedPostgres("human assignee unread comments", () => {
|
||||
const ctx = useEmbeddedPostgres("paperclip-assignee-unread-", {
|
||||
resetEach: async db => {
|
||||
await db.delete(activityLog);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await resetCompanyIssueFixtures(db);
|
||||
await db.delete(authUsers);
|
||||
},
|
||||
});
|
||||
|
||||
it.each([false, true])("keeps another human's real comment unread through recency updates (prior read: %s)", async priorRead => {
|
||||
const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Assigned human task");
|
||||
const otherCompany = await seedCompanyWithBoardAccess(ctx.db, "Unrelated company");
|
||||
const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes);
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Human inbox comment", status: "in_progress", createdByUserId: userId, assigneeUserId: userId,
|
||||
});
|
||||
if (priorRead) await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
|
||||
const comment = await svc.addComment(issue.id, "Another human supplied an update", { userId: `author-${randomUUID()}` });
|
||||
const [afterComment] = await ctx.db.select().from(issues).where(eq(issues.id, issue.id));
|
||||
// Exercise the real insert and recency update. No historical timestamps or
|
||||
// mocked comment state hide the ordering that used to clear unread status.
|
||||
expect(afterComment!.updatedAt.getTime()).toBeGreaterThanOrEqual(new Date(comment.createdAt).getTime());
|
||||
const path = `/api/companies/${companyId}/issues`;
|
||||
const expectUnread = async (unread: boolean) => {
|
||||
for (const projection of ["", "&view=rows", "&view=index"]) {
|
||||
const result = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&limit=100${projection}`).expect(200);
|
||||
expect(result.body).toHaveLength(1);
|
||||
expect(result.body[0].isUnreadForMe).toBe(unread);
|
||||
const filtered = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&unreadForUserId=me&limit=100${projection}`).expect(200);
|
||||
expect(filtered.body.map((row: { id: string }) => row.id)).toEqual(unread ? [issue.id] : []);
|
||||
}
|
||||
expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(unread ? 1 : 0);
|
||||
expect(await svc.unreadInboxIssueIds(companyId, userId, sql<boolean>`true`)).toEqual(unread ? [issue.id] : []);
|
||||
};
|
||||
await expectUnread(true);
|
||||
// Legacy compact responses have a short TTL. Check their SQL projection
|
||||
// once; immediate mutation readback belongs to ordinary/rows/index.
|
||||
const compact = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&view=compact`).expect(200);
|
||||
expect(compact.body[0].isUnreadForMe).toBe(true);
|
||||
const derived = deriveIssueUserContext(afterComment!, userId, {
|
||||
myLastCommentAt: null, myLastReadAt: null, lastExternalCommentAt: comment.createdAt,
|
||||
});
|
||||
expect(derived.isUnreadForMe).toBe(true);
|
||||
await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
|
||||
await expectUnread(false);
|
||||
await request(app).delete(`/api/issues/${issue.id}/read`).expect(200);
|
||||
await expectUnread(true);
|
||||
await svc.addComment(issue.id, "I have read this update", { userId });
|
||||
await expectUnread(false);
|
||||
await request(app).get(`/api/companies/${otherCompany.companyId}/issues?unreadForUserId=me`).expect(403);
|
||||
expect(await svc.countUnreadTouchedByUser(otherCompany.companyId, userId)).toBe(0);
|
||||
});
|
||||
|
||||
it("shows existing comments as unread when a task is subsequently assigned to a human", async () => {
|
||||
const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "New assignment");
|
||||
const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes);
|
||||
const issue = await svc.create(companyId, { title: "Assigned after discussion", status: "backlog" });
|
||||
await svc.addComment(issue.id, "Context written before assignment", { userId: `author-${randomUUID()}` });
|
||||
await request(app).patch(`/api/issues/${issue.id}`).send({ assigneeUserId: userId, status: "in_progress" }).expect(200);
|
||||
const result = await request(app).get(`/api/companies/${companyId}/issues?q=${encodeURIComponent(issue.title)}&unreadForUserId=me`).expect(200);
|
||||
expect(result.body.map((row: { id: string }) => row.id)).toEqual([issue.id]);
|
||||
expect(result.body[0].isUnreadForMe).toBe(true);
|
||||
await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
|
||||
expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(0);
|
||||
});
|
||||
|
||||
it("excludes deleted external comments but retains the historical touch from a deleted own reply", async () => {
|
||||
const { companyId, userId } = await seedCompanyWithBoardAccess(ctx.db, "Deleted comment controls");
|
||||
const svc = issueService(ctx.db);
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Deleted inbox activity", status: "in_progress", assigneeUserId: userId,
|
||||
});
|
||||
const authorId = `author-${randomUUID()}`;
|
||||
const first = await svc.addComment(issue.id, "Keep this live update", { userId: authorId });
|
||||
const second = await svc.addComment(issue.id, "Remove this later update", { userId: authorId });
|
||||
const observation = async () => {
|
||||
const [row] = await svc.list(companyId, { touchedByUserId: userId });
|
||||
return { row, count: await svc.countUnreadTouchedByUser(companyId, userId) };
|
||||
};
|
||||
await svc.tombstoneComment(second.id, { actorType: "user", userId });
|
||||
expect((await observation()).count).toBe(1);
|
||||
expect((await observation()).row!.lastExternalCommentAt).toEqual(new Date(first.createdAt));
|
||||
await svc.tombstoneComment(first.id, { actorType: "user", userId });
|
||||
const removed = await observation();
|
||||
expect(removed.count).toBe(0);
|
||||
expect(removed.row!.isUnreadForMe).toBe(false);
|
||||
expect(removed.row!.lastExternalCommentAt).toBeNull();
|
||||
await svc.addComment(issue.id, "New live activity", { userId: authorId });
|
||||
expect((await observation()).count).toBe(1);
|
||||
const ownReply = await svc.addComment(issue.id, "I saw the live update", { userId });
|
||||
expect((await observation()).count).toBe(0);
|
||||
await svc.tombstoneComment(ownReply.id, { actorType: "user", userId });
|
||||
const seen = await observation();
|
||||
expect(seen.count).toBe(0);
|
||||
expect(seen.row!.isUnreadForMe).toBe(false);
|
||||
expect(seen.row!.myLastTouchAt).toEqual(new Date(ownReply.createdAt));
|
||||
});
|
||||
|
||||
});
|
||||
@@ -56,4 +56,30 @@ describeEmbeddedPostgres("issue list status query parsing", () => {
|
||||
it("returns every status when ?status is absent", async () => {
|
||||
expect(await listStatuses("")).toEqual(["done", "in_progress", "todo"]);
|
||||
});
|
||||
it.each(["", "&view=compact", "&view=compact&projection=index"])("rejects unsupported creation sorting for blocked attention (%s)", async projection => {
|
||||
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Blocked sorting");
|
||||
await request(routeApp(ctx.db, actor, issueRoutes))
|
||||
.get(`/api/companies/${companyId}/issues?attention=blocked&sortField=created&sortDir=asc${projection}`)
|
||||
.expect(422);
|
||||
});
|
||||
|
||||
it.each(["", "&view=compact", "&view=compact&projection=index"])("sorts routine tasks by creation time before pagination (%s)", async (projection) => {
|
||||
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Creation sorting");
|
||||
const ids = Array.from({ length: 4 }, (_, index) => `00000000-0000-4000-8000-00000000000${index}`);
|
||||
await ctx.db.insert(issues).values([
|
||||
...ids.map((id, index) => ({ id, companyId, title: "Routine sorting", originKind: "routine_execution", status: "todo",
|
||||
createdAt: new Date(index < 2 ? "2026-01-01" : "2026-02-01"),
|
||||
updatedAt: new Date(index < 2 ? "2026-03-01" : "2026-02-01"), priority: index < 2 ? "critical" : "low" })),
|
||||
{ companyId, title: "Routine sorting manual", originKind: "manual", createdAt: new Date("2026-04-01") },
|
||||
]);
|
||||
const app = routeApp(ctx.db, actor, issueRoutes);
|
||||
const path = `/api/companies/${companyId}/issues?originKind=routine_execution&includeRoutineExecutions=true&sortField=created&limit=2${projection}`;
|
||||
for (const sortDir of ["asc", "desc"]) {
|
||||
const first = await request(app).get(`${path}&sortDir=${sortDir}`).expect(200);
|
||||
const second = await request(app).get(`${path}&sortDir=${sortDir}&offset=2`).expect(200);
|
||||
expect([...first.body, ...second.body].map((issue: { id: string }) => issue.id)).toEqual(sortDir === "asc" ? ids : [...ids].reverse());
|
||||
}
|
||||
await request(app).get(`${path}&sortDir=invalid`).expect(400);
|
||||
});
|
||||
|
||||
});
|
||||
@@ -4166,6 +4166,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
|
||||
await db.delete(executionWorkspaces);
|
||||
await db.delete(projectWorkspaces);
|
||||
await db.delete(projects);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agents);
|
||||
await db.delete(instanceSettings);
|
||||
await db.delete(companies);
|
||||
@@ -4720,6 +4721,42 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["covered", "foreign-run", "unfinished", "failed-finalize", "late-finish", "later-operation"])(
|
||||
"requires a successful owning finalize to cover a nested snapshot: %s", async kind => {
|
||||
const f = await seedSharedWorkspaceDependency();
|
||||
const runId = randomUUID(), foreignRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values([runId, foreignRunId].map(id => ({
|
||||
id, companyId: f.companyId, agentId: f.assigneeAgentId, status: "succeeded",
|
||||
})));
|
||||
const started = new Date("2026-05-23T22:00:00.000Z");
|
||||
const finished = new Date("2026-05-23T22:00:02.000Z");
|
||||
await db.insert(workspaceOperations).values([
|
||||
{ companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
|
||||
heartbeatRunId: runId, phase: "workspace_finalize",
|
||||
status: kind === "failed-finalize" ? "failed" : "succeeded", startedAt: started, finishedAt: finished },
|
||||
{ companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
|
||||
heartbeatRunId: kind === "foreign-run" ? foreignRunId : runId, phase: "remote_workspace_snapshot",
|
||||
status: kind === "unfinished" ? "running" : "succeeded",
|
||||
startedAt: new Date(started.getTime() + 100),
|
||||
finishedAt: kind === "unfinished" ? null : new Date(finished.getTime() + (kind === "late-finish" ? 100 : -100)) },
|
||||
...(kind === "later-operation" ? [{
|
||||
companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
|
||||
heartbeatRunId: runId, phase: "worktree_prepare", status: "succeeded",
|
||||
startedAt: new Date(finished.getTime() + 100), finishedAt: new Date(finished.getTime() + 200),
|
||||
}] : []),
|
||||
]);
|
||||
const ready = kind === "covered";
|
||||
expect((await svc.listWakeableBlockedDependents(f.blockerId)).map(issue => issue.id))
|
||||
.toEqual(ready ? [f.dependentId] : []);
|
||||
await expect(svc.getDependencyReadiness(f.dependentId)).resolves.toMatchObject({
|
||||
isDependencyReady: ready,
|
||||
pendingFinalizeBlockerIssueIds: ready ? [] : [f.blockerId],
|
||||
});
|
||||
const listed = (await svc.list(f.companyId, { status: "blocked" })).find(issue => issue.id === f.dependentId);
|
||||
expect(listed?.blockerAttention?.pendingFinalizeBlockerIssueIds ?? []).toEqual(ready ? [] : [f.blockerId]);
|
||||
},
|
||||
);
|
||||
|
||||
it("reports dependency readiness for blocked issue chains", async () => {
|
||||
const companyId = randomUUID();
|
||||
await seedIssueCompany(db, companyId);
|
||||
|
||||
@@ -76,7 +76,7 @@ describe("deriveIssueUserContext", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("uses issue updated time as fallback touch point for assignee", () => {
|
||||
it("uses stable creation time for assignees instead of treating task recency as a read", () => {
|
||||
const context = deriveIssueUserContext(
|
||||
makeIssue({ assigneeUserId: "user-1", updatedAt: new Date("2026-03-06T15:00:00.000Z") }),
|
||||
"user-1",
|
||||
@@ -87,8 +87,8 @@ describe("deriveIssueUserContext", () => {
|
||||
},
|
||||
);
|
||||
|
||||
expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T15:00:00.000Z");
|
||||
expect(context.isUnreadForMe).toBe(false);
|
||||
expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T10:00:00.000Z");
|
||||
expect(context.isUnreadForMe).toBe(true);
|
||||
});
|
||||
|
||||
it("uses latest read timestamp to clear unread without requiring a comment", () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
|
||||
@@ -54,12 +54,12 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk,
|
||||
}).where(eq(heartbeatRuns.id, f.runId)).returning();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
|
||||
try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); }
|
||||
finally { capture.flush(); }
|
||||
const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
|
||||
const groups = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
|
||||
const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select");
|
||||
expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1);
|
||||
expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
@@ -86,7 +86,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
patch: { resultJson: { workspaceRestoreFailure: "restore_failed", futureResult: bulk } },
|
||||
}))!;
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
|
||||
try { returned = await terminalizeLegacyExecution({ db, run: failed, status: "failed", recordRestoreFailureOnly,
|
||||
@@ -94,7 +94,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
...(recordRestoreFailureOnly ? { patch: { resultJson: failed.resultJson } } : {}),
|
||||
}); }
|
||||
finally { capture.flush(); }
|
||||
const writes = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const writes = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && event.operation === "update");
|
||||
expect(writes.reduce((sum, group) => sum + Number(group.rows), 0)).toBe(1);
|
||||
expect(returned).toMatchObject({ id: f.runId, status: "failed", finishedAt: failed.finishedAt,
|
||||
|
||||
@@ -11,7 +11,7 @@ import { setTimeout as delay } from "node:timers/promises";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { environmentRuntimeService, type EnvironmentDriver } from "../services/environment-runtime.js";
|
||||
@@ -180,7 +180,7 @@ describe("native environment release scheduling", () => {
|
||||
}
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { nativeRunFinalizations, heartbeatRuns, environmentLeases }, capture);
|
||||
const settle = vi.fn(async () => {});
|
||||
const work = nativeEnvironmentSettlementWork(db, settle);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { spawn } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
@@ -222,11 +222,11 @@ suite("settled native run process identity", () => {
|
||||
const saved = await readRun(runId);
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try { expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); }
|
||||
finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.responseValueBytes).toBeLessThan(8192);
|
||||
const after = await readRun(runId);
|
||||
expect(after).toEqual({ ...saved, processPid: null, processGroupId: null, processStartedAt: null,
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { databaseWorkPending } from "@paperclipai/db/work-signals";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog, agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases,
|
||||
environments, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, plugins, sandboxCreationOperations,
|
||||
} from "@paperclipai/db";
|
||||
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
|
||||
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
|
||||
import {
|
||||
acknowledgeNativeStartupCleanup, NATIVE_STARTUP_CLEANUP_PENDING, readNativeStartupCleanup,
|
||||
requestNativeStartupCleanup, verifyNativeStartupCleanup, clearNativeStartupContext,
|
||||
} from "../services/native-runtime/native-startup-cleanup.js";
|
||||
import { isNativeRunnerOwnershipHeld } from "../services/native-runtime/native-runner-ownership.js";
|
||||
import { cancelNativeSession } from "../services/native-runtime/native-session-executor.js";
|
||||
import * as nativeSessionExecutor from "../services/native-runtime/native-session-executor.js";
|
||||
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
|
||||
import type { NativeExecutionInput } from "../vendor/paperclip-runner/index.js";
|
||||
import { callbacks } from "../services/heartbeat/__tests__/run-control-fixtures.js";
|
||||
import { createHeartbeatRunControl } from "../services/heartbeat/run-control.js";
|
||||
import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js";
|
||||
import { executionControlSchedule } from "../services/execution-control-work.js";
|
||||
import { claimNativeRestartRecoveries } from "../services/native-runtime/native-restart-recovery.js";
|
||||
import { reconcileAbandonedExecutionControl } from "../services/execution-control-reconciliation.js";
|
||||
import { createPluginWorkerManager, type PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
|
||||
import { createOwnedSandbox, queueSandboxCreationCleanup, recordSandboxCreationResource, recoverSandboxCreations } from "../services/sandbox-creation-operations.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "../services/heartbeat/recovery.js";
|
||||
|
||||
const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
describePostgres("durable cleanup after a native startup timeout", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let home: ReturnType<typeof createIsolatedTestHome>;
|
||||
const originalHome = process.env.PAPERCLIP_HOME;
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
|
||||
beforeAll(async () => {
|
||||
home = createIsolatedTestHome();
|
||||
process.env.PAPERCLIP_HOME = home.home;
|
||||
temporary = await startRecoveryPostgres("native-startup-cleanup-");
|
||||
db = createDb(temporary.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Startup cleanup", issuePrefix: "NSC" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Runner", role: "engineer", status: "running",
|
||||
adapterType: "paperclip_runner", adapterConfig: { provider: "codex" }, runtimeConfig: {}, permissions: {} });
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterAll(async () => {
|
||||
await temporary?.cleanup();
|
||||
home?.cleanup();
|
||||
if (originalHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = originalHome;
|
||||
});
|
||||
|
||||
async function fixture(requiresPreparationStop = false) {
|
||||
const runId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID(), taskId = randomUUID();
|
||||
const contextId = randomUUID(), leaseOwner = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, identifier: `NSC-${issueId}`, title: "Timed out startup",
|
||||
status: "in_progress", priority: "medium", assigneeAgentId: agentId });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running",
|
||||
runtimeMode: "native", nativeIssueId: issueId, invocationSource: "manual", contextSnapshot: { issueId } }).returning();
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Timed out startup", description: null }, environmentLeaseId: leaseId });
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1,
|
||||
controllerGeneration: 1, controllerBootId: randomUUID(), controllerPid: process.pid,
|
||||
controllerProcessStartedAt: new Date(), leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const [environment] = await db.insert(environments).values({ name: `Cleanup ${runId}`, driver: "plugin", status: "active", config: {} }).returning();
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId,
|
||||
issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment",
|
||||
metadata: { runnerTaskLease: true, runnerTaskReused: true } });
|
||||
const request = { companyId, agentId, runId, contextId, leaseOwner, deadlineAtMs: Date.now() - 1, requiresPreparationStop };
|
||||
const reload = () => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then(rows => rows[0]!);
|
||||
const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!);
|
||||
const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!);
|
||||
const stopReceipt = async (override: Record<string, unknown> = {}, preparationStopped = false) => {
|
||||
const current = await lease();
|
||||
const receipt = remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" });
|
||||
await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success",
|
||||
metadata: { ...current.metadata, ...(preparationStopped ? { runnerTaskPreparationStopped: true } : {}), remoteExecutionTermination: { ...receipt, ...override } } }).where(eq(environmentLeases.id, leaseId));
|
||||
};
|
||||
const acceptedResult = async () => {
|
||||
const current = await reload(), resultId = randomUUID();
|
||||
await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId,
|
||||
completionContractId: current.completionContractId!, serverFingerprint: resultId, schemaStatus: "accepted",
|
||||
resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed work" }, canonicalSha256: "a".repeat(64) });
|
||||
await db.update(nativeRunFinalizations).set({ resultId, phase: "workspace_finalizing" }).where(eq(nativeRunFinalizations.runId, runId));
|
||||
return resultId;
|
||||
};
|
||||
return { request, reload, owner, lease, stopReceipt, acceptedResult };
|
||||
}
|
||||
|
||||
async function saveEarlyContext(f: Awaited<ReturnType<typeof fixture>>) {
|
||||
const run = await f.reload(), owner = await f.owner(), lease = await f.lease();
|
||||
const context = { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId: run.id,
|
||||
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, leaseId: lease.id,
|
||||
leaseOwner: owner.leaseOwner, controllerGeneration: owner.controllerGeneration, attempt: owner.attempt };
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, nativeStartupContext: context } }).where(eq(heartbeatRuns.id, run.id));
|
||||
return context;
|
||||
}
|
||||
|
||||
it.each(["pending", "needs_reconciliation"])("keeps ownership until a %s replacement queued after the first teardown snapshot is verified stopped", async state => {
|
||||
const f = await fixture(true), original = await f.lease();
|
||||
let reply!: (result: { providerLeaseId: string }) => void, creating!: () => void;
|
||||
const entered = new Promise<void>(resolve => { creating = resolve; });
|
||||
const acquiring = createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId,
|
||||
issueId: original.issueId, provider: original.provider!, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!,
|
||||
metadata: { runnerTaskLease: true, runnerTaskCreationContextId: f.request.contextId } }, async () => new Promise(resolve => { reply = resolve; creating(); }));
|
||||
await entered;
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await f.stopReceipt({}, true);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending");
|
||||
const [pending] = await db.select().from(sandboxCreationOperations).where(and(
|
||||
eq(sandboxCreationOperations.runId, f.request.runId), eq(sandboxCreationOperations.requestKey, original.providerLeaseId!)));
|
||||
if (state === "needs_reconciliation") {
|
||||
await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }).where(eq(sandboxCreationOperations.id, pending.id));
|
||||
await recoverSandboxCreations(db, async () => null);
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, pending.id)))[0].state).toBe(state);
|
||||
const inspect = vi.fn(async () => null);
|
||||
await recoverSandboxCreations(db, inspect);
|
||||
expect(inspect).not.toHaveBeenCalled();
|
||||
await expect(queueSandboxCreationCleanup(db, { id: pending.id, companyId, environmentId: original.environmentId!,
|
||||
providerLeaseId: "unproven-task" })).rejects.toThrow("ownership changed");
|
||||
}
|
||||
reply({ providerLeaseId: "late-replacement" });
|
||||
const fresh = await acquiring;
|
||||
await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId,
|
||||
environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId, providerMetadata: { binding: "actual" } });
|
||||
if (state === "pending") await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) })
|
||||
.where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId)));
|
||||
expect(await nextEnvironmentCleanupAt(db)).not.toBeNull();
|
||||
const restarted = createDb(temporary.connectionString);
|
||||
try { await recoverSandboxCreations(restarted, async () => { throw new Error("A recorded task identity needs no new provider inspection"); }); }
|
||||
finally { await restarted.$client.end(); }
|
||||
const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId)));
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending");
|
||||
const replacement = (await environmentService(db).getLeaseById(creation.leaseId!))!;
|
||||
await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].leaseId).toBe(replacement.id);
|
||||
await expect(queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: "foreign-task" })).rejects.toThrow("ownership changed");
|
||||
await environmentService(db).releaseLease(replacement.id, "released", { cleanupStatus: "success",
|
||||
remoteExecutionTermination: remoteTerminationReceipt(replacement, { providerLeaseId: replacement.providerLeaseId, state: "stopped" }) });
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].state).toBe("resolved");
|
||||
await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0])
|
||||
.toMatchObject({ state: "resolved", leaseId: replacement.id });
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), async () => {});
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["before_inventory", "before_write", "complete_before_write", "ordinary"])("preserves startup cleanup through orphan recovery: %s", async phase => {
|
||||
const f = await fixture(), deps = callbacks(db);
|
||||
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseExpiresAt: new Date(0) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
if (phase === "before_inventory") await requestNativeStartupCleanup(db, f.request);
|
||||
const write = deps.setRunStatusFromLive.getMockImplementation()!;
|
||||
deps.setRunStatusFromLive.mockImplementation(async (...args) => {
|
||||
if (phase === "before_write" || phase === "complete_before_write") {
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
if (phase === "complete_before_write") await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt());
|
||||
}
|
||||
return write(...args);
|
||||
});
|
||||
const recovery = createHeartbeatRecovery(db, { ...deps, environmentRuntime: environmentRuntimeService(db),
|
||||
environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set<Promise<void>>(),
|
||||
mergeRunStopMetadataForAgent: (_agent: unknown, _status: unknown, options: { resultJson?: object }) => options?.resultJson,
|
||||
classifyAndPersistRunLiveness: async (run: unknown) => run, releaseEnvironmentLeasesForRun: vi.fn(),
|
||||
releaseIssueExecutionAndPromote: vi.fn(), scheduleInteractionContinuationInfrastructureRetryIfEligible: async () => null,
|
||||
} as unknown as HeartbeatRecoveryDependencies);
|
||||
await recovery.recoverActiveExecutions({ runId: f.request.runId }, { staleThresholdMs: 0 });
|
||||
expect((await f.reload()).status).toBe(phase === "ordinary" ? "failed" : "running");
|
||||
if (phase === "ordinary") return;
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt());
|
||||
expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(f.request.runId, "Startup cleanup retry"))
|
||||
.toMatchObject({ status: "cancelled" });
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["request", "acknowledge"])("registers queue intent before cleanup %s writes", async phase => {
|
||||
const f = await fixture(true);
|
||||
const writes: boolean[][] = [];
|
||||
const observe = async (queues: string[], work: () => Promise<unknown>) => {
|
||||
const transaction = db.transaction.bind(db);
|
||||
const spy = vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: typeof db) => Promise<unknown>) => transaction(async tx => {
|
||||
const update = tx.update.bind(tx);
|
||||
tx.update = ((table: unknown) => {
|
||||
if ([heartbeatRuns, nativeRunFinalizations, environmentLeases].includes(table as never)) {
|
||||
writes.push(queues.map(queue => databaseWorkPending(db, queue)));
|
||||
}
|
||||
return update(table as never);
|
||||
}) as typeof tx.update;
|
||||
return callback(tx as unknown as typeof db);
|
||||
})) as typeof db.transaction);
|
||||
try { await work(); } finally { spy.mockRestore(); }
|
||||
};
|
||||
if (phase === "request") {
|
||||
await observe([DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.environmentCleanup], () => requestNativeStartupCleanup(db, f.request));
|
||||
expect(writes).toEqual([[true, true], [true, true], [true, true]]);
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
} else {
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await observe([DELIVERY_QUEUES.agentLifecycle, DELIVERY_QUEUES.environmentCleanup], () => acknowledgeNativeStartupCleanup(db, f.request));
|
||||
expect(writes).toEqual([[true, true]]);
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the deadline queue to quiet and wakes again for the next startup failure", async () => {
|
||||
const schedule = executionControlSchedule(db);
|
||||
expect(await schedule.finalization()).toBeNull();
|
||||
for (let failure = 0; failure < 2; failure += 1) {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
expect(await schedule.finalization()).not.toBeNull();
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
await reconcileAbandonedExecutionControl(db, new Date(Date.now() + 1_000), { startupCleanup: async scope => {
|
||||
if (scope.runId !== f.request.runId) return;
|
||||
await control.cancelRunInternal(scope.runId, "Startup cleanup retry");
|
||||
await acknowledgeNativeStartupCleanup(db, scope);
|
||||
} });
|
||||
expect((await f.reload()).status).toBe("cancelled");
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null });
|
||||
expect(await schedule.finalization()).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("discovers a crash after warm prepare dispatch but before a cleanup intent was recorded", async () => {
|
||||
const f = await fixture(true), lease = await f.lease();
|
||||
await db.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: {
|
||||
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId,
|
||||
} } }).where(eq(environmentLeases.id, lease.id));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
const onClaim = vi.fn();
|
||||
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim });
|
||||
expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]);
|
||||
expect(onClaim).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId,
|
||||
state: "pending", requiresPreparationStop: true });
|
||||
expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["wrong_run", "future_deadline", "live_owner", "checkpointed", "missing_marker"])(
|
||||
"does not infer a timed-out startup from %s", async variant => {
|
||||
const f = await fixture(true), lease = await f.lease();
|
||||
if (variant !== "missing_marker") await db.update(environmentLeases).set({ metadata: { ...lease.metadata,
|
||||
runnerTaskStartup: { contextId: f.request.contextId, runId: variant === "wrong_run" ? randomUUID() : f.request.runId,
|
||||
deadlineAtMs: variant === "future_deadline" ? Date.now() + 60_000 : f.request.deadlineAtMs },
|
||||
} }).where(eq(environmentLeases.id, lease.id));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + (variant === "live_owner" ? 60_000 : -1_000)) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
if (variant === "checkpointed") await db.update(heartbeatRuns).set({ sessionIdAfter: "current-run-checkpoint" })
|
||||
.where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
it("persists an audited cancellation intent that a new database handle can recover without a generic hold", async () => {
|
||||
const f = await fixture();
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
|
||||
const run = await f.reload(), marker = readNativeStartupCleanup(run)!;
|
||||
expect(marker).toMatchObject({ state: "pending", contextId: f.request.contextId, leaseOwner: f.request.leaseOwner,
|
||||
controllerGeneration: 1, attempt: 1, taskId: (await f.lease()).providerLeaseId });
|
||||
expect(run).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING });
|
||||
expect(run.resultJson?.startupCancellation).toMatchObject({ requestedAt: expect.any(String) });
|
||||
expect(isNativeRunnerOwnershipHeld(run)).toBe(false);
|
||||
expect(await db.select({ id: activityLog.id }).from(activityLog).where(and(eq(activityLog.id, marker.auditId),
|
||||
eq(activityLog.companyId, companyId), eq(activityLog.action, "native.startup_cleanup_requested")))).toHaveLength(1);
|
||||
const restarted = createDb(temporary.connectionString), release = vi.fn(f.stopReceipt);
|
||||
await verifyNativeStartupCleanup(restarted, await f.reload(), release);
|
||||
await verifyNativeStartupCleanup(restarted, await f.reload(), release);
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id });
|
||||
expect((await f.owner()).controlDeadlineAt).not.toBeNull();
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, run.id));
|
||||
await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id });
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null });
|
||||
});
|
||||
|
||||
it("keeps cleanup bound to the explicit current lease when a released alias belongs to the same run", async () => {
|
||||
const f = await fixture(true), current = await f.lease();
|
||||
const aliasId = randomUUID();
|
||||
await db.insert(environmentLeases).values({ ...current, id: aliasId, providerLeaseId: randomUUID(),
|
||||
status: "released", releasedAt: new Date(), cleanupStatus: "success" });
|
||||
expect(await requestNativeStartupCleanup(db, { ...f.request, leaseId: current.id })).toBe(true);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ leaseId: current.id, taskId: current.providerLeaseId });
|
||||
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, aliasId)))[0]?.metadata?.runnerTaskStartupCleanup).toBeUndefined();
|
||||
});
|
||||
|
||||
it("recovers an exact Stop receipt committed before Core records completion", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await f.stopReceipt();
|
||||
const release = vi.fn(async () => { throw new Error("Already retired task must not be released twice"); });
|
||||
await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
});
|
||||
|
||||
it("keeps accelerated preparation pending when only the provider has stopped", async () => {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow(/pending|preparation/i);
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
});
|
||||
|
||||
it("automatically finishes pending cleanup after a late real worker reply and exact preparation Stop proof", async () => {
|
||||
const f = await fixture(true), lease = await f.lease();
|
||||
const entrypointPath = path.join(home.home, `cleanup-worker-${f.request.runId}.cjs`);
|
||||
await writeFile(entrypointPath, `
|
||||
const rl=require("node:readline").createInterface({input:process.stdin,crlfDelay:Infinity});
|
||||
const held=[];let stopped=false;
|
||||
const send=(id,result)=>process.stdout.write(JSON.stringify({jsonrpc:"2.0",id,result})+"\\n");
|
||||
rl.on("line",line=>{const m=JSON.parse(line);
|
||||
if(m.method==="initialize")send(m.id,{ok:true,supportedMethods:["environmentTask","configChanged"]});
|
||||
else if(m.method==="environmentTask"){
|
||||
const kind=m.params.operation.kind;
|
||||
if(kind==="prepare")held.push(m.id);
|
||||
else if(kind==="status")send(m.id,{kind:"status",taskId:m.params.taskId,phase:stopped?"cancelled":"running",executionStopped:stopped,preparationStopped:stopped&&held.length===0});
|
||||
else {if(kind==="stop")stopped=true;send(m.id,{kind:"accepted",taskId:m.params.taskId});}
|
||||
}else if(m.method==="configChanged"){for(const id of held.splice(0))send(id,{});send(m.id,{});}
|
||||
else if(m.method==="shutdown"){send(m.id,{});setImmediate(()=>process.exit(0));}
|
||||
});
|
||||
`, { mode: 0o600 });
|
||||
const manifest: PaperclipPluginManifestV1 = { id: "test.startup-cleanup", apiVersion: 1, version: "1.0.0",
|
||||
displayName: "Cleanup fixture", description: "Cleanup fixture", author: "Paperclip", categories: ["automation"],
|
||||
capabilities: [], entrypoints: { worker: "worker.cjs" } };
|
||||
const manager = createPluginWorkerManager();
|
||||
await manager.startWorker(manifest.id, { entrypointPath, manifest, config: {},
|
||||
instanceInfo: { instanceId: "cleanup", hostVersion: "1.0.0" }, apiVersion: 1, hostHandlers: {} });
|
||||
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId: f.request.runId } } as NativeExecutionInput, 100);
|
||||
const params = { companyId, agentId, runId: f.request.runId, environmentId: lease.environmentId, issueId: lease.issueId,
|
||||
driverKey: "tasks", taskId: lease.providerLeaseId!, config: {}, projectIds: [], lease: { providerLeaseId: lease.providerLeaseId! } };
|
||||
const operation = { kind: "prepare", projectIds: [], requirements: { schema: "paperclip.runner.launch-requirements.v1",
|
||||
identity: { runnerInstanceId: "fixture", environmentLeaseId: lease.id, runId: f.request.runId,
|
||||
normalizedSessionId: "session", turnId: "turn", itemId: "item" }, provider: "codex", model: null,
|
||||
instructions: "Synthetic", runtimeContext: null, settings: {}, approvalPolicy: "never", readOnly: false,
|
||||
preparedConversation: false, includeCollaborationModeInstructions: false } } as PluginEnvironmentTaskOperation;
|
||||
const release = async () => {
|
||||
await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "stop" } });
|
||||
const status = await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "status" } });
|
||||
if (status.kind === "status" && status.executionStopped && status.preparationStopped) {
|
||||
await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "complete" } });
|
||||
await f.stopReceipt({}, true);
|
||||
}
|
||||
};
|
||||
try {
|
||||
await expect(manager.call(manifest.id, "environmentTask", { ...params, operation }, 10, undefined, context.trackWorker())).rejects.toThrow("timed out");
|
||||
if (!context.signal.aborted) await new Promise<void>(resolve => context.signal.addEventListener("abort", () => resolve(), { once: true }));
|
||||
expect(context.pending).toBe(true);
|
||||
expect(await requestNativeStartupCleanup(db, { ...f.request, contextId: context.id, deadlineAtMs: context.deadlineAtMs })).toBe(true);
|
||||
expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false);
|
||||
await expect(cancelNativeSession(f.request.runId, "Startup timeout", { db, scope: "run" })).resolves.toMatchObject({ auditId: expect.any(String) });
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("pending");
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
await manager.call(manifest.id, "configChanged", { companyId, config: { release: true } });
|
||||
expect(context.pending).toBe(false);
|
||||
await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release);
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect((await f.lease()).metadata?.runnerTaskPreparationStopped).toBe(true);
|
||||
} finally { context.finish(); await manager.stopAll(); }
|
||||
});
|
||||
|
||||
it.each([false, true])("verifies an abandoned ephemeral startup without an inactive-owner acknowledgement (receipt=%s)", async stopped => {
|
||||
const prior = process.env.PAPERCLIP_EPHEMERAL_SERVER;
|
||||
process.env.PAPERCLIP_EPHEMERAL_SERVER = "true";
|
||||
try {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, async () => { if (stopped) await f.stopReceipt({}, true); }));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
if (stopped) {
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Startup timeout")).toMatchObject({ status: "cancelled" });
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
} else {
|
||||
await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("Task cleanup is still pending");
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
}
|
||||
expect(deps.verifyStartupCleanup).toHaveBeenCalled();
|
||||
} finally {
|
||||
if (prior === undefined) delete process.env.PAPERCLIP_EPHEMERAL_SERVER;
|
||||
else process.env.PAPERCLIP_EPHEMERAL_SERVER = prior;
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("terminalizes through assembled run control only after verified cleanup succeeds", async () => {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
let drained = false;
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("pending");
|
||||
expect(await f.reload()).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING });
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
drained = true;
|
||||
const result = await control.cancelRunInternal(f.request.runId, "Startup timeout");
|
||||
expect(result).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null });
|
||||
});
|
||||
|
||||
it("joins cleanup recorded while an operator Stop waits for native startup", async () => {
|
||||
const f = await fixture(true);
|
||||
let entered!: () => void, resume!: () => void;
|
||||
const waiting = new Promise<void>(resolve => { entered = resolve; });
|
||||
const settled = new Promise<void>(resolve => { resume = resolve; });
|
||||
const cancel = nativeSessionExecutor.cancelNativeSession;
|
||||
const spy = vi.spyOn(nativeSessionExecutor, "cancelNativeSession").mockImplementation(async (...args) => {
|
||||
const result = await cancel(...args);
|
||||
if (args[0] === f.request.runId) { entered(); await settled; }
|
||||
return result;
|
||||
});
|
||||
let drained = false;
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained)));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
try {
|
||||
const stopping = control.cancelRunInternal(f.request.runId, "Operator Stop");
|
||||
const outcome = stopping.then(value => ({ value }), error => ({ error }));
|
||||
await waiting;
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
|
||||
resume();
|
||||
expect(await outcome).toMatchObject({ error: expect.objectContaining({ message: "Task cleanup is still pending" }) });
|
||||
expect(deps.verifyStartupCleanup).toHaveBeenCalledOnce();
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
drained = true;
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
} finally { resume(); spy.mockRestore(); }
|
||||
});
|
||||
|
||||
it("rejects a cancellation commit when cleanup arrives after its final read", async () => {
|
||||
const f = await fixture(true);
|
||||
const deps = callbacks(db), commit = deps.setRunStatusFromLive.getMockImplementation()!;
|
||||
deps.setRunStatusFromLive.mockImplementationOnce(async (...args) => {
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
|
||||
return commit(...args);
|
||||
});
|
||||
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true)));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "running" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
expect(deps.startNextQueuedRunForAgent).not.toHaveBeenCalled();
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
it("restarts the real cancellation flow after remote Stop committed before Core acknowledgement", async () => {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await f.stopReceipt({}, true);
|
||||
const restarted = createDb(temporary.connectionString), release = vi.fn(async () => { throw new Error("Stop receipt already persisted"); });
|
||||
const deps = callbacks(restarted);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(restarted, run, release);
|
||||
const control = createHeartbeatRunControl(restarted, deps);
|
||||
const reconcile = vi.fn(async (scope: { companyId: string; runId: string }) => {
|
||||
if (scope.runId !== f.request.runId) return;
|
||||
expect(scope.companyId).toBe(companyId);
|
||||
await control.cancelRunInternal(scope.runId, "Startup timeout recovered after restart");
|
||||
await acknowledgeNativeStartupCleanup(restarted, scope);
|
||||
});
|
||||
await reconcileAbandonedExecutionControl(restarted, new Date(Date.now() + 1_000), { startupCleanup: reconcile });
|
||||
expect(reconcile.mock.calls.some(([scope]) => scope.runId === f.request.runId)).toBe(true);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
expect((await f.reload()).status).toBe("cancelled");
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null });
|
||||
});
|
||||
|
||||
it.each(["providerLeaseId", "runId", "leaseId"])("rejects a Stop receipt for another %s", async field => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({ [field]: randomUUID() }))).rejects.toThrow("pending");
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it("does not issue a Stop against a newer controller generation", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
const release = vi.fn(f.stopReceipt);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("ownership changed");
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it("rejects a stale generation that changes while Stop is in flight", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {
|
||||
await f.stopReceipt();
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
})).rejects.toThrow("ownership changed");
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it("preserves a final result that wins while verified cleanup is in flight", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
let resultId: string;
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {
|
||||
await f.stopReceipt();
|
||||
resultId = await f.acceptedResult();
|
||||
})).rejects.toThrow("ownership changed");
|
||||
expect(await f.owner()).toMatchObject({ resultId: resultId!, phase: "workspace_finalizing" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it.each(["native_adopted_runner_authentication_timeout", "native_execution_ownership_unverified"])("preserves the existing %s hold", async errorCode => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ errorCode, nativePhase: "terminal_failure" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(false);
|
||||
const current = await f.reload();
|
||||
expect(current).toMatchObject({ status: "running", errorCode, nativePhase: "terminal_failure" });
|
||||
expect(readNativeStartupCleanup(current)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not replace a final result with a timeout intent", async () => {
|
||||
const f = await fixture();
|
||||
const resultId = await f.acceptedResult();
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(false);
|
||||
expect((await f.owner()).resultId).toBe(resultId);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
it("promotes a pre-capability context before an ordinary deadline after the audited intent write fails", async () => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
await db.execute(sql`create function reject_startup_intent() returns trigger language plpgsql as $$ begin
|
||||
if new.action = 'native.startup_cleanup_requested' then raise exception 'Startup intent write unavailable'; end if; return new; end $$`);
|
||||
await db.execute(sql`create trigger reject_startup_intent before insert on activity_log for each row execute function reject_startup_intent()`);
|
||||
try { await expect(requestNativeStartupCleanup(db, f.request)).rejects.toThrow(); }
|
||||
finally { await db.execute(sql`drop trigger reject_startup_intent on activity_log`); await db.execute(sql`drop function reject_startup_intent()`); }
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
await db.update(heartbeatRuns).set({ executionControlDeadlineAt: new Date(Date.now() - 1_000) }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await reconcileAbandonedExecutionControl(db);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId, requiresPreparationStop: false, state: "pending" });
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt());
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
});
|
||||
|
||||
it.each([false, true])("recovers an early context without replacing stronger preparation proof (prepared=%s)", async prepared => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
if (prepared) { const lease = await f.lease(); await db.update(environmentLeases).set({ metadata: { ...lease.metadata,
|
||||
runnerTaskStartup: { contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId } } }).where(eq(environmentLeases.id, lease.id)); }
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
const onClaim = vi.fn();
|
||||
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim });
|
||||
expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]);
|
||||
expect(onClaim).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: prepared, state: "pending" });
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, prepared));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
});
|
||||
|
||||
it.each([false, true])("defers an early context to a concrete owner deadline (locally active=%s)", async active => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
const owner = await f.owner();
|
||||
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
liveRunExecutions: active ? new Set([f.request.runId]) : undefined,
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(result).toMatchObject([{ kind: "awaiting_evidence", runId: f.request.runId, retryAt: owner.leaseExpiresAt }]);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
it("does not turn an early context into cleanup after the current-run checkpoint", async () => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
await db.update(heartbeatRuns).set({ sessionIdAfter: "current-checkpoint" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
it("locks the selected lease before choosing the stronger preparation proof", async () => {
|
||||
const f = await fixture(), lease = await f.lease();
|
||||
let changed!: () => void, commit!: () => void;
|
||||
const written = new Promise<void>(resolve => { changed = resolve; });
|
||||
const release = new Promise<void>(resolve => { commit = resolve; });
|
||||
const preparing = db.transaction(async tx => {
|
||||
await tx.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: {
|
||||
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId,
|
||||
} } }).where(eq(environmentLeases.id, lease.id));
|
||||
changed();
|
||||
await release;
|
||||
});
|
||||
await written;
|
||||
const requesting = requestNativeStartupCleanup(db, f.request);
|
||||
try {
|
||||
await vi.waitFor(async () => {
|
||||
const rows = await db.execute(sql`select query from pg_stat_activity where wait_event_type = 'Lock'
|
||||
and query like 'select%from "environment_leases"%'`);
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
});
|
||||
} finally { commit(); }
|
||||
await preparing;
|
||||
await requesting;
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: true });
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow("pending");
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
|
||||
});
|
||||
|
||||
it("clears only its owned startup context before fallback and leaves no expired cleanup authority", async () => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
const scope = { companyId, runId: f.request.runId, contextId: f.request.contextId,
|
||||
leaseOwner: f.request.leaseOwner, controllerGeneration: 1 };
|
||||
await expect(clearNativeStartupContext(db, { ...scope, controllerGeneration: 2 })).rejects.toThrow("lost authority");
|
||||
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined();
|
||||
await clearNativeStartupContext(db, scope);
|
||||
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeUndefined();
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["cleanup", "terminal"])("does not clear the startup breadcrumb after %s wins", async winner => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
if (winner === "cleanup") await requestNativeStartupCleanup(db, f.request);
|
||||
else await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await expect(clearNativeStartupContext(db, { companyId, runId: f.request.runId, contextId: f.request.contextId,
|
||||
leaseOwner: f.request.leaseOwner, controllerGeneration: 1 })).rejects.toThrow("lost authority");
|
||||
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined();
|
||||
if (winner === "cleanup") expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ state: "pending" });
|
||||
});
|
||||
|
||||
it("requires exact task Stop proof when retrying cleanup of a never-dispatched replacement", async () => {
|
||||
const f = await fixture(true), original = await f.lease(), pluginId = randomUUID();
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-abandoned-task", version: "1", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
|
||||
await db.update(environments).set({ companyId, config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } })
|
||||
.where(eq(environments.id, original.environmentId!));
|
||||
const fresh = await createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId,
|
||||
issueId: original.issueId, provider: original.provider!, pluginId, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!,
|
||||
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, agentId,
|
||||
runnerTaskCreationContextId: f.request.contextId } }, async () => ({ providerLeaseId: "late-never-dispatched" }));
|
||||
await queueSandboxCreationCleanup(db, { id: String(fresh.metadata!.creationOperationId), companyId,
|
||||
environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
|
||||
const creation = async () => (await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId))))[0]!;
|
||||
const replacement = async () => (await environmentService(db).getLeaseById((await creation()).leaseId!))!;
|
||||
expect((await replacement()).metadata?.runnerTaskDispatched).not.toBe(true);
|
||||
await requestNativeStartupCleanup(db, { ...f.request, leaseId: original.id });
|
||||
await f.stopReceipt({}, true);
|
||||
let proof: "void" | "running" | "foreign" | "exact" = "void";
|
||||
const calls: string[] = [];
|
||||
const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentTask", "environmentReleaseLease"] }),
|
||||
drainStartup: vi.fn(async () => undefined),
|
||||
call: async (_plugin: string, method: string, params: any) => {
|
||||
calls.push(method === "environmentTask" ? params.operation.kind : method);
|
||||
if (method === "environmentReleaseLease") return undefined;
|
||||
if (params.operation.kind === "status") return proof === "void" ? undefined : {
|
||||
kind: "status", taskId: proof === "foreign" ? "another-task" : params.taskId,
|
||||
phase: proof === "running" ? "running" : "cancelled", executionStopped: proof !== "running",
|
||||
};
|
||||
return { kind: "accepted", taskId: params.taskId };
|
||||
},
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
|
||||
const recovery = createHeartbeatRecovery(db, { ...callbacks(db), environmentRuntime: runtime,
|
||||
environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set<Promise<void>>(),
|
||||
acknowledgeRemoteStop: async () => undefined, resumeRemoteStopComments: async () => undefined,
|
||||
} as unknown as HeartbeatRecoveryDependencies);
|
||||
const stopDeps = callbacks(db);
|
||||
stopDeps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, async () => {});
|
||||
const control = createHeartbeatRunControl(db, stopDeps);
|
||||
for (const missing of ["void", "running", "foreign"] as const) {
|
||||
proof = missing;
|
||||
const swept = await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } });
|
||||
const cancelled = await control.cancelRunInternal(f.request.runId, "Startup cleanup retry").then(() => true, () => false);
|
||||
const lease = await replacement();
|
||||
expect({ destroyed: swept.destroyed, status: lease.status, verifiedReceipt: hasRemoteTerminationReceipt(lease),
|
||||
creationState: (await creation()).state, cancelled, leaseOwner: (await f.owner()).leaseOwner }).toEqual({
|
||||
destroyed: 0, status: "pending_cleanup", verifiedReceipt: false, creationState: "cleanup_pending", cancelled: false, leaseOwner: f.request.leaseOwner,
|
||||
});
|
||||
}
|
||||
proof = "exact";
|
||||
expect((await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } })).destroyed).toBe(1);
|
||||
expect(calls).toContain("stop");
|
||||
expect(calls).toContain("status");
|
||||
expect(calls).toContain("complete");
|
||||
expect(workers!.drainStartup).toHaveBeenCalledWith(f.request.contextId);
|
||||
expect(hasRemoteTerminationReceipt(await replacement())).toBe(true);
|
||||
expect((await creation()).state).toBe("resolved");
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Startup cleanup retry")).toMatchObject({ status: "cancelled" });
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
});
|
||||
@@ -332,6 +332,36 @@ describe("clampProviderSpanAttributes", () => {
|
||||
});
|
||||
|
||||
describe("nested worker timing receipts",()=>{
|
||||
it("keeps lease verification beneath its worker and clamps provider state", () => {
|
||||
mockRecordSpan.mockReset();
|
||||
const rootParent = { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "1111111111111111", traceFlags: 1 };
|
||||
const verifyParent = { ...rootParent, spanId: "2222222222222222" };
|
||||
mockRecordSpan.mockReturnValueOnce(rootParent).mockReturnValueOnce(verifyParent);
|
||||
const now = Date.now();
|
||||
recordWorkerProviderSpan({ name: "plugin.worker", startTimeMs: now - 100, endTimeMs: now, children: [
|
||||
{ name: "lease.verify", startTimeMs: now - 90, endTimeMs: now - 10, children: [
|
||||
{ name: "lease.sentinel", startTimeMs: now - 80, endTimeMs: now - 20,
|
||||
attributes: { "paperclip.sandbox.startup.state": "archived", token: "private-token" } },
|
||||
] },
|
||||
] }, { traceparent: VALID_TRACEPARENT });
|
||||
expect(mockRecordSpan.mock.calls[1][0]).toMatchObject({ name: "sandbox.daytona.lease.verify", parent: rootParent });
|
||||
expect(mockRecordSpan.mock.calls[2][0]).toMatchObject({ name: "sandbox.daytona.lease.sentinel", parent: verifyParent,
|
||||
startTimeMs: now - 80, endTimeMs: now - 20, attributes: { "paperclip.sandbox.startup.state": "archived" } });
|
||||
expect(JSON.stringify(mockRecordSpan.mock.calls)).not.toContain("private-token");
|
||||
expect(clampProviderSpanAttributes({ "paperclip.sandbox.startup.state": "private-state" })).toEqual({});
|
||||
});
|
||||
|
||||
it("admits every closed lease phase and rejects dynamic phase names", () => {
|
||||
mockRecordSpan.mockReset();
|
||||
const names = ["lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup",
|
||||
"lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", "lease.shell",
|
||||
"lease.create", "lease.expiry", "lease.cleanup", "lease.private-resource-id"];
|
||||
for (const name of names) recordWorkerProviderSpan({ name }, { traceparent: VALID_TRACEPARENT });
|
||||
expect(mockRecordSpan.mock.calls.map(call => call[0].name)).toEqual([
|
||||
...names.slice(0, -1).map(name => `sandbox.daytona.${name}`), "sandbox.daytona.other",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses host-generated parents, clamps content and bounds child clocks",()=>{
|
||||
mockRecordSpan.mockReset();
|
||||
const childParent={traceId:"0af7651916cd43dd8448eb211c80319c",spanId:"1111111111111111",traceFlags:1};
|
||||
|
||||
@@ -899,3 +899,73 @@ second-line\" status=401`,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("redaction of long letter/digit runs", () => {
|
||||
const typescriptSource = [
|
||||
"export function compactRunLogChunk(chunk: string, maxChars = 65536) {",
|
||||
" const normalized = redactSensitiveText(chunk);",
|
||||
" if (normalized.length <= maxChars) return normalized;",
|
||||
" return normalized.slice(0, maxChars);",
|
||||
"}",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
function base64OfTypeScriptSource(length: number) {
|
||||
const repeatedSource = typescriptSource.repeat(Math.ceil(length / typescriptSource.length));
|
||||
return Buffer.from(repeatedSource).toString("base64").slice(0, length);
|
||||
}
|
||||
|
||||
function redactWithinTimeBudget(input: string) {
|
||||
const startedAt = performance.now();
|
||||
const output = redactSensitiveText(input);
|
||||
expect(performance.now() - startedAt).toBeLessThan(1_000);
|
||||
return output;
|
||||
}
|
||||
|
||||
it("redacts secret field names that carry a prefix in plain and escaped JSON", () => {
|
||||
const cases = [
|
||||
[`{"myApiKey": "plain-prefixed-secret"}`, `{"myApiKey": "${REDACTED_EVENT_VALUE}"}`],
|
||||
[`{"x_auth_token":"snake-prefixed-secret","status":"ok"}`, `{"x_auth_token":"${REDACTED_EVENT_VALUE}","status":"ok"}`],
|
||||
[`GITHUB_TOKEN: "upper-prefixed-secret"`, `GITHUB_TOKEN: "${REDACTED_EVENT_VALUE}"`],
|
||||
[`{'db_password': 'single-quoted-secret'}`, `{'db_password': '${REDACTED_EVENT_VALUE}'}`],
|
||||
[`text"apiKey":"glued-quote-secret"`, `text"apiKey":"${REDACTED_EVENT_VALUE}"`],
|
||||
[`aGVsbG8apiKey:"glued-run-secret"`, `aGVsbG8apiKey:"${REDACTED_EVENT_VALUE}"`],
|
||||
[String.raw`{\"myApiKey\":\"escaped-prefixed-secret\"}`, String.raw`{\"myApiKey\":\"${REDACTED_EVENT_VALUE}\"}`],
|
||||
[
|
||||
String.raw`{\"x_auth_token\":\"escaped-snake-secret\",\"status\":\"ok\"}`,
|
||||
String.raw`{\"x_auth_token\":\"${REDACTED_EVENT_VALUE}\",\"status\":\"ok\"}`,
|
||||
],
|
||||
[String.raw`aGVsbG8\"GITHUB_TOKEN\":\"escaped-glued-secret\"`, String.raw`aGVsbG8\"GITHUB_TOKEN\":\"${REDACTED_EVENT_VALUE}\"`],
|
||||
];
|
||||
|
||||
for (const [input, expected] of cases) {
|
||||
expect(redactSensitiveText(input)).toBe(expected);
|
||||
expect(redactSensitiveText(expected)).toBe(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps base64 of source text intact and fast while redacting a secret glued to its end", () => {
|
||||
const blob = base64OfTypeScriptSource(48 * 1024);
|
||||
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
|
||||
|
||||
const shellCommand = `printf '%s' '${blob}' | base64 -d > src/compact.ts`;
|
||||
expect(redactWithinTimeBudget(shellCommand)).toBe(shellCommand);
|
||||
|
||||
const toolCallLine = JSON.stringify({ type: "tool_use", input: { command: shellCommand } });
|
||||
expect(redactWithinTimeBudget(toolCallLine)).toBe(toolCallLine);
|
||||
const serializedToolCallLine = JSON.stringify(toolCallLine);
|
||||
expect(redactWithinTimeBudget(serializedToolCallLine)).toBe(serializedToolCallLine);
|
||||
|
||||
const gluedSecret = redactWithinTimeBudget(`${blob}apiKey:"run-glued-secret"`);
|
||||
expect(gluedSecret).toBe(`${blob}apiKey:"${REDACTED_EVENT_VALUE}"`);
|
||||
});
|
||||
|
||||
it("scans long dash-separated runs in linear time", () => {
|
||||
for (const run of ["a-".repeat(60_000), "a--".repeat(40_000)]) {
|
||||
expect(redactWithinTimeBudget(`${run} Authorization: Bearer dash-run-secret`))
|
||||
.toBe(`${run} Authorization: ${REDACTED_EVENT_VALUE}`);
|
||||
expect(redactWithinTimeBudget(`${run}Authorization: Bearer glued-run-secret`))
|
||||
.toBe(`${run}Authorization: ${REDACTED_EVENT_VALUE}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -152,15 +152,20 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
});
|
||||
const runtime = { execute, syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">;
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: runtime });
|
||||
const app = filesApp(ids.company, remoteFiles);
|
||||
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
|
||||
}
|
||||
|
||||
function filesApp(companyId: string, remoteFiles: ReturnType<typeof createRemoteWorkspaceFiles>) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", userId: "local-board", companyIds: [ids.company], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
|
||||
req.actor = { type: "board", userId: "local-board", companyIds: [companyId], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
|
||||
next();
|
||||
});
|
||||
app.use("/api", fileResourceRoutes(db, { remoteFiles }));
|
||||
app.use(errorHandler);
|
||||
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
|
||||
return app;
|
||||
}
|
||||
|
||||
it("lists, resolves, previews, and downloads live bytes without a controller checkout", async () => {
|
||||
@@ -323,7 +328,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422, details: { code: "file_changed" } });
|
||||
});
|
||||
|
||||
async function savedFileFixture() {
|
||||
async function savedFileFixture(keepLocation = false) {
|
||||
const f = await fixture(), storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "saved-file-budget-"));
|
||||
directories.push(storageRoot);
|
||||
const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects")));
|
||||
@@ -335,11 +340,113 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
const saved = await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
|
||||
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: keepLocation ? f.authority.location : null, snapshotId: snapshot.id } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
return { ...f, storage, saved, input: { companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId: snapshot.id } };
|
||||
}
|
||||
|
||||
it("serves committed saved bytes through ordinary routes while the live allocation is unavailable", async () => {
|
||||
const f = await savedFileFixture(true);
|
||||
f.execute.mockRejectedValue(new Error("synthetic live allocation unavailable"));
|
||||
const load = createSavedWorkspaceLoader(db, f.storage), closed: Promise<void>[] = [];
|
||||
const loadSavedWorkspace = vi.fn(async (input: Parameters<typeof load>[0]) => {
|
||||
const saved = await load(input);
|
||||
let finish!: () => void, fail!: (error: unknown) => void;
|
||||
closed.push(new Promise<void>((resolve, reject) => { finish = resolve; fail = reject; }));
|
||||
return { ...saved, cleanup: async () => {
|
||||
try { await saved.cleanup(); finish(); } catch (error) { fail(error); throw error; }
|
||||
} };
|
||||
});
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const app = filesApp(f.ids.company, remoteFiles), base = `/api/issues/${f.ids.issue}/file-resources`;
|
||||
const preview = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
|
||||
expect(preview.body).toMatchObject({ content: { data: "Remote useful result\n" }, resource: { source: { kind: "saved", snapshotId: f.saved.id } } });
|
||||
const listed = await request(app).get(`${base}/list`).query({ workspace: "execution", path: "docs" }).expect(200);
|
||||
expect(listed.body).toMatchObject({ workspace: { source: { kind: "saved", snapshotId: f.saved.id } } });
|
||||
const resolved = await request(app).get(`${base}/resolve`).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
|
||||
expect(resolved.body).toMatchObject({ source: { kind: "saved", snapshotId: f.saved.id }, capabilities: { preview: true, download: true } });
|
||||
const download = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md", download: true }).expect(200);
|
||||
expect(download.text).toBe("Remote useful result\n");
|
||||
expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledTimes(4);
|
||||
// A streamed response can finish at the client before its server finally block.
|
||||
await Promise.all(closed);
|
||||
for (const loaded of loadSavedWorkspace.mock.results) await expect(fs.stat((await loaded.value).root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
expect(workspace!.metadata?.remoteWorkspace).toMatchObject({ location: f.authority.location, dirty: false, writerRunId: null });
|
||||
});
|
||||
|
||||
it.each(["dirty", "writer"] as const)("keeps %s work on the live path and never substitutes its older save", async state => {
|
||||
const f = await savedFileFixture(true);
|
||||
let writerRunId: string | null = null;
|
||||
if (state === "writer") {
|
||||
const [agent] = await db.insert(agents).values({ companyId: f.ids.company, name: "Current workspace writer" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.ids.company, agentId: agent!.id, status: "running" }).returning();
|
||||
writerRunId = run!.id;
|
||||
}
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: state === "dirty", writerRunId } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
await fs.writeFile(path.join(f.root, "docs", "result.md"), "Unsaved live result\n");
|
||||
const loadSavedWorkspace = vi.fn(createSavedWorkspaceLoader(db, f.storage));
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const app = filesApp(f.ids.company, remoteFiles), route = `/api/issues/${f.ids.issue}/file-resources/content`;
|
||||
const current = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
|
||||
expect(current.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Unsaved live result\n" } });
|
||||
f.execute.mockRejectedValueOnce(new Error("synthetic live allocation unavailable"));
|
||||
const unavailable = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(422);
|
||||
expect(unavailable.body).not.toHaveProperty("content"); expect(loadSavedWorkspace).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([true, false])("preserves the optional saved-loader boundary with a recorded location=%s", async keepLocation => {
|
||||
const f = await savedFileFixture(keepLocation);
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
|
||||
.query({ workspace: "execution", path: "docs/result.md" }).expect(keepLocation ? 200 : 422);
|
||||
if (keepLocation) {
|
||||
expect(response.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Remote useful result\n" } });
|
||||
expect(f.execute).toHaveBeenCalled();
|
||||
} else {
|
||||
expect(response.body).not.toHaveProperty("content"); expect(f.execute).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["company", "workspace", "uncommitted"] as const)("rejects a saved snapshot with an invalid %s binding before object retrieval", async invalid => {
|
||||
const f = await savedFileFixture(true);
|
||||
if (invalid === "company") {
|
||||
const [other] = await db.insert(companies).values({ name: "Other saved-file company" }).returning();
|
||||
await db.update(executionWorkspaceSnapshots).set({ companyId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
} else if (invalid === "workspace") {
|
||||
const [other] = await db.insert(executionWorkspaces).values({ companyId: f.ids.company, mode: "isolated_workspace",
|
||||
strategyType: "project_primary", providerType: "cloud_sandbox", name: "Other saved-file workspace" }).returning();
|
||||
await db.update(executionWorkspaceSnapshots).set({ executionWorkspaceId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
} else await db.update(executionWorkspaceSnapshots).set({ status: "uploaded" }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
const getObject = vi.spyOn(f.storage, "getObject");
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace: createSavedWorkspaceLoader(db, f.storage),
|
||||
environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
|
||||
.query({ workspace: "execution", path: "docs/result.md" });
|
||||
expect(response.status).toBeGreaterThanOrEqual(400); expect(response.body).not.toHaveProperty("content");
|
||||
expect(getObject).not.toHaveBeenCalled(); expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
|
||||
getObject.mockRestore();
|
||||
});
|
||||
|
||||
it("fences a saved read if a new writer claims the workspace during extraction and removes its scratch files", async () => {
|
||||
const f = await savedFileFixture(true), load = createSavedWorkspaceLoader(db, f.storage);
|
||||
let savedRoot: string | undefined;
|
||||
const loadSavedWorkspace = async (input: Parameters<typeof load>[0]) => {
|
||||
const saved = await load(input); savedRoot = saved.root;
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: true, epoch: 2 } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
return saved;
|
||||
};
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
|
||||
.query({ workspace: "execution", path: "docs/result.md" }).expect(422);
|
||||
expect(response.body.details.code).toBe("workspace_changed"); expect(response.body).not.toHaveProperty("content");
|
||||
expect(savedRoot).toBeDefined(); await expect(fs.stat(savedRoot!)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shares fail-fast saved Files admission across loader instances until owned cleanup completes", async () => {
|
||||
const f = await savedFileFixture(), availableBytes = async () => 8n * 1024n ** 3n;
|
||||
const first = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
|
||||
@@ -625,7 +732,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
});
|
||||
|
||||
it.each(["confirmed_stop", "unknown_stop", "dirty", "foreign_receipt"])(
|
||||
"uses saved bytes only for a clean workspace with an exact stopped receipt (%s)", async state => {
|
||||
"uses clean saved authority independently of an allocation's stop receipt (%s)", async state => {
|
||||
const f = await fixture();
|
||||
const snapshotId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
@@ -650,7 +757,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
|
||||
const content = workspaceFileResourceService(db, undefined, { remoteFiles }).readContent(f.ids.issue,
|
||||
{ workspace: "execution", path: "docs/result.md" });
|
||||
if (state === "confirmed_stop") {
|
||||
if (state !== "dirty") {
|
||||
await expect(content).resolves.toMatchObject({ content: { data: "Saved stopped result\n" },
|
||||
resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } } });
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId });
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { eq, sql, type SQL } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createDb, plugins, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared";
|
||||
import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js";
|
||||
import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js";
|
||||
@@ -39,12 +39,12 @@ describe.skipIf(!support.supported)("sandbox provider schema reads", () => {
|
||||
const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey });
|
||||
async function measure<T>(client: Db, operation: () => Promise<T>) {
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
|
||||
installDatabaseResponseCapture(client, { plugins }, capture);
|
||||
try {
|
||||
const value = await operation();
|
||||
capture.flush();
|
||||
const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
|
||||
const entries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
|
||||
return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0),
|
||||
calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) };
|
||||
} finally { capture.flush(); }
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, UploadPartCommand, DeleteObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import { Readable } from "node:stream";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createServer, type ServerResponse } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { putS3Multipart } from "../storage/s3-multipart.js";
|
||||
import { createS3StorageProvider } from "../storage/s3-provider.js";
|
||||
@@ -7,6 +10,94 @@ import { createStorageService } from "../storage/service.js";
|
||||
import { STORAGE_FAILURE_CLEANUP_MS } from "../storage/failure-cleanup.js";
|
||||
|
||||
describe("streamed S3 multipart uploads", () => {
|
||||
it("sends concurrent exact part bodies through the real S3 HTTP client", async () => {
|
||||
const received = new Map<number, string>();
|
||||
const responses = new Map<number, ServerResponse>();
|
||||
let completedParts = "";
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url!, "http://localhost");
|
||||
const digest = createHash("sha256");
|
||||
let text = "";
|
||||
request.on("data", chunk => { digest.update(chunk); if (request.method === "POST") text += chunk; });
|
||||
request.on("end", () => {
|
||||
if (request.method === "PUT") {
|
||||
const number = Number(url.searchParams.get("partNumber"));
|
||||
received.set(number, digest.digest("hex"));
|
||||
responses.set(number, response);
|
||||
} else if (url.searchParams.has("uploads")) {
|
||||
response.end('<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>');
|
||||
} else {
|
||||
completedParts = text;
|
||||
response.end('<CompleteMultipartUploadResult><ETag>complete</ETag></CompleteMultipartUploadResult>');
|
||||
}
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); });
|
||||
const client = new S3Client({ region: "us-east-1", endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
|
||||
forcePathStyle: true, requestChecksumCalculation: "WHEN_REQUIRED", credentials: { accessKeyId: "fixture", secretAccessKey: "fixture" } });
|
||||
const cancellation = new AbortController();
|
||||
const body = Readable.from((async function* () {
|
||||
for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024, Math.floor(index / 128) + 1);
|
||||
})());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body, signal: cancellation.signal }).catch(error => error);
|
||||
try {
|
||||
await expect.poll(() => received.size, { timeout: 3000 }).toBe(4);
|
||||
for (const number of [4, 2, 3, 1]) {
|
||||
expect(received.get(number)).toBe(createHash("sha256").update(Buffer.alloc(8 * 1024 * 1024, number)).digest("hex"));
|
||||
responses.get(number)!.writeHead(200, { ETag: `part-${number}` }).end();
|
||||
}
|
||||
expect(await saving).toBeUndefined();
|
||||
expect([...completedParts.matchAll(/<PartNumber>(\d+)<\/PartNumber>/g)].map(match => Number(match[1]))).toEqual([1, 2, 3, 4]);
|
||||
} finally {
|
||||
cancellation.abort();
|
||||
await saving;
|
||||
client.destroy();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>(resolve => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
it("overlaps slow part acknowledgements and completes exact bytes in source order", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const received = new Map<number, Buffer>();
|
||||
const acknowledgements = new Map<number, () => void>();
|
||||
let completed = false;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of command.input.Body as Readable) chunks.push(chunk);
|
||||
received.set(command.input.PartNumber, Buffer.concat(chunks));
|
||||
await new Promise<void>(resolve => acknowledgements.set(command.input.PartNumber, resolve));
|
||||
return { ETag: `part-${command.input.PartNumber}` };
|
||||
}
|
||||
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
|
||||
expect(command.input.MultipartUpload.Parts).toEqual([1, 2, 3, 4].map(PartNumber => ({ PartNumber, ETag: `part-${PartNumber}` })));
|
||||
completed = true;
|
||||
return {};
|
||||
});
|
||||
const length = 25 * 1024 * 1024;
|
||||
const body = Readable.from((async function* () {
|
||||
for (let offset = 0; offset < length; offset += 64 * 1024) yield Buffer.alloc(Math.min(64 * 1024, length - offset), offset / (64 * 1024) % 251);
|
||||
})());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: length, body });
|
||||
try {
|
||||
await expect.poll(() => received.size, { timeout: 1000 }).toBe(4);
|
||||
expect(completed).toBe(false);
|
||||
for (const number of [4, 2, 3, 1]) acknowledgements.get(number)!();
|
||||
await saving;
|
||||
const actual = Buffer.concat([1, 2, 3, 4].map(number => received.get(number)!));
|
||||
expect(actual.length).toBe(length);
|
||||
for (let offset = 0; offset < length; offset += 64 * 1024) {
|
||||
expect(actual.subarray(offset, Math.min(length, offset + 64 * 1024)).every(byte => byte === offset / (64 * 1024) % 251)).toBe(true);
|
||||
}
|
||||
expect(completed).toBe(true);
|
||||
} finally {
|
||||
const release = setInterval(() => { for (const acknowledge of acknowledgements.values()) acknowledge(); }, 1);
|
||||
try { await saving; } finally { clearInterval(release); send.mockRestore(); client.destroy(); }
|
||||
}
|
||||
});
|
||||
|
||||
it("streams exact ordered parts without buffering the full response", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const lengths: number[] = [];
|
||||
@@ -33,6 +124,109 @@ describe("streamed S3 multipart uploads", () => {
|
||||
expect(lengths).toEqual([8, 8, 1].map(mb => mb * 1024 * 1024));
|
||||
} finally { send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
it("keeps source backpressure and at most four part requests while acknowledgements stall", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
let releaseReading!: () => void, releaseAcknowledgements!: () => void;
|
||||
const reading = new Promise<void>(resolve => { releaseReading = resolve; });
|
||||
const acknowledgements = new Promise<void>(resolve => { releaseAcknowledgements = resolve; });
|
||||
let yielded = 0, active = 0, peak = 0, received = 0;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
active++; peak = Math.max(peak, active);
|
||||
await reading;
|
||||
for await (const chunk of command.input.Body as Readable) expect(chunk.length).toBeLessThanOrEqual(64 * 1024);
|
||||
received++;
|
||||
await acknowledgements;
|
||||
active--;
|
||||
return { ETag: `part-${command.input.PartNumber}` };
|
||||
}
|
||||
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
|
||||
return {};
|
||||
});
|
||||
const body = Readable.from((async function* () {
|
||||
for (let index = 0; index < 1600; index++) { yielded++; yield Buffer.alloc(64 * 1024); }
|
||||
})());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 100 * 1024 * 1024, body });
|
||||
try {
|
||||
await expect.poll(() => active).toBe(1);
|
||||
expect(yielded).toBeLessThanOrEqual(3);
|
||||
releaseReading();
|
||||
await expect.poll(() => received).toBe(4);
|
||||
expect(yielded).toBeLessThanOrEqual(4 * 128 + 3);
|
||||
expect(peak).toBe(4);
|
||||
releaseAcknowledgements();
|
||||
await saving;
|
||||
expect(received).toBe(13);
|
||||
expect(yielded).toBe(1600);
|
||||
expect(peak).toBe(4);
|
||||
} finally { releaseReading(); releaseAcknowledgements(); await saving; send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
|
||||
it("settles cancelled siblings before aborting a failed multipart upload", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const cancellations: AbortSignal[] = [];
|
||||
let failPart!: () => void, releaseSiblings!: () => void;
|
||||
const settlement = new Promise<void>(resolve => { releaseSiblings = resolve; });
|
||||
let received = 0, settled = 0, aborted = false, completed = false;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
for await (const _chunk of command.input.Body as Readable) {}
|
||||
received++;
|
||||
if (command.input.PartNumber === 2) return new Promise((_resolve, reject) => { failPart = () => reject(new Error("second part failed")); });
|
||||
cancellations.push(options.abortSignal);
|
||||
await new Promise<void>(resolve => options.abortSignal.addEventListener("abort", () => resolve(), { once: true }));
|
||||
await settlement;
|
||||
settled++;
|
||||
throw options.abortSignal.reason;
|
||||
}
|
||||
if (command instanceof CompleteMultipartUploadCommand) { completed = true; return {}; }
|
||||
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
|
||||
expect(settled).toBe(3);
|
||||
aborted = true;
|
||||
return {};
|
||||
});
|
||||
const body = Readable.from((async function* () { for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024); })());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body }).catch(error => error);
|
||||
try {
|
||||
await expect.poll(() => received).toBe(4);
|
||||
failPart();
|
||||
await expect.poll(() => cancellations.every(signal => signal.aborted)).toBe(true);
|
||||
expect(aborted).toBe(false);
|
||||
releaseSiblings();
|
||||
expect((await saving).message).toBe("second part failed");
|
||||
expect(aborted).toBe(true);
|
||||
expect(completed).toBe(false);
|
||||
expect(body.destroyed).toBe(true);
|
||||
} finally { failPart?.(); releaseSiblings(); await saving; send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
|
||||
it.each(["truncated", "excess", "source error"])("rejects %s input without publishing an object", async kind => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
let aborted = false;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
for await (const _chunk of command.input.Body as Readable) {}
|
||||
return { ETag: `part-${command.input.PartNumber}` };
|
||||
}
|
||||
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
|
||||
aborted = true;
|
||||
return {};
|
||||
});
|
||||
const expected = { truncated: "S3 upload source was truncated", excess: "S3 upload source exceeds its declared size", "source error": "source unavailable" };
|
||||
const body = Readable.from((async function* () {
|
||||
for (let index = 0; index < (kind === "excess" ? 274 : 270); index++) yield Buffer.alloc(64 * 1024);
|
||||
if (kind === "source error") throw new Error("source unavailable");
|
||||
})());
|
||||
try {
|
||||
await expect(putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 17 * 1024 * 1024, body })).rejects.toThrow(expected[kind]);
|
||||
expect(aborted).toBe(true);
|
||||
expect(body.destroyed).toBe(true);
|
||||
} finally { send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
|
||||
it("aborts an incomplete upload and closes its source on failure", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
@@ -79,4 +273,43 @@ describe("streamed S3 multipart uploads", () => {
|
||||
await expect(storage.putFile({ companyId: "company", namespace: "skill-packages", originalFilename: "next", contentType: "text/plain", body: Buffer.from("next") })).resolves.toMatchObject({ byteSize: 4 });
|
||||
} finally { send.mockRestore(); vi.useRealTimers(); }
|
||||
});
|
||||
|
||||
it("bounds cancellation while an asynchronous source read is stalled", async () => {
|
||||
vi.useFakeTimers();
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const cancellation = new AbortController();
|
||||
let release!: () => void, waiting = false, settled = false;
|
||||
const sourceGate = new Promise<void>(resolve => { release = resolve; });
|
||||
const body = Readable.from((async function* () {
|
||||
yield Buffer.alloc(64 * 1024);
|
||||
waiting = true;
|
||||
await sourceGate;
|
||||
yield Buffer.alloc(64 * 1024);
|
||||
})());
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
(command.input.Body as Readable).resume();
|
||||
return new Promise((_resolve, reject) => options.abortSignal.addEventListener("abort", () => reject(options.abortSignal.reason), { once: true }));
|
||||
}
|
||||
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
|
||||
return {};
|
||||
});
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "text/plain", contentLength: 128 * 1024, body, signal: cancellation.signal })
|
||||
.catch(error => error).finally(() => { settled = true; });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(waiting).toBe(true);
|
||||
cancellation.abort(new Error("cancelled during source read"));
|
||||
await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1);
|
||||
expect(settled).toBe(true);
|
||||
expect((await saving).message).toBe("cancelled during source read");
|
||||
expect(body.destroyed).toBe(true);
|
||||
} finally {
|
||||
release();
|
||||
await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1);
|
||||
await saving;
|
||||
send.mockRestore(); client.destroy(); vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
agents, companies, createDb, toolApplications, toolCatalogEntries,
|
||||
toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { toolAccessService } from "../services/tool-access.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
@@ -76,6 +76,20 @@ describeDatabase("effective tool assignments", () => {
|
||||
expect(full.allowedTools[0].annotations).toEqual(f.read.annotations);
|
||||
});
|
||||
|
||||
it("keeps exclusions local to each profile when effective profiles grant the same tool", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id));
|
||||
await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id,
|
||||
selectorType: "catalog_entry", effect: "exclude", catalogEntryId: f.read.id });
|
||||
const [second] = await db.insert(toolProfiles).values({ companyId: f.company.id, profileKey: "second", name: "Second", defaultAction: "deny" }).returning();
|
||||
await db.insert(toolProfileBindings).values({ companyId: f.company.id, profileId: second.id, targetType: "agent", targetId: f.agent.id });
|
||||
const [include] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: second.id,
|
||||
selectorType: "catalog_entry", effect: "include", catalogEntryId: f.read.id }).returning();
|
||||
expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.read.id, f.write.id]);
|
||||
await db.delete(toolProfileEntries).where(eq(toolProfileEntries.id, include.id));
|
||||
expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.write.id]);
|
||||
});
|
||||
|
||||
it("keeps unresolved exact names and observes changed catalog, profile and binding state", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(toolProfileEntries).values([
|
||||
@@ -149,14 +163,14 @@ describeDatabase("effective tool assignments", () => {
|
||||
const before = await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id);
|
||||
expect(before.allowedTools).toHaveLength(102);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { toolCatalogEntries, toolProfileEntries }, capture);
|
||||
let scoped: Awaited<ReturnType<typeof service.getConnectionAssignmentsForAgent>>;
|
||||
try { scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); }
|
||||
finally { capture.flush(); }
|
||||
expect(Number(events.at(-1)?.completedQueries)).toBeGreaterThan(0);
|
||||
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(16 * 1024);
|
||||
const catalogQueries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const catalogQueries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "tool_catalog_entries" && event.operation === "select");
|
||||
expect(catalogQueries).toHaveLength(1);
|
||||
expect(catalogQueries[0]).toMatchObject({ calls: 1, rows: 2 });
|
||||
@@ -192,13 +206,13 @@ describeDatabase("effective tool assignments", () => {
|
||||
const service = toolAccessService(db), before = await compare(f);
|
||||
const savedRules = await db.select().from(toolProfileEntries);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { toolProfileEntries, toolProfileBindings }, capture);
|
||||
try {
|
||||
const scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]);
|
||||
capture.flush();
|
||||
const rules = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_entries");
|
||||
const bindings = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
|
||||
const rules = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_entries");
|
||||
const bindings = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
|
||||
expect(rules).toHaveLength(1);
|
||||
expect(Number(rules[0].responseValueBytes)).toBeLessThan(4096);
|
||||
expect(rules[0]).toMatchObject({ calls: 1, rows: 2 });
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { profileCatalogMatcher, profileEntryMatchesCatalog, type ProfileCatalogMatch, type ProfileEntryMatch } from "../services/tool-access/profiles/projection.js";
|
||||
|
||||
describe("indexed profile catalog matching", () => {
|
||||
it("preserves all selector kinds, null matches and exact strings in mixed rule sets", () => {
|
||||
const catalog: ProfileCatalogMatch[] = Array.from({ length: 75 }, (_, index) => ({
|
||||
id: `tool-${index}`, applicationId: index % 3 ? `app-${index % 5}` : null,
|
||||
connectionId: `connection-${index % 7}`, toolName: index % 4 ? `tool-${index}` : "quoted' path\\tool_🙂",
|
||||
riskLevel: index % 2 ? "read" : "write",
|
||||
}));
|
||||
const selectors = ["application", "connection", "catalog_entry", "tool_name", "risk_level", "unknown"];
|
||||
const rules: ProfileEntryMatch[] = catalog.flatMap(entry => selectors.map(selectorType => ({
|
||||
selectorType, applicationId: entry.applicationId, connectionId: entry.connectionId,
|
||||
catalogEntryId: entry.id, toolName: entry.toolName, riskLevel: entry.riskLevel,
|
||||
})));
|
||||
for (const selected of [[], ...rules.map(rule => [rule]), rules.slice(0, 25), rules]) {
|
||||
const matches = profileCatalogMatcher(selected);
|
||||
expect(catalog.filter(matches)).toEqual(catalog.filter(entry => selected.some(rule => profileEntryMatchesCatalog(rule, entry))));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { pendingNewToolsForProfile, profileCoversCatalogScope, profileEntryMatchesCatalog, type ProfileCatalogEntry } from "../services/tool-access/profiles/projection.js";
|
||||
import type { toolProfiles, toolProfileEntries } from "@paperclipai/db";
|
||||
|
||||
type Input = Parameters<typeof pendingNewToolsForProfile>[0];
|
||||
const date = new Date("2026-01-01T00:00:00Z");
|
||||
const later = new Date("2026-01-02T00:00:00Z");
|
||||
function profile(overrides: Partial<typeof toolProfiles.$inferSelect> = {}): typeof toolProfiles.$inferSelect {
|
||||
return { id: "profile", companyId: "company", profileKey: "selected", name: "Selected", description: null, status: "active", defaultAction: "deny", newToolsReviewedAt: null, metadata: {}, createdAt: date, updatedAt: date, ...overrides };
|
||||
}
|
||||
function rule(overrides: Partial<typeof toolProfileEntries.$inferSelect>): typeof toolProfileEntries.$inferSelect {
|
||||
return { id: "rule", companyId: "company", profileId: "profile", selectorType: "catalog_entry", effect: "include", applicationId: null, connectionId: null, catalogEntryId: null, toolName: null, riskLevel: null, conditions: null, createdAt: date, updatedAt: date, ...overrides };
|
||||
}
|
||||
function legacy(input: Input) {
|
||||
if (input.profile.status !== "active" || input.profile.defaultAction !== "deny") return [];
|
||||
const catalogById = new Map(input.catalog.map(entry => [entry.id, entry]));
|
||||
return input.catalog.filter(entry => (entry.status === "active" || entry.status === "quarantined")
|
||||
&& entry.firstSeenAt > (input.profile.newToolsReviewedAt ?? input.profile.createdAt)
|
||||
&& input.entries.some(rule => profileCoversCatalogScope({ entry: rule, catalogEntry: entry, catalogById }))
|
||||
&& !input.entries.some(rule => profileEntryMatchesCatalog(rule, entry))).map(entry => ({
|
||||
catalogEntryId: entry.id, applicationId: entry.applicationId,
|
||||
applicationName: entry.applicationId ? input.applicationsById?.get(entry.applicationId)?.name ?? null : null,
|
||||
connectionId: entry.connectionId, connectionName: input.connectionsById?.get(entry.connectionId)?.name ?? null,
|
||||
toolName: entry.toolName, title: entry.title ?? null, description: entry.description ?? null,
|
||||
capability: entry.riskLevel, riskLevel: entry.riskLevel, addedAt: entry.firstSeenAt, firstSeenAt: entry.firstSeenAt,
|
||||
}));
|
||||
}
|
||||
const catalog: ProfileCatalogEntry[] = Array.from({ length: 48 }, (_, i): ProfileCatalogEntry => ({
|
||||
id: `tool-${i}`, applicationId: i % 3 ? `app-${i % 4}` : null,
|
||||
connectionId: `connection-${i % 6}`, toolName: i % 7 ? `name-${i}` : "quoted' path\\tool_🙂",
|
||||
riskLevel: i % 2 ? "read" : "write", status: i % 5 ? "active" : "quarantined",
|
||||
firstSeenAt: i % 11 ? later : date,
|
||||
}));
|
||||
|
||||
describe("pending profile projection", () => {
|
||||
it("matches the previous ordered projection for every selector, effect, watermark and nullable scope", () => {
|
||||
const rules = catalog.flatMap(entry => (["include", "exclude"] as const).flatMap(effect => [
|
||||
rule({ effect, selectorType: "application", applicationId: entry.applicationId }),
|
||||
rule({ effect, selectorType: "connection", connectionId: entry.connectionId }),
|
||||
rule({ effect, selectorType: "catalog_entry", catalogEntryId: entry.id }),
|
||||
rule({ effect, selectorType: "tool_name", toolName: entry.toolName }),
|
||||
rule({ effect, selectorType: "risk_level", riskLevel: entry.riskLevel }),
|
||||
]));
|
||||
const scopes = [rule({ selectorType: "application", applicationId: null }), rule({ selectorType: "connection", connectionId: "connection-1" }), rule({ catalogEntryId: "tool-2" }), rule({ catalogEntryId: "missing" })];
|
||||
for (const scope of scopes) for (const selected of [[], ...rules.map(entry => [entry]), rules.slice(0, 50), rules]) {
|
||||
const input = { profile: profile(), catalog, entries: [scope, ...selected], applicationsById: new Map([["app-1", { id: "app-1", name: "Application" }]]), connectionsById: new Map([["connection-1", { id: "connection-1", name: "Connection" }]]) };
|
||||
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
|
||||
}
|
||||
for (const changes of [{ status: "archived" as const }, { defaultAction: "allow" as const }, { newToolsReviewedAt: later }]) {
|
||||
const input = { profile: profile(changes), catalog, entries: scopes };
|
||||
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
|
||||
}
|
||||
});
|
||||
|
||||
it("does not broaden an individual tool's null application scope to unrelated null applications", () => {
|
||||
const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-0" })], catalog };
|
||||
const ids = pendingNewToolsForProfile(input).map(entry => entry.catalogEntryId);
|
||||
expect(ids).toContain("tool-6");
|
||||
expect(ids).not.toContain("tool-3");
|
||||
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
|
||||
});
|
||||
|
||||
it("recomputes catalog references, exclusions and scope after the input changes", () => {
|
||||
const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-2" })], catalog };
|
||||
expect(pendingNewToolsForProfile(input).length).toBeGreaterThan(0);
|
||||
expect(pendingNewToolsForProfile({ ...input, catalog: catalog.filter(entry => entry.id !== "tool-2") })).toEqual([]);
|
||||
expect(pendingNewToolsForProfile({ ...input, entries: [rule({ catalogEntryId: "tool-2", effect: "exclude" })] })).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { createDb, toolCatalogEntries, toolProfileEntries, toolProfiles } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createProfileProjectionFixture } from "./helpers/tool-profile-projection-fixture.js";
|
||||
import { createToolProfileService } from "../services/tool-access/profiles/service.js";
|
||||
import { createToolAppAttention } from "../services/tool-access/catalog/attention.js";
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const describeDatabase = support.supported ? describe : describe.skip;
|
||||
describeDatabase("pending profile tools through profile and attention services", () => {
|
||||
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("profile-review-service-"); db = createDb(database.connectionString); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
it("preserves fresh pending counts in both owning services after review and catalog changes", async () => {
|
||||
const f = await createProfileProjectionFixture(db, { profiles: 2, tools: 8, rules: 4 });
|
||||
const now = () => new Date("2026-02-01T00:00:00Z"), service = createToolProfileService(db, now), attention = createToolAppAttention(db, now);
|
||||
const assertCounts = async (expected: number) => {
|
||||
const profiles = await service.listProfiles(f.company.id);
|
||||
expect(profiles.find(p => p.id === f.profiles[0].id)?.newToolsPendingCount).toBe(expected);
|
||||
expect(profiles.find(p => p.id === f.profiles[1].id)?.newToolsPendingCount).toBe(0);
|
||||
expect((await service.listProfileNewTools(f.profiles[0].id, f.company.id)).pendingCount).toBe(expected);
|
||||
expect((await attention(f.company.id)).totals.newToolsPendingReview).toBe(expected);
|
||||
};
|
||||
await assertCounts(4);
|
||||
const [exclude] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profiles[0].id,
|
||||
selectorType: "tool_name", effect: "exclude", toolName: f.catalog[4].toolName }).returning();
|
||||
await assertCounts(3);
|
||||
await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.catalog[5].id));
|
||||
await assertCounts(2);
|
||||
await db.update(toolProfiles).set({ newToolsReviewedAt: now() }).where(eq(toolProfiles.id, f.profiles[0].id));
|
||||
await assertCounts(0);
|
||||
const foreign = await createProfileProjectionFixture(db, { profiles: 2, tools: 5, rules: 2 });
|
||||
expect((await service.listProfiles(foreign.company.id)).map(p => p.companyId)).toEqual([foreign.company.id, foreign.company.id]);
|
||||
expect((await db.select().from(toolProfileEntries).where(eq(toolProfileEntries.id, exclude.id)))[0].effect).toBe("exclude");
|
||||
});
|
||||
});
|
||||
@@ -122,16 +122,21 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
await quiet(); expect(fetchReport).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(deadline - Date.now());
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
// Keep time fixed during database I/O. Advancing it on every poll can
|
||||
// make the first reports due again before the backlog has settled.
|
||||
const next = coordinator!.nextWakeAt();
|
||||
if (next !== null && next < deadline + 60_000) {
|
||||
await vi.advanceTimersByTimeAsync(Math.max(0, next - Date.now()));
|
||||
}
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
expect(fetchReport).toHaveBeenCalledTimes(105);
|
||||
}, { timeout: 5000 });
|
||||
}, { interval: 0, timeout: 15_000 });
|
||||
expect(fetchReport.mock.calls.map(([call]) => call.id).sort()).toEqual(ids.sort());
|
||||
expect((await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, companyId)))
|
||||
.every(row => row.attempts === 1)).toBe(true);
|
||||
await quiet();
|
||||
} finally { reads.mockRestore(); }
|
||||
});
|
||||
}, 30_000);
|
||||
it("leaves configured idle, live calls, old history and unidentified calls quiet", async () => {
|
||||
await session({ state: "active", endedAt: null });
|
||||
await session({ endedAt: new Date(Date.now() - day - 1000) });
|
||||
|
||||
@@ -145,6 +145,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
expect(await row(call.id)).toBeUndefined(); expect(coordinator!.nextWakeAt()).toBeNull(); await quiet();
|
||||
});
|
||||
it("rounds PostgreSQL submillisecond deadlines up and never deletes early", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false });
|
||||
const call = await admission(); const dueAt = Math.trunc(Date.now()) + 60_000;
|
||||
await db.execute(sql`UPDATE chat_voice_inbound_calls SET updated_at = to_timestamp(${dueAt - week}::double precision / 1000) + interval '0.0005 seconds' WHERE id = ${call.id}`);
|
||||
const service = voiceInboundRetention(db), [entry] = await service.load({ kind: "admission", id: call.id });
|
||||
|
||||
@@ -12,6 +12,7 @@ import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js";
|
||||
import { createLifecycleDriver } from "./services/agent-lifecycle-driver.js";
|
||||
import { startAgentLifecycle } from "./services/agent-lifecycle.js";
|
||||
import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js";
|
||||
import { databaseOperationMiddleware } from "./middleware/database-operation.js";
|
||||
import { staticUiFiles } from "./middleware/static-ui.js";
|
||||
import { retainedUiAssets } from "./retained-ui-assets.js";
|
||||
import { createStorageProviderFromConfig } from "./storage/provider-registry.js";
|
||||
@@ -470,6 +471,7 @@ export async function createApp(
|
||||
},
|
||||
) {
|
||||
const app = express();
|
||||
app.use(databaseOperationMiddleware);
|
||||
app.use(idleAdmissionMiddleware);
|
||||
const staticUi = express.Router();
|
||||
app.locals.paperclipDb = db;
|
||||
|
||||
+7
-1
@@ -81,6 +81,7 @@ import { deliverExecutionStatuses } from "./services/execution-status-delivery.j
|
||||
import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./services/execution-recovery-resolution.js";
|
||||
import { reconcileSafeNativeReplacements } from "./services/native-runtime/native-safe-replacement.js";
|
||||
import { reconcileAbandonedExecutionControl } from "./services/execution-control-reconciliation.js";
|
||||
import { acknowledgeNativeStartupCleanup } from "./services/native-runtime/native-startup-cleanup.js";
|
||||
import { executionControlSchedule, registerExecutionControlWork } from "./services/execution-control-work.js";
|
||||
import { connectionIntentDeliveryService } from "./services/connection-intent-delivery.js";
|
||||
import { existsSync, readFileSync, rmSync } from "node:fs";
|
||||
@@ -1376,7 +1377,12 @@ async function startServerWithDatabaseTeardown(
|
||||
const unsubscribeLocalLoginCleanup = localLoginCleanup.subscribeCleanupReady(localLoginCleanupWork.wake);
|
||||
server.on("close", unsubscribeLocalLoginCleanup);
|
||||
registerExecutionControlWork(deliveryWork, [
|
||||
{ queue: DELIVERY_QUEUES.executionFinalization, run: () => reconcileAbandonedExecutionControl(db), next: executionSchedule.finalization },
|
||||
{ queue: DELIVERY_QUEUES.executionFinalization, run: () => reconcileAbandonedExecutionControl(db, new Date(), {
|
||||
startupCleanup: async scope => {
|
||||
await accountingHeartbeat.cancelRun(scope.runId, "Native startup timed out; cancelling and verifying cleanup");
|
||||
await acknowledgeNativeStartupCleanup(db, scope);
|
||||
},
|
||||
}), next: executionSchedule.finalization },
|
||||
{ queue: DELIVERY_QUEUES.executionStatus, run: () => deliverExecutionStatuses(db), next: executionSchedule.status },
|
||||
{ queue: DELIVERY_QUEUES.executionDisposition, run: () => settleUnrecoverableExecutions(db), next: executionSchedule.disposition },
|
||||
...(heartbeat ? [
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
import { withDatabaseOperation } from "@paperclipai/db";
|
||||
import type { RequestHandler } from "express";
|
||||
|
||||
export const databaseOperationMiddleware: RequestHandler = (_req, _res, next) =>
|
||||
withDatabaseOperation("http.request", next);
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
@@ -40,7 +40,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => {
|
||||
const scope = { companyId, runId: before.id };
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const native = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, scope));
|
||||
@@ -51,7 +51,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => {
|
||||
expect(native.run).not.toHaveProperty("stdoutExcerpt");
|
||||
expect(native.run).not.toHaveProperty("stderrExcerpt");
|
||||
} finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(1);
|
||||
expect(summary.responseValueBytes).toBeLessThan(8192);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, before.id)))[0]).toEqual(before);
|
||||
@@ -97,12 +97,12 @@ describe.skipIf(!support.supported)("wake release run reads", () => {
|
||||
contextSnapshot: { ...context, ...launch }, resultJson: evidence }).returning();
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let selected: Awaited<ReturnType<typeof readReleaseRun>>;
|
||||
try { selected = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, { companyId, runId: before.id })); }
|
||||
finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(1);
|
||||
expect(summary.responseValueBytes).toBeLessThan(8192);
|
||||
expect(selected!.run?.contextSnapshot).toEqual(context);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { redactCommandText } from "@paperclipai/adapter-utils";
|
||||
import { isPublicExecutorToolSelector, looksLikeCredentialJwt } from "@paperclipai/adapter-utils/command-redaction";
|
||||
|
||||
const SECRET_FIELD_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|access[-_]?token|auth(?:_?token)?|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring|browser[-_]?code|login[-_]?url)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`;
|
||||
const SECRET_FIELD_NAME_START = String.raw`(?<![A-Za-z0-9_-])`;
|
||||
|
||||
const SECRET_PAYLOAD_KEY_RE = new RegExp(`^(?:${SECRET_FIELD_NAME_PATTERN})(?:_plain|_ref)?$`, "i");
|
||||
// Authorization reasons are policy decision codes, not credentials. They must
|
||||
@@ -369,13 +370,13 @@ const CLI_SECRET_FLAG_RE = new RegExp(
|
||||
"i",
|
||||
);
|
||||
const JSON_SECRET_FIELD_TEXT_RE = new RegExp(
|
||||
String.raw`((?:"|')?${SECRET_FIELD_NAME_PATTERN}(?:"|')?\s*:\s*(?:"|'))[^"'` +
|
||||
String.raw`((?:"|')?${SECRET_FIELD_NAME_START}${SECRET_FIELD_NAME_PATTERN}(?:"|')?\s*:\s*(?:"|'))[^"'` +
|
||||
"`" +
|
||||
String.raw`\r\n]+((?:"|'))`,
|
||||
"gi",
|
||||
);
|
||||
const ESCAPED_JSON_SECRET_FIELD_TEXT_RE = new RegExp(
|
||||
String.raw`((?:\\")?${SECRET_FIELD_NAME_PATTERN}(?:\\")?\s*:\s*(?:\\"))[^\\\r\n]+((?:\\"))`,
|
||||
String.raw`((?:\\")?${SECRET_FIELD_NAME_START}${SECRET_FIELD_NAME_PATTERN}(?:\\")?\s*:\s*(?:\\"))[^\\\r\n]+((?:\\"))`,
|
||||
"gi",
|
||||
);
|
||||
const SECRET_TEXT_HINTS = [
|
||||
@@ -666,7 +667,7 @@ function authorizationCredentialRange(
|
||||
function redactAuthorizationCredentials(input: string): string {
|
||||
// Include compound diagnostic labels such as proxyAuthorization while the
|
||||
// value parser still requires a Basic/Bearer scheme before redacting.
|
||||
const authorizationWord = /\b[A-Za-z0-9_-]*Authorization[A-Za-z0-9_-]*\b/gi;
|
||||
const authorizationWord = /(?<![A-Za-z0-9_-])[A-Za-z0-9_-]*Authorization[A-Za-z0-9_-]*\b/gi;
|
||||
const parts: string[] = [];
|
||||
let copiedThrough = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
@@ -251,6 +251,22 @@ export function aiConnectionRoutes(db: Db, options: Parameters<typeof supportsLo
|
||||
res.json(await service.probeUsage(companyId, responsibleUserForAiRequest(req)!, connectionId.data, grantId.data));
|
||||
},
|
||||
);
|
||||
router.get(
|
||||
"/companies/:companyId/ai-connections/:connectionId/identity", managementGuard("connections:read", "GET /companies/:companyId/ai-connections/:connectionId/identity", { personal: true }),
|
||||
async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertBoardOrAgent(req);
|
||||
assertCompanyAccess(req, companyId);
|
||||
const connectionId = z.string().uuid().safeParse(req.params.connectionId);
|
||||
const grantId = z.string().uuid().optional().safeParse(req.query.grantId);
|
||||
if (!connectionId.success || !grantId.success) throw unprocessable("Invalid connection or grant ID");
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
const actor = getActorInfo(req);
|
||||
res.json(await service.accountIdentity(companyId, responsibleUserForAiRequest(req)!, connectionId.data, grantId.data, {
|
||||
actorType: actor.actorType, actorId: actor.actorId, heartbeatRunId: actor.runId,
|
||||
}));
|
||||
},
|
||||
);
|
||||
router.get(
|
||||
"/companies/:companyId/ai-connections/:connectionId/active-runs", managementGuard("connections:read", "GET /companies/:companyId/ai-connections/:connectionId/active-runs", { personal: true }),
|
||||
async (req, res) => {
|
||||
|
||||
@@ -8379,6 +8379,7 @@ export function issueRoutes(
|
||||
title: row.title,
|
||||
identifier: row.identifier,
|
||||
status: row.status as IssueSummary["status"],
|
||||
displayStatus: row.displayStatus,
|
||||
updatedAt: row.updatedAt,
|
||||
externalConversationState: row.externalConversationState,
|
||||
}));
|
||||
@@ -8548,10 +8549,10 @@ export function issueRoutes(
|
||||
res.status(400).json({ error: "offset must be a non-negative integer" });
|
||||
return;
|
||||
}
|
||||
if (sortField !== undefined && sortField !== "updated" && sortField !== "id") {
|
||||
if (sortField !== undefined && sortField !== "updated" && sortField !== "created" && sortField !== "id") {
|
||||
res
|
||||
.status(400)
|
||||
.json({ error: "sortField must be 'updated' or 'id' when provided" });
|
||||
.json({ error: "sortField must be 'updated', 'created', or 'id' when provided" });
|
||||
return;
|
||||
}
|
||||
if (sortDir !== undefined && sortDir !== "asc" && sortDir !== "desc") {
|
||||
@@ -8619,6 +8620,7 @@ export function issueRoutes(
|
||||
readCondition: await issueReadSqlCondition(db, req.actor),
|
||||
attention: attention === "blocked" ? "blocked" : undefined,
|
||||
status: req.query.status as string | string[] | undefined,
|
||||
displayStatus: req.query.displayStatus as string | string[] | undefined,
|
||||
assigneeAgentId,
|
||||
participantAgentId: uuidFilters.participantAgentId,
|
||||
assigneeUserId,
|
||||
@@ -8657,7 +8659,7 @@ export function issueRoutes(
|
||||
q: req.query.q as string | undefined,
|
||||
limit,
|
||||
offset,
|
||||
sortField: sortField === "updated" || sortField === "id" ? sortField : undefined,
|
||||
sortField: sortField === "updated" || sortField === "created" || sortField === "id" ? sortField : undefined,
|
||||
afterId: req.query.afterId as string | undefined,
|
||||
sortDir: sortDir === "asc" || sortDir === "desc" ? sortDir : undefined,
|
||||
updatedSince: rawUpdatedSince,
|
||||
@@ -8869,6 +8871,7 @@ export function issueRoutes(
|
||||
readCondition: await issueReadSqlCondition(db, req.actor),
|
||||
attention: "blocked",
|
||||
status: req.query.status as string | string[] | undefined,
|
||||
displayStatus: req.query.displayStatus as string | string[] | undefined,
|
||||
assigneeAgentId: req.query.assigneeAgentId as string | undefined,
|
||||
participantAgentId: countUuidFilters.participantAgentId,
|
||||
assigneeUserId: req.query.assigneeUserId as string | undefined,
|
||||
|
||||
@@ -11311,6 +11311,17 @@ registerCurrentRoute({
|
||||
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
|
||||
});
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:read",
|
||||
method: "get",
|
||||
path: "/api/companies/{companyId}/ai-connections/{connectionId}/identity",
|
||||
tags: ["ai-connections"],
|
||||
summary: "Read the selected authorized AI account's provider email and organization",
|
||||
query: z.object({ grantId: z.string().uuid().optional() }),
|
||||
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 422: r.unprocessable },
|
||||
});
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:read",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { issueDisplayStatusSql } from "../services/issue-display-status.js";
|
||||
import { Router } from "express";
|
||||
import { and, desc, eq, gte, inArray, isNull, or, sql, type SQLWrapper } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -347,6 +348,7 @@ export function userProfileRoutes(db: Db) {
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
|
||||
@@ -172,6 +172,11 @@ const GIT_INFO_CACHE_TTL_MS = 3000;
|
||||
const processStartedAt = new Date().toISOString();
|
||||
let gitInfoCache: { value: ServerGitInfo; expiresAt: number } | null = null;
|
||||
|
||||
// Lifecycle identity must not run the synchronous Git probes used by diagnostics.
|
||||
export function getServerProcessStartedAt(): string {
|
||||
return processStartedAt;
|
||||
}
|
||||
|
||||
export function getServerInfoSnapshot(
|
||||
opts: {
|
||||
now?: number;
|
||||
|
||||
@@ -2,7 +2,7 @@ import { notifyInstructionCollection } from "./instruction-collection-notificati
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
|
||||
import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm";
|
||||
import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, withDedicatedDbConnection, type Db } from "@paperclipai/db";
|
||||
import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh";
|
||||
import { prepareAdapterExecutionTargetRuntime, runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget, type PreparedAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target";
|
||||
@@ -16,7 +16,7 @@ import type { AuthorizationActor } from "./authorization.js";
|
||||
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
|
||||
import type { Environment, EnvironmentLease } from "@paperclipai/shared";
|
||||
import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory } from "./agent-directory-probe.js";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "./remote-execution-termination.js";
|
||||
import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js";
|
||||
import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
@@ -46,7 +46,7 @@ function baseline(row: Copy) {
|
||||
function actor(row: Copy): AuthorizationActor {
|
||||
return { type: "agent", companyId: row.companyId, agentId: row.agentId, runId: row.runId, onBehalfOfUserId: row.responsibleUserId };
|
||||
}
|
||||
export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string, runId: string) => Promise<Copy | null>, patch: (row: Copy, values: Partial<typeof copies.$inferInsert>) => Promise<Copy>, environmentRuntime?: EnvironmentRuntimeService) {
|
||||
export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string, runId: string) => Promise<Copy | null>, patch: (row: Copy, values: Partial<typeof copies.$inferInsert>, condition?: SQL) => Promise<Copy>, environmentRuntime?: EnvironmentRuntimeService) {
|
||||
const store = agentFileStore(db);
|
||||
async function insertCopy(values: typeof copies.$inferInsert) {
|
||||
return db.transaction(async tx => {
|
||||
@@ -503,7 +503,15 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
return row.state === "superseded" || !row.processStoppedAt ||
|
||||
completed.has(row.state) && row.receipt?.cleanupPending === false;
|
||||
}
|
||||
async function settleCompletedCleanupRetry(row: Copy) {
|
||||
if (completed.has(row.state) && row.processStoppedAt && !row.receipt?.retainedByRunId && row.receipt?.cleanupPending === false && row.nextAttemptAt) {
|
||||
return patch(row, { nextAttemptAt: null }, sql`${copies.receipt} = ${JSON.stringify(row.receipt)}::jsonb
|
||||
AND ${copies.processStoppedAt} IS NOT NULL`);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
async function release(row: Copy, target?: AdapterExecutionTarget | null) {
|
||||
row = await settleCompletedCleanupRetry(row);
|
||||
if (releaseIsNoop(row) || !await owns(row)) return;
|
||||
const destroyedOnly = row.receipt?.cleanupDestroyedOnly === true;
|
||||
if (row.receipt?.retainedByRunId || row.state === "superseded" || !await owns(row) || !row.processStoppedAt) return;
|
||||
@@ -559,7 +567,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
: eq(environmentLeases.environmentId, row.location.slice("remote:".length)),
|
||||
));
|
||||
if (leases.length !== 1) return false;
|
||||
const lease = leases[0]!;
|
||||
const lease = await resolveRemoteTerminationLease(db, leases[0]!);
|
||||
if (lease.environmentId !== null && row.location !== `remote:${lease.environmentId}`) return false;
|
||||
const remoteCwd = cleanup?.remoteCwd ?? lease.metadata?.remoteCwd;
|
||||
return typeof remoteCwd === "string" &&
|
||||
@@ -588,7 +596,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
cleanup?.leaseId ? eq(environmentLeases.id, cleanup.leaseId) : eq(environmentLeases.environmentId, row.location.slice("remote:".length)),
|
||||
));
|
||||
if (leases.length !== 1) return null;
|
||||
const lease = leases[0]!;
|
||||
const lease = await resolveRemoteTerminationLease(db, leases[0]!);
|
||||
const destroyedWithoutEnvironment = lease.environmentId === null && hasRemoteTerminationReceipt(lease)
|
||||
&& (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed";
|
||||
if (row.location !== `remote:${lease.environmentId}` && !destroyedWithoutEnvironment) return null;
|
||||
@@ -658,10 +666,12 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
checkpointWarm: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => current.receipt?.warm === true ? checkpoint(current, target) : Promise.resolve(current), "agent_directory_checkpoint"),
|
||||
collectStopped: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => collectStopped(current, target), "agent_directory_collect"),
|
||||
release: async (row: Copy) => {
|
||||
const current = await get(row.companyId, row.runId);
|
||||
let current = await get(row.companyId, row.runId);
|
||||
if (!current) return;
|
||||
current = await settleCompletedCleanupRetry(current);
|
||||
// This check changes no receipt or bytes. A concurrent stop merely leaves
|
||||
// cleanup to its owner or the recovery sweep; it grants no new authority.
|
||||
if (!current || releaseIsNoop(current)) return;
|
||||
if (releaseIsNoop(current)) return;
|
||||
await serial(current, release, "agent_directory_release");
|
||||
},
|
||||
};
|
||||
|
||||
@@ -129,8 +129,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
|
||||
};
|
||||
return db.transaction(tx => write(tx as unknown as Db));
|
||||
}
|
||||
async function patch(row: Copy, values: Partial<typeof copies.$inferInsert>) {
|
||||
const updated = await tryPatch(row, values);
|
||||
async function patch(row: Copy, values: Partial<typeof copies.$inferInsert>, condition?: SQL) {
|
||||
const updated = await tryPatch(row, values, false, condition);
|
||||
// A late cleanup must not overwrite an explicit resolution or a newer
|
||||
// baseline. Canonical content has its own independent head CAS.
|
||||
return updated ?? (await get(row.companyId, row.runId))!;
|
||||
@@ -499,8 +499,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
|
||||
function directoryCleanupEntry(row: Copy | null): DirectoryCleanupEntry | null {
|
||||
return row && ["saved", "unchanged", "resolved", "unavailable"].includes(row.state)
|
||||
&& row.processStoppedAt && isAgentDirectoryCopy(row) && !row.receipt?.retainedByRunId
|
||||
&& row.receipt?.cleanupPending !== false
|
||||
&& (Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true)
|
||||
&& (row.receipt?.cleanupPending === false ? row.nextAttemptAt !== null
|
||||
: Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true)
|
||||
? { id: row.runId, companyId: row.companyId, dueAt: row.nextAttemptAt?.getTime() ?? 0 } : null;
|
||||
}
|
||||
function directoryCleanupPredicate() {
|
||||
@@ -508,8 +508,9 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
|
||||
AND ${copies.processStoppedAt} IS NOT NULL
|
||||
AND ${copies.receipt}->>'schema' = 'paperclip.agent-files.v1'
|
||||
AND coalesce(${copies.receipt}->>'retainedByRunId', '') = ''
|
||||
AND ${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false'
|
||||
AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')`;
|
||||
AND ((${copies.receipt}->>'cleanupPending' = 'false' AND ${copies.nextAttemptAt} IS NOT NULL)
|
||||
OR (${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false'
|
||||
AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')))`;
|
||||
}
|
||||
async function loadDirectoryCleanup(scope: DirectoryCleanupScope | undefined, admitted: () => boolean) {
|
||||
const entries: DirectoryCleanupEntry[] = [];
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { createServer } from "node:http";
|
||||
import { once } from "node:events";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { probeAiAccountIdentity } from "./ai-account-identity.js";
|
||||
|
||||
const metadata = { provider: "anthropic", method: "subscription" } as const;
|
||||
function fixture(body: unknown, status = 200) {
|
||||
return vi.fn<typeof fetch>().mockResolvedValue(Response.json(body, { status }));
|
||||
}
|
||||
|
||||
describe("AI account identity", () => {
|
||||
it("returns only safe account details from the selected bearer profile", async () => {
|
||||
const request = fixture({ account: { uuid: "private-seat", email: "avery@example.test", secret: "provider-private" },
|
||||
organization: { uuid: "private-organization", name: "Example team" }, access_token: "private-token" });
|
||||
const result = await probeAiAccountIdentity(metadata, "selected-credential", request);
|
||||
expect(result).toMatchObject({ status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" });
|
||||
expect(request).toHaveBeenCalledWith("https://api.anthropic.com/api/oauth/profile", expect.objectContaining({
|
||||
redirect: "error", signal: expect.any(AbortSignal),
|
||||
headers: { Authorization: "Bearer selected-credential", "anthropic-beta": "oauth-2025-04-20" },
|
||||
}));
|
||||
for (const secret of ["private-seat", "private-organization", "provider-private", "private-token", "selected-credential"]) {
|
||||
expect(JSON.stringify(result)).not.toContain(secret);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([401, 403, 429, 500])("reports HTTP %s without exposing the provider response", async status => {
|
||||
const result = await probeAiAccountIdentity(metadata, "selected-credential", fixture({ error: "private-provider-error" }, status));
|
||||
expect(result).toMatchObject({ accountLabel: null, organizationLabel: null });
|
||||
expect(result.status).toBe(status === 401 || status === 403 ? "unavailable" : "error");
|
||||
if (status === 403) expect(result).toMatchObject({ errorCode: "permission_denied", message: expect.stringContaining("did not permit") });
|
||||
expect(JSON.stringify(result)).not.toContain("private-provider-error");
|
||||
});
|
||||
|
||||
it.each([{}, { account: { email: "not-an-email" } }, { account: { email: "avery@example.test\nprivate" } }])("does not infer an email from an incomplete profile %j", async profile => {
|
||||
expect(await probeAiAccountIdentity(metadata, "selected-credential", fixture(profile))).toMatchObject({
|
||||
status: "unavailable", errorCode: "invalid_response", accountLabel: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not call a provider for API keys or other sign-in methods", async () => {
|
||||
const request = fixture({});
|
||||
expect((await probeAiAccountIdentity({ provider: "anthropic", method: "api_key" }, "", request)).errorCode).toBe("unsupported");
|
||||
expect((await probeAiAccountIdentity({ provider: "openai", method: "subscription" }, "", request)).errorCode).toBe("unsupported");
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("aborts a stalled real response body while unrelated requests remain responsive", async () => {
|
||||
const server = createServer((req, res) => {
|
||||
if (req.url === "/profile") { res.writeHead(200, { "Content-Type": "application/json" }); res.flushHeaders(); }
|
||||
else res.end("responsive");
|
||||
});
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Missing fixture address");
|
||||
const origin = `http://127.0.0.1:${address.port}`;
|
||||
const timeout = AbortSignal.timeout.bind(AbortSignal);
|
||||
const deadline = vi.spyOn(AbortSignal, "timeout").mockImplementation(() => timeout(100));
|
||||
try {
|
||||
const pending = probeAiAccountIdentity(metadata, "selected-credential", (_url, init) => fetch(`${origin}/profile`, init));
|
||||
expect(await (await fetch(`${origin}/unrelated`)).text()).toBe("responsive");
|
||||
const result = await pending;
|
||||
expect(result.status).toBe("error");
|
||||
expect(result.accountLabel).toBeNull();
|
||||
expect(deadline).toHaveBeenCalledWith(15_000);
|
||||
} finally {
|
||||
deadline.mockRestore();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>(resolve => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import type { AiConnectionAccountIdentity, AiConnectionMetadata } from "@paperclipai/shared";
|
||||
import { readUsage } from "./ai-connection-usage.js";
|
||||
|
||||
type Observation = Omit<AiConnectionAccountIdentity, "connectionId" | "grantId">;
|
||||
type FailureCode = NonNullable<Observation["errorCode"]>;
|
||||
const messages: Record<FailureCode, string> = {
|
||||
unsupported: "Account details are not available for this sign-in method.",
|
||||
connection_unavailable: "Reconnect or enable this connection to check its account details.",
|
||||
authentication_required: "Reconnect this account to check its account details.",
|
||||
permission_denied: "Anthropic did not permit this connection to read the account email.",
|
||||
rate_limited: "Anthropic limited this account check. Try again later.",
|
||||
provider_unavailable: "Could not read the account details. Try again later.",
|
||||
invalid_response: "Anthropic did not return an account email.",
|
||||
};
|
||||
const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
const label = (value: unknown): string | null => typeof value === "string" && value.trim().length > 0
|
||||
&& value.length <= 254 && !/[\u0000-\u001f\u007f]/.test(value) ? value.trim() : null;
|
||||
|
||||
/** Never uses ambient CLI identity or exposes provider IDs, tokens, or raw responses. */
|
||||
export async function probeAiAccountIdentity(
|
||||
metadata: AiConnectionMetadata,
|
||||
credential: string,
|
||||
request: typeof fetch = fetch,
|
||||
): Promise<Observation> {
|
||||
const base = { checkedAt: new Date().toISOString(), accountLabel: null, organizationLabel: null };
|
||||
if (metadata.provider !== "anthropic" || metadata.method !== "subscription" || metadata.routing) {
|
||||
return { ...base, status: "unavailable", errorCode: "unsupported", message: messages.unsupported };
|
||||
}
|
||||
try {
|
||||
const profile = await readUsage("https://api.anthropic.com/api/oauth/profile", {
|
||||
Authorization: `Bearer ${credential}`, "anthropic-beta": "oauth-2025-04-20",
|
||||
}, request);
|
||||
const accountLabel = label(object(profile.account).email);
|
||||
if (!accountLabel || !/^[^\s@/\\]+@[^\s@/\\]+\.[^\s@/\\]+$/.test(accountLabel)) {
|
||||
return { ...base, status: "unavailable", errorCode: "invalid_response", message: messages.invalid_response };
|
||||
}
|
||||
return { ...base, status: "available", accountLabel, organizationLabel: label(object(profile.organization).name) };
|
||||
} catch (error) {
|
||||
const code = object(error).code;
|
||||
const errorCode: FailureCode = typeof code === "string" && Object.hasOwn(messages, code) ? code as FailureCode : "provider_unavailable";
|
||||
return { ...base, status: errorCode === "authentication_required" || errorCode === "permission_denied" ? "unavailable" : "error", errorCode, message: messages[errorCode] };
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
type AiConnectionAttribution,
|
||||
type AiConnectionMetadata,
|
||||
type AiManagedConnectionSummary,
|
||||
type AiConnectionAccountIdentity,
|
||||
type AiConnectionUsage,
|
||||
type CreateAiConnection,
|
||||
type AiConnectionLoginIntent,
|
||||
@@ -49,6 +50,7 @@ import { logger } from "../middleware/logger.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import { secretService } from "./secrets.js";
|
||||
import { probeAiConnectionUsage } from "./ai-connection-usage.js";
|
||||
import { probeAiAccountIdentity } from "./ai-account-identity.js";
|
||||
|
||||
/** Same human audience displayed by the existing Connections identity controls. */
|
||||
function canUseCredential(
|
||||
@@ -225,28 +227,46 @@ export function aiConnectionService(db: Db) {
|
||||
});
|
||||
});
|
||||
}
|
||||
/** On-demand account observation for either runner; never probes ambient CLI auth. */
|
||||
async function probeUsage(companyId: string, userId: string, connectionId: string, grantId?: string): Promise<AiConnectionUsage> {
|
||||
async function readableAccount(companyId: string, userId: string, connectionId: string, grantId?: string) {
|
||||
if (!(await membership(companyId, userId))) throw forbidden("An active company member is required");
|
||||
const candidates = (await rows(companyId)).filter((row) => row.connection.id === connectionId && (!grantId || row.grant.id === grantId));
|
||||
const audience = await db.select().from(connectionGrantMembers).where(eq(connectionGrantMembers.companyId, companyId));
|
||||
const visible = candidates.filter(({ grant }) => canUseCredential(grant, userId, audience.filter((member) => member.grantId === grant.id)));
|
||||
if (!visible.length) throw notFound("AI connection not found");
|
||||
if (visible.length > 1) throw unprocessable("Choose a credential grant for this usage check");
|
||||
if (visible.length > 1) throw unprocessable("Choose a credential grant for this account check");
|
||||
const row = visible[0]!;
|
||||
const metadata = aiConnectionMetadataSchema.safeParse(row.connection.config.ai);
|
||||
if (!metadata.success) throw notFound("AI connection not found");
|
||||
const identity = { connectionId: row.connection.id, grantId: row.grant.id, ...metadata.data };
|
||||
return { row, metadata: metadata.data };
|
||||
}
|
||||
/** On-demand account observation for either runner; never probes ambient CLI auth. */
|
||||
async function probeUsage(companyId: string, userId: string, connectionId: string, grantId?: string): Promise<AiConnectionUsage> {
|
||||
const { row, metadata } = await readableAccount(companyId, userId, connectionId, grantId);
|
||||
const identity = { connectionId: row.connection.id, grantId: row.grant.id, ...metadata };
|
||||
const empty = { ...identity, checkedAt: new Date().toISOString(), source: null, planType: null, limits: [], overage: null };
|
||||
if (row.grant.status !== "active" || !row.connection.enabled || row.connection.status !== "active" ||
|
||||
row.connection.healthStatus !== "ok" || aiSubscriptionNeedsIsolatedLogin(row.connection.config)) {
|
||||
return { ...empty, status: "unavailable", errorCode: "connection_unavailable", message: "Reconnect or enable this AI connection before checking usage." };
|
||||
}
|
||||
if (!supportsAiConnectionUsage(metadata.data.provider, metadata.data.method)) {
|
||||
return { ...identity, ...await probeAiConnectionUsage(metadata.data, "") };
|
||||
if (!supportsAiConnectionUsage(metadata.provider, metadata.method)) {
|
||||
return { ...identity, ...await probeAiConnectionUsage(metadata, "") };
|
||||
}
|
||||
const value = await credential(row);
|
||||
return { ...identity, ...await probeAiConnectionUsage(metadata.data, value) };
|
||||
return { ...identity, ...await probeAiConnectionUsage(metadata, value) };
|
||||
}
|
||||
async function accountIdentity(companyId: string, userId: string, connectionId: string, grantId?: string,
|
||||
audit?: { actorType: "user" | "agent" | "system"; actorId: string; heartbeatRunId?: string | null }): Promise<AiConnectionAccountIdentity> {
|
||||
const { row, metadata } = await readableAccount(companyId, userId, connectionId, grantId);
|
||||
const identity = { connectionId: row.connection.id, grantId: row.grant.id };
|
||||
if (row.grant.status !== "active" || !row.connection.enabled || row.connection.status !== "active" ||
|
||||
row.connection.healthStatus !== "ok" || aiSubscriptionNeedsIsolatedLogin(row.connection.config)) {
|
||||
return { ...identity, status: "unavailable", checkedAt: new Date().toISOString(), accountLabel: null, organizationLabel: null,
|
||||
errorCode: "connection_unavailable", message: "Reconnect or enable this connection to check its account details." };
|
||||
}
|
||||
// Unsupported methods must not open their credentials just to report this limit.
|
||||
const value = metadata.provider === "anthropic" && metadata.method === "subscription" && !metadata.routing
|
||||
? await credential(row, 0, { actorType: "user", actorId: userId, ...audit, responsibleUserId: userId }) : "";
|
||||
return { ...identity, ...await probeAiAccountIdentity(metadata, value) };
|
||||
}
|
||||
async function select(input: {
|
||||
companyId: string;
|
||||
@@ -1218,5 +1238,5 @@ export function aiConnectionService(db: Db) {
|
||||
return [{ ...row, summary }];
|
||||
});
|
||||
}
|
||||
return { list, selectDecision, selectFastResponse, selectPromptInjectionPrevention, quotaAccounts, subscriptionAccounts: (companyId: string, userId: string) => quotaAccounts(companyId, userId, true), refreshQuotaCredential, select, credential, runtimeCredential, probeUsage, save, setDefault, membership, markAuthenticationFailed };
|
||||
return { list, selectDecision, selectFastResponse, selectPromptInjectionPrevention, quotaAccounts, subscriptionAccounts: (companyId: string, userId: string) => quotaAccounts(companyId, userId, true), refreshQuotaCredential, select, credential, runtimeCredential, probeUsage, accountIdentity, save, setDefault, membership, markAuthenticationFailed };
|
||||
}
|
||||
@@ -1,18 +1,27 @@
|
||||
import { trace, type Span } from "@opentelemetry/api";
|
||||
import { withDatabaseOperation } from "@paperclipai/db";
|
||||
|
||||
type BudgetOperation = "budget.accounting.lock_wait" | "budget.accounting.recover"
|
||||
| "budget.accounting.settlement" | "budget.accounting.transaction"
|
||||
| "budget.admission.evaluate" | "budget.admission.reserve"
|
||||
| "budget.cancellation.deliver" | "budget.enforcement.deliver"
|
||||
| "budget.enforcement.scope" | "budget.exposure.evaluate";
|
||||
|
||||
/** Operator-gated OpenTelemetry only; these spans contain no prompt, receipt,
|
||||
* credential, or provider error text. Diagnostics never change accounting. */
|
||||
export async function withBudgetSpan<T>(name: string, work: () => Promise<T>, attributes?: Record<string, number | boolean>) {
|
||||
if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work();
|
||||
let operation: Promise<T> | undefined;
|
||||
const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => {
|
||||
try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ }
|
||||
export function withBudgetSpan<T>(name: BudgetOperation, work: () => Promise<T>, attributes?: Record<string, number | boolean>) {
|
||||
return withDatabaseOperation(name, async () => {
|
||||
if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work();
|
||||
let operation: Promise<T> | undefined;
|
||||
const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => {
|
||||
try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ }
|
||||
});
|
||||
try {
|
||||
const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span));
|
||||
// Some external SDK implementations can fail before or after the callback.
|
||||
// Own the accounting promise so either failure still executes work once.
|
||||
void Promise.resolve(traced).catch(() => undefined);
|
||||
} catch { /* Preserve the accounting operation if span creation fails. */ }
|
||||
return operation ?? run();
|
||||
});
|
||||
try {
|
||||
const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span));
|
||||
// Some external SDK implementations can fail before or after the callback.
|
||||
// Own the accounting promise so either failure still executes work once.
|
||||
void Promise.resolve(traced).catch(() => undefined);
|
||||
} catch { /* Preserve the accounting operation if span creation fails. */ }
|
||||
return operation ?? run();
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
@@ -49,7 +49,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => {
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, scope.failedRunId));
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const selected = await db.transaction(tx => readChatAuthorizationRun(tx, scope, true));
|
||||
@@ -58,7 +58,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => {
|
||||
expect(selected.run).not.toHaveProperty("runnerProfileJson");
|
||||
expect(selected.run).not.toHaveProperty("stdoutExcerpt");
|
||||
} finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBeGreaterThan(0);
|
||||
expect(summary.responseValueBytes).toBeLessThan(4096);
|
||||
const retry = await db.transaction(tx => readChatAuthorizationRun(tx, scope, false));
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createHmac } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { applyPendingMigrations, chatActions, createDb } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
|
||||
import { cleanupIngressFixture, githubIngressFixture as createFixture } from "../../../__tests__/helpers/github-webhook-ingress.js";
|
||||
@@ -36,7 +36,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh
|
||||
const f = await githubIngressFixture(db, createGitHubWebhookIngress, 768 * 1024);
|
||||
const action = await f.insertAction();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatActions }, capture);
|
||||
try { expect((await f.service.processGitHubWebhookIngress(action.id))?.status).toBe(202); }
|
||||
finally { capture.flush(); }
|
||||
@@ -59,7 +59,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh
|
||||
// A corrupt body must not let a nonclaimant overwrite a live or delayed claim.
|
||||
const action = await f.insertAction({ status, result, payload: { ...f.payload, bodySha256: "0".repeat(64) } });
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatActions }, capture);
|
||||
let response: Response | null;
|
||||
try { response = await f.service.processGitHubWebhookIngress(action.id); }
|
||||
@@ -67,7 +67,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh
|
||||
expect(response?.status ?? null).toBe(accepted ? 202 : null);
|
||||
expect(await f.readAction(action.id)).toEqual(action);
|
||||
expect(f.calls).toEqual([]);
|
||||
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024);
|
||||
expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024);
|
||||
});
|
||||
|
||||
it.each(["received", "failed", "processing"])("settles eligible malformed %s payloads without replay", async status => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { issueDisplayStatusSql } from "./issue-display-status.js";
|
||||
import { externalConversationStateSql } from "./slack-conversation-state.js";
|
||||
import { and, desc, eq, gte, inArray, isNotNull, isNull, notInArray, or, sql } from "drizzle-orm";
|
||||
import type { SQL } from "drizzle-orm";
|
||||
@@ -21,6 +22,7 @@ import {
|
||||
COMPANY_ARTIFACTS_MAX_QUERY_LENGTH,
|
||||
ISSUE_PRIORITIES,
|
||||
ISSUE_STATUSES,
|
||||
ISSUE_DISPLAY_STATUSES,
|
||||
SYSTEM_ISSUE_DOCUMENT_KEYS,
|
||||
type CompanyArtifact,
|
||||
type CompanySearchArtifactSummary,
|
||||
@@ -50,6 +52,7 @@ type IssueSearchRow = {
|
||||
title: string;
|
||||
description: string | null;
|
||||
status: string;
|
||||
displayStatus: string;
|
||||
priority: string;
|
||||
assigneeAgentId: string | null;
|
||||
assigneeUserId: string | null;
|
||||
@@ -193,6 +196,7 @@ function emptySearchCounts(): Record<CompanySearchCountType, number> {
|
||||
function emptyFilterOptionCounts(): CompanySearchFilterOptionCounts {
|
||||
return {
|
||||
status: {},
|
||||
displayStatus: {},
|
||||
priority: {},
|
||||
assigneeAgentId: {},
|
||||
assigneeUserId: {},
|
||||
@@ -224,7 +228,7 @@ function updatedWithinStart(value: string | undefined, now = new Date()): Date |
|
||||
}
|
||||
|
||||
function issueOnlyFiltersActive(query: CompanySearchQuery) {
|
||||
return query.status.length > 0
|
||||
return (query.status.length > 0 || query.displayStatus.length > 0)
|
||||
|| query.priority.length > 0
|
||||
|| query.assigneeAgentId !== undefined
|
||||
|| Boolean(query.assigneeUserId)
|
||||
@@ -236,7 +240,7 @@ function issueOnlyFiltersActive(query: CompanySearchQuery) {
|
||||
|
||||
function activeIssueFilters(query: CompanySearchQuery): Array<{ key: CompanySearchIssueFilterKey; values: string[] }> {
|
||||
const filters: Array<{ key: CompanySearchIssueFilterKey; values: string[] }> = [];
|
||||
if (query.status.length > 0) filters.push({ key: "status", values: query.status });
|
||||
if (query.status.length > 0 || query.displayStatus.length > 0) filters.push({ key: "status", values: [...query.status, ...query.displayStatus] });
|
||||
if (query.assigneeAgentId !== undefined) filters.push({ key: "assigneeAgentId", values: [query.assigneeAgentId ?? "null"] });
|
||||
if (query.assigneeUserId) filters.push({ key: "assigneeUserId", values: [query.assigneeUserId] });
|
||||
if (query.projectId) filters.push({ key: "projectId", values: [query.projectId] });
|
||||
@@ -251,6 +255,7 @@ function queryWithoutFilter(query: CompanySearchQuery, key: CompanySearchIssueFi
|
||||
return {
|
||||
...query,
|
||||
status: key === "status" ? [] : query.status,
|
||||
displayStatus: key === "status" ? [] : query.displayStatus,
|
||||
priority: key === "priority" ? [] : query.priority,
|
||||
assigneeAgentId: key === "assigneeAgentId" ? undefined : query.assigneeAgentId,
|
||||
assigneeUserId: key === "assigneeUserId" ? undefined : query.assigneeUserId,
|
||||
@@ -265,6 +270,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery
|
||||
return {
|
||||
...query,
|
||||
status: [],
|
||||
displayStatus: [],
|
||||
priority: [],
|
||||
assigneeAgentId: undefined,
|
||||
assigneeUserId: undefined,
|
||||
@@ -277,6 +283,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery
|
||||
|
||||
function issueFilterConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] {
|
||||
const conditions: SQL[] = [];
|
||||
if (omit !== "status" && query.displayStatus.length) conditions.push(inArray(issueDisplayStatusSql(), query.displayStatus));
|
||||
if (omit !== "status" && query.status.length > 0) {
|
||||
conditions.push(query.status.length === 1 ? eq(issues.status, query.status[0]!) : inArray(issues.status, query.status));
|
||||
}
|
||||
@@ -315,6 +322,7 @@ function issueFilterConditions(companyId: string, query: CompanySearchQuery, omi
|
||||
// matched-issues CTE (plain columns, no drizzle table references).
|
||||
function matchedFacetConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] {
|
||||
const conditions: SQL[] = [];
|
||||
if (omit !== "status" && query.displayStatus.length) conditions.push(sql`m.display_status = ANY(${sqlTextArray(query.displayStatus)})`);
|
||||
if (omit !== "status" && query.status.length > 0) {
|
||||
conditions.push(sql`m.status = ANY(${sqlTextArray(query.status)})`);
|
||||
}
|
||||
@@ -447,6 +455,7 @@ function issueResult(row: IssueSearchRow, prefix: string, normalizedQuery: strin
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status as CompanySearchIssueSummary["status"],
|
||||
displayStatus: row.displayStatus as CompanySearchIssueSummary["displayStatus"],
|
||||
priority: row.priority as CompanySearchIssueSummary["priority"],
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
@@ -626,7 +635,7 @@ export function companySearchService(db: Db) {
|
||||
conditions.length > 0 ? sql`WHERE ${sql.join(conditions, sql` AND `)}` : sql``;
|
||||
// Count branches must match the result branch's column list; the
|
||||
// trailing NULLs pad the issue data columns.
|
||||
const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`;
|
||||
const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`;
|
||||
const branches: SQL[] = [];
|
||||
|
||||
const wantResultRows = scopeIncludesIssues(scope)
|
||||
@@ -652,6 +661,7 @@ export function companySearchService(db: Db) {
|
||||
(SELECT issue_text.description FROM issues issue_text
|
||||
WHERE issue_text.id = m.id AND issue_text.company_id = ${companyId}) AS description,
|
||||
m.status,
|
||||
m.display_status AS "displayStatus",
|
||||
m.priority,
|
||||
m.assignee_agent_id AS "assigneeAgentId",
|
||||
m.assignee_user_id AS "assigneeUserId",
|
||||
@@ -690,6 +700,7 @@ export function companySearchService(db: Db) {
|
||||
GROUP BY 2
|
||||
`;
|
||||
branches.push(facetBranch("facet:status", sql`m.status`, "status"));
|
||||
branches.push(facetBranch("facet:displayStatus", sql`m.display_status`, "status"));
|
||||
branches.push(facetBranch("facet:priority", sql`m.priority`, "priority"));
|
||||
branches.push(facetBranch("facet:assigneeAgentId", sql`m.assignee_agent_id`, "assigneeAgentId", [sql`m.assignee_agent_id IS NOT NULL`]));
|
||||
branches.push(facetBranch("facet:assigneeUserId", sql`m.assignee_user_id`, "assigneeUserId", [sql`m.assignee_user_id IS NOT NULL`]));
|
||||
@@ -757,6 +768,7 @@ export function companySearchService(db: Db) {
|
||||
title: row.title,
|
||||
description: row.description,
|
||||
status: row.status,
|
||||
displayStatus: row.displayStatus,
|
||||
priority: row.priority,
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
@@ -780,6 +792,9 @@ export function companySearchService(db: Db) {
|
||||
if (row.kind === "type:issue") aggregates.typeCounts.issue = count;
|
||||
else if (row.kind === "type:comment") aggregates.typeCounts.comment = count;
|
||||
else if (row.kind === "type:document") aggregates.typeCounts.document = count;
|
||||
else if (row.kind === "facet:displayStatus" && row.value && (ISSUE_DISPLAY_STATUSES as readonly string[]).includes(row.value)) {
|
||||
aggregates.filterOptionCounts.displayStatus![row.value as keyof NonNullable<CompanySearchFilterOptionCounts["displayStatus"]>] = count;
|
||||
}
|
||||
else if (row.kind === "facet:status" && row.value && (ISSUE_STATUSES as readonly string[]).includes(row.value)) {
|
||||
aggregates.filterOptionCounts.status[row.value as keyof CompanySearchFilterOptionCounts["status"]] = count;
|
||||
} else if (row.kind === "facet:priority" && row.value && (ISSUE_PRIORITIES as readonly string[]).includes(row.value)) {
|
||||
|
||||
@@ -32,7 +32,7 @@ type PreparedAuthorization = {
|
||||
expandedTargets: Set<string>;
|
||||
referencedConnections: string[];
|
||||
currentGatewayResource: ConnectionManagementResource | undefined;
|
||||
savedNonAiConnection: boolean;
|
||||
savedConnection: boolean;
|
||||
};
|
||||
type ConnectionCapability = { allowed: boolean; reason: string; operation: string };
|
||||
|
||||
@@ -88,7 +88,7 @@ export function connectionManagementService(db: Db) {
|
||||
const expandedTargets = new Set(options.targetAgentIds ?? []);
|
||||
const referencedConnections: string[] = [];
|
||||
let currentGatewayResource: ConnectionManagementResource | undefined;
|
||||
let savedNonAiConnection = false;
|
||||
let savedConnection = false;
|
||||
if (resource.gatewayId && !options.gatewayContextChecked) {
|
||||
const [gateway] = await db.select().from(toolMcpGateways).where(and(eq(toolMcpGateways.id, resource.gatewayId), eq(toolMcpGateways.companyId, resource.companyId)));
|
||||
if (!gateway) throw forbidden("Gateway is unavailable in this company");
|
||||
@@ -159,7 +159,7 @@ export function connectionManagementService(db: Db) {
|
||||
if (resource.connectionId) {
|
||||
const [connection] = await db.select().from(toolConnections).where(and(eq(toolConnections.id, resource.connectionId), eq(toolConnections.companyId, resource.companyId)));
|
||||
if (!connection) throw forbidden("Connection is unavailable in this company");
|
||||
savedNonAiConnection = connection.connectionPurpose !== "ai";
|
||||
savedConnection = true;
|
||||
resource = { ...resource, selfServiceUserId: connection.createdByUserId,
|
||||
purpose: connection.connectionPurpose === "ai" ? "ai" : connection.connectionPurpose === "channel" ? "chat" : "tool",
|
||||
provider: typeof connection.config.sourceTemplateKey === "string" ? connection.config.sourceTemplateKey
|
||||
@@ -241,7 +241,7 @@ export function connectionManagementService(db: Db) {
|
||||
}
|
||||
}
|
||||
return { actor, action, resource, options, responsibleUserId, workContext, capturedRun,
|
||||
expandedTargets, referencedConnections, currentGatewayResource, savedNonAiConnection };
|
||||
expandedTargets, referencedConnections, currentGatewayResource, savedConnection };
|
||||
}
|
||||
|
||||
async function evaluateTarget(prepared: PreparedAuthorization, targetAgentId: string | undefined): Promise<AuthorizationDecision> {
|
||||
@@ -388,7 +388,9 @@ export function connectionManagementService(db: Db) {
|
||||
}
|
||||
const resolved = prepared.resource;
|
||||
const isCanonicalAgentId = (id: string) => isUuidLike(id) && id === id.toLowerCase();
|
||||
const eligible = actor.type === "board" && prepared.savedNonAiConnection
|
||||
// Direct AI accounts use the same target checks. Pools with members still
|
||||
// use the full evaluator through referencedConnections below.
|
||||
const eligible = actor.type === "board" && prepared.savedConnection
|
||||
&& [resolved.profileId, resolved.gatewayId, resolved.routineId, resolved.projectId, resolved.issueId].every(id => id == null)
|
||||
&& !Object.hasOwn(resolved, "requestedProvider") && !Object.hasOwn(resolved, "requestedInstallCompanyIds")
|
||||
&& !resolved.selfServiceAgentInstalls
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "@paperclipai/db/work-signals";
|
||||
import { withDatabaseOperation } from "@paperclipai/db";
|
||||
import { createWorkScheduler } from "./work-scheduler.js";
|
||||
import { type DeliveryQueue } from "./delivery-work-notifications.js";
|
||||
import { beginIdleTrackedWork } from "./task-admission.js";
|
||||
@@ -70,7 +71,7 @@ export function createDeliveryWorkCoordinator(input: {
|
||||
const attempt = new AbortController();
|
||||
let runFailed = false;
|
||||
controller = attempt;
|
||||
running = Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue))
|
||||
running = withDatabaseOperation(`delivery.${queue}`, () => Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue))
|
||||
.then(() => workerStopped || !(input.canRun() || (task.runDuringDrain && input.canReconcile?.())) ? undefined : task.run(attempt.signal).catch(error => { runFailed = true; throw error; })).then(() => workerStopped ? false : task.hasPending()).then(pending => {
|
||||
pendingWork = pending || databaseWorkPending(input.owner, queue);
|
||||
if (pendingWork) schedule(task.retryMs);
|
||||
@@ -98,7 +99,7 @@ export function createDeliveryWorkCoordinator(input: {
|
||||
controller = null;
|
||||
finish();
|
||||
if (dirty) schedule(0);
|
||||
});
|
||||
}));
|
||||
}
|
||||
function wake() {
|
||||
if (workerStopped) return;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { agentWakeupRequests, chatActions, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db";
|
||||
import { and, eq, inArray, or, sql, type SQLWrapper } from "drizzle-orm";
|
||||
import { agentWakeupRequests, chatActions, chatConversations, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db";
|
||||
import { and, eq, exists, inArray, or, sql, type SQLWrapper } from "drizzle-orm";
|
||||
import { HttpError } from "../errors.js";
|
||||
|
||||
export interface CommittedChatResponseAuthorizationInput {
|
||||
@@ -233,6 +233,22 @@ export interface DeferredChatWakeScope {
|
||||
issueId: string;
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
/** Identify saved input by its canonical receipt, never by a caller's source
|
||||
* label. This is selection only; the inbound owner rechecks live authority. */
|
||||
export function deferredChatInputPredicate(db: Db) {
|
||||
return exists(db.select({ id: chatActions.id }).from(chatActions)
|
||||
.innerJoin(chatConversations, and(eq(chatConversations.companyId, chatActions.companyId),
|
||||
eq(chatConversations.id, chatActions.conversationId)))
|
||||
.innerJoin(chatEndpoints, and(eq(chatEndpoints.companyId, chatActions.companyId),
|
||||
eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.id, chatConversations.endpointId)))
|
||||
.where(and(eq(chatActions.companyId, agentWakeupRequests.companyId), eq(chatActions.id, agentWakeupRequests.id),
|
||||
eq(chatActions.kind, "inbound_wakeup"), eq(chatActions.status, "processed"),
|
||||
eq(chatEndpoints.assignedAgentId, agentWakeupRequests.agentId), eq(chatEndpoints.externalExecutionPolicy, "restricted"),
|
||||
sql`${chatConversations.issueId}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
sql`${chatActions.payload}->>'issueId' = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
sql`${chatActions.payload}->>'agentId' = ${agentWakeupRequests.agentId}::text`)));
|
||||
}
|
||||
const deferredChatWakeOwners = new WeakMap<Db, (input: DeferredChatWakeScope) => Promise<void>>();
|
||||
|
||||
/** The live inbound owner reauthorizes the original source. Cleanup owns no chat authority. */
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { and, eq, or, sql } from "drizzle-orm";
|
||||
import { environmentLeases, sandboxCreationOperations, type Db } from "@paperclipai/db";
|
||||
import { hasDeferredEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
|
||||
export const ENVIRONMENT_CLEANUP_RETRY_MS = 5 * 60_000;
|
||||
|
||||
export function recoverableSandboxCreationCondition() {
|
||||
return or(eq(sandboxCreationOperations.state, "pending"), and(
|
||||
eq(sandboxCreationOperations.state, "needs_reconciliation"),
|
||||
sql`coalesce(${sandboxCreationOperations.metadata}->>'creationReconciliationReason', '') <> 'company_deleted'`,
|
||||
sql`${sandboxCreationOperations.metadata}->>'runnerTaskLease' = 'true'`,
|
||||
sql`jsonb_typeof(${sandboxCreationOperations.metadata}->'runnerTaskCreationContextId') = 'string' and ${sandboxCreationOperations.metadata}->>'runnerTaskCreationContextId' <> ''`,
|
||||
sql`jsonb_typeof(${sandboxCreationOperations.metadata}->'creationProviderLeaseId') = 'string' and ${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' <> ''`,
|
||||
));
|
||||
}
|
||||
|
||||
export function pendingCleanupDeadlineSql(now = Date.now()) {
|
||||
const deadline = sql`case
|
||||
when ${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'
|
||||
@@ -21,7 +31,7 @@ export async function nextEnvironmentCleanupAt(db: Db, backoffMs = ENVIRONMENT_C
|
||||
extract(epoch from ${environmentLeases.updatedAt}) * 1000 + ${backoffMs},
|
||||
${pendingCleanupDeadlineSql()}))` }).from(environmentLeases).where(eq(environmentLeases.status, "pending_cleanup"));
|
||||
const [creation] = await db.select({ next: sql<string | null>`min(extract(epoch from ${sandboxCreationOperations.nextAttemptAt}) * 1000)` })
|
||||
.from(sandboxCreationOperations).where(eq(sandboxCreationOperations.state, "pending"));
|
||||
.from(sandboxCreationOperations).where(recoverableSandboxCreationCondition());
|
||||
const deadlines = [row?.next, creation?.next].filter((value): value is string => value != null).map(Number);
|
||||
const saved = deadlines.length ? Math.min(...deadlines) : null;
|
||||
const buffered = hasDeferredEnvironmentCleanup(db) ? Date.now() + backoffMs : null;
|
||||
|
||||
@@ -3382,7 +3382,7 @@ function createPluginEnvironmentDriver(
|
||||
async releaseRunLease(input) {
|
||||
await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId)));
|
||||
const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) ||
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true);
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup);
|
||||
const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input);
|
||||
await workerManager.call(plugin.id, "environmentReleaseLease", {
|
||||
driverKey,
|
||||
@@ -3425,7 +3425,7 @@ function createPluginEnvironmentDriver(
|
||||
async destroyRunLease(input) {
|
||||
await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"destroy"}'::jsonb` }).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId)));
|
||||
const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) ||
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true);
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup);
|
||||
const { pluginKey, driverKey, driverConfig } = await resolvePluginDriverForRelease(input);
|
||||
await destroyPluginEnvironmentLease({
|
||||
db,
|
||||
@@ -3462,7 +3462,7 @@ function createPluginEnvironmentDriver(
|
||||
|
||||
async retryPendingSandboxTeardown(input) {
|
||||
const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) ||
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true);
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup);
|
||||
const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input);
|
||||
const environmentId = input.environment?.id ?? readString(input.lease.metadata?.environmentId);
|
||||
if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity");
|
||||
@@ -3778,7 +3778,7 @@ export function environmentRuntimeService(
|
||||
},
|
||||
|
||||
/** Dispatch plugin-provided Runner execution, including resource preparation and task lifecycle. */
|
||||
async task(input: { companyId: string; leaseId: string; operation: PluginEnvironmentTaskOperation }) {
|
||||
async task(input: { companyId: string; leaseId: string; operation: PluginEnvironmentTaskOperation; executionContext?: unknown }) {
|
||||
if (!options.pluginWorkerManager) throw new Error("Plugin-provided Runner execution worker manager unavailable");
|
||||
return executeEnvironmentTask(db, options.pluginWorkerManager, input);
|
||||
},
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
import { attachSandboxCreation, environmentService } from "./environments.js";
|
||||
import { createOwnedSandbox, recordSandboxCreationResource } from "./sandbox-creation-operations.js";
|
||||
import { createOwnedSandbox, queueSandboxCreationCleanup, recordSandboxCreationResource } from "./sandbox-creation-operations.js";
|
||||
import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js";
|
||||
import { readWorkspaceProviderBinding } from "@paperclipai/shared";
|
||||
import { settlePluginTaskCredentialHandoff } from "./plugin-task-credential-writeback.js";
|
||||
import { and, eq, inArray, or, sql } from "drizzle-orm";
|
||||
import { environmentLeases, executionWorkspaces, environments, heartbeatRuns, issues, projects, type Db } from "@paperclipai/db";
|
||||
import { freshWorkspaceSchema, environmentTaskOperationSchema, parseEnvironmentTaskResult, type PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk";
|
||||
import { freshWorkspaceSchema, environmentTaskOperationSchema, parseEnvironmentTaskResult,
|
||||
type HostToWorkerMethodName, type HostToWorkerMethods, type PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk";
|
||||
import { pluginRegistryService } from "./plugin-registry.js";
|
||||
import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
||||
import { NativeExecutionContext } from "./native-runtime/native-execution-context.js";
|
||||
|
||||
export async function pluginEnvironmentSupportsTasks(db: Db, workers: PluginWorkerManager, lease: { companyId: string; metadata: Record<string, unknown> | null }) {
|
||||
const metadata = lease.metadata ?? {};
|
||||
@@ -31,6 +33,7 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
companyId: string;
|
||||
leaseId: string;
|
||||
operation: PluginEnvironmentTaskOperation;
|
||||
executionContext?: unknown;
|
||||
}) {
|
||||
const operation = environmentTaskOperationSchema.parse(input.operation);
|
||||
const [row] = await db.select({ lease: environmentLeases, environment: environments, run: heartbeatRuns })
|
||||
@@ -40,6 +43,13 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
.where(and(eq(environmentLeases.id, input.leaseId), eq(environmentLeases.companyId, input.companyId))).limit(1);
|
||||
if (!row || !row.lease.providerLeaseId) throw new Error("Plugin-provided Runner execution lease unavailable");
|
||||
const { lease, environment, run } = row;
|
||||
const context = input.executionContext;
|
||||
if (context !== undefined) {
|
||||
if (!(context instanceof NativeExecutionContext) || !run) {
|
||||
throw new Error("native_execution_context_unavailable");
|
||||
}
|
||||
context.assertScope({ companyId: input.companyId, agentId: run.agentId, runId: run.id });
|
||||
}
|
||||
const taskId = row.lease.providerLeaseId;
|
||||
const metadata = lease.metadata ?? {};
|
||||
const probe = metadata.runnerTaskProbe as { runId?: unknown; sessionId?: unknown; expiresAtMs?: unknown } | undefined;
|
||||
@@ -128,6 +138,19 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
!workers.getWorker(plugin.id)?.supportedMethods.includes("environmentTask")) {
|
||||
throw new Error("Plugin-provided Runner execution provider unavailable");
|
||||
}
|
||||
const callWorker = async <M extends HostToWorkerMethodName>(method: M, params: HostToWorkerMethods[M][0], timeout?: number): Promise<HostToWorkerMethods[M][1]> => {
|
||||
if (!context) {
|
||||
const cleanup = metadata.runnerTaskStartupCleanup as { contextId?: unknown; requiresPreparationStop?: unknown } | undefined;
|
||||
if (["stop", "status", "complete", "credentials"].includes(operation.kind) && typeof cleanup?.contextId === "string" && typeof cleanup.requiresPreparationStop === "boolean") {
|
||||
return workers.call(plugin.id, method, params, timeout, undefined, undefined, cleanup.contextId);
|
||||
}
|
||||
return timeout === undefined ? workers.call(plugin.id, method, params) : workers.call(plugin.id, method, params, timeout);
|
||||
}
|
||||
const settled = context.trackWorker();
|
||||
const result = await workers.call(plugin.id, method, params, Math.max(1, Math.min(timeout ?? 15_000, context.deadlineAtMs - Date.now())), undefined, settled, context.id);
|
||||
context.assertOpen();
|
||||
return result;
|
||||
};
|
||||
const issue = scopesProjects && lease.issueId ? await db.select({ id: issues.id }).from(issues)
|
||||
.where(and(eq(issues.id, lease.issueId), eq(issues.companyId, input.companyId))).limit(1).then(rows => rows[0]) : null;
|
||||
if (scopesProjects && lease.issueId && !issue) throw new Error("Plugin-provided Runner execution issue unavailable");
|
||||
@@ -219,7 +242,7 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
}
|
||||
try {
|
||||
if (operation.kind === "describe" && metadata.runnerTaskReused === true) {
|
||||
const state = parseEnvironmentTaskResult({kind:"status"}, taskId, await workers.call(plugin.id, "environmentTask", {
|
||||
const state = parseEnvironmentTaskResult({kind:"status"}, taskId, await callWorker("environmentTask", {
|
||||
driverKey: metadata.driverKey, companyId: input.companyId, environmentId: environment?.id ?? null,
|
||||
issueId: lease.issueId, config: driverConfig, taskId, runId: run?.id ?? null, agentId: run?.agentId ?? null,
|
||||
projectIds, lease: {providerLeaseId: taskId, metadata}, operation:{kind:"status"},
|
||||
@@ -233,37 +256,74 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
runnerTaskSessionRetained:state.sessionStateAvailable === true})}::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, metadata.runnerTaskPreviousLeaseId), eq(environmentLeases.companyId, input.companyId)));
|
||||
const replacementDriverKey = metadata.driverKey;
|
||||
let abandoned = false;
|
||||
const fresh = await createOwnedSandbox(db, {
|
||||
companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId,
|
||||
issueId: lease.issueId, executionWorkspaceId: lease.executionWorkspaceId, provider: lease.provider!,
|
||||
pluginId: plugin.id, driverKey: metadata.driverKey, config: {}, requestKey: taskId,
|
||||
metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: plugin.pluginKey, driverKey: metadata.driverKey, runnerTaskLease: true },
|
||||
}, identity => workers.call(plugin.id, "environmentAcquireLease", {driverKey:replacementDriverKey,
|
||||
companyId:input.companyId,environmentId:environment!.id,issueId:lease.issueId,config:driverConfig,
|
||||
runId:identity.runId,creationOperationId:identity.id,agentId:run!.agentId,adapterType:"paperclip_runner",
|
||||
...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? {requestedExpiresAt:metadata.runnerTaskRequestedExpiresAt} : {})}));
|
||||
metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: plugin.pluginKey, driverKey: metadata.driverKey,
|
||||
runnerTaskLease: true, agentId: run!.agentId,
|
||||
...(context ? { runnerTaskCreationContextId: context.id } : {}) },
|
||||
}, async identity => {
|
||||
const params = { driverKey: replacementDriverKey, companyId: input.companyId, environmentId: environment!.id,
|
||||
issueId: lease.issueId, config: driverConfig, runId: identity.runId, creationOperationId: identity.id,
|
||||
agentId: run!.agentId, adapterType: "paperclip_runner",
|
||||
...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? { requestedExpiresAt: metadata.runnerTaskRequestedExpiresAt } : {}) };
|
||||
if (!context) return workers.call(plugin.id, "environmentAcquireLease", params, 30_000);
|
||||
const settled = context.trackWorker();
|
||||
let observedResource: Parameters<typeof queueSandboxCreationCleanup>[1] | undefined;
|
||||
let dispatched = false;
|
||||
try {
|
||||
return await context.wait(() => {
|
||||
dispatched = true;
|
||||
return workers.call(plugin.id, "environmentAcquireLease", params, 30_000, undefined, settled, context.id,
|
||||
async result => {
|
||||
if (!result.providerLeaseId || result.providerLeaseId === taskId) throw new Error("Plugin replacement task identity unavailable");
|
||||
const resource = { id: identity.id, companyId: input.companyId, environmentId: environment!.id,
|
||||
providerLeaseId: result.providerLeaseId, providerMetadata: result.metadata };
|
||||
await recordSandboxCreationResource(db, resource);
|
||||
observedResource = resource;
|
||||
if (abandoned || !context.active || context.signal.aborted || Date.now() >= context.deadlineAtMs) {
|
||||
await queueSandboxCreationCleanup(db, resource);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
if (!dispatched) settled();
|
||||
abandoned = true;
|
||||
if (observedResource) await queueSandboxCreationCleanup(db, observedResource);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
if (!fresh.providerLeaseId || fresh.providerLeaseId === taskId) throw new Error("Plugin replacement task identity unavailable");
|
||||
await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId,
|
||||
environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId });
|
||||
const {runnerTaskIdentity:_identity,runnerTaskPriorIdentity:_prior,runnerTaskCredentialHandoff:_handoff,
|
||||
runnerTaskResumeFrom:_resume,runnerTaskPreviousLeaseId:_previous,...rest}=metadata;
|
||||
const replacementMetadata={...rest,providerMetadata:fresh.metadata ?? {},runnerTaskReused:false,runnerTaskDispatched:false,
|
||||
runnerTaskExecutionStopped:false,runnerTaskSessionRetained:false};
|
||||
{
|
||||
try {
|
||||
context?.assertOpen();
|
||||
if (!context) await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId,
|
||||
environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId });
|
||||
const expiresAt=fresh.expiresAt ? new Date(fresh.expiresAt) : null;
|
||||
const deadline=typeof metadata.runnerTaskRequestedExpiresAt === "string" ? Date.parse(metadata.runnerTaskRequestedExpiresAt) : null;
|
||||
if ((expiresAt && (!Number.isFinite(expiresAt.getTime()) || expiresAt.getTime() <= Date.now())) ||
|
||||
(deadline !== null && (!Number.isFinite(deadline) || !expiresAt || expiresAt.getTime() > deadline)))
|
||||
throw new Error("Plugin replacement task expiry is not bounded");
|
||||
await attachSandboxCreation(db, String(fresh.metadata!.creationOperationId), { companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId }, async tx => {
|
||||
context?.assertOpen();
|
||||
const [updated] = await tx.update(environmentLeases).set({providerLeaseId:fresh.providerLeaseId,expiresAt,
|
||||
metadata:{...replacementMetadata,creationOperationId:fresh.metadata!.creationOperationId},
|
||||
}).where(and(eq(environmentLeases.id,lease.id),eq(environmentLeases.providerLeaseId,taskId),
|
||||
eq(environmentLeases.status,"active"))).returning({id:environmentLeases.id});
|
||||
eq(environmentLeases.status,"active"), sql`coalesce(${environmentLeases.metadata}->>'runnerTaskRetiring', 'false') != 'true'`)).returning({id:environmentLeases.id});
|
||||
if (!updated) throw new Error("Plugin task replacement ownership changed");
|
||||
return (await environmentService(tx).getLeaseById(updated.id))!;
|
||||
const attached = (await environmentService(tx).getLeaseById(updated.id))!;
|
||||
context?.assertOpen();
|
||||
return attached;
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
if (context) await queueSandboxCreationCleanup(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId,
|
||||
environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId, providerMetadata: fresh.metadata });
|
||||
throw error;
|
||||
}
|
||||
return executeEnvironmentTask(db, workers, input);
|
||||
}
|
||||
@@ -275,14 +335,17 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
return snapshot.credential;
|
||||
});
|
||||
}
|
||||
const result = await workers.call(plugin.id, "environmentTask", {
|
||||
const params = {
|
||||
driverKey: metadata.driverKey, companyId: input.companyId, environmentId: environment?.id ?? null,
|
||||
issueId: lease.issueId, config: driverConfig,
|
||||
taskId, runId: run?.id ?? (activeProbe ? String(probe.runId) : null), agentId: run?.agentId ?? null, projectIds,
|
||||
lease: { providerLeaseId: lease.providerLeaseId, metadata: lease.metadata ?? undefined },
|
||||
...(workspaceBinding ? { workspace: workspaceBinding } : {}),
|
||||
...(freshWorkspace && scopesProjects ? { fresh: freshWorkspace } : {}), operation,
|
||||
}, operation.kind === "copilot_metadata" ? Math.max(1, Math.min(40_000, Number(probe?.expiresAtMs) - Date.now())) : 15_000);
|
||||
};
|
||||
const timeout = operation.kind === "copilot_metadata" ? Math.max(1, Math.min(40_000, Number(probe?.expiresAtMs) - Date.now()))
|
||||
: 15_000;
|
||||
const result = await callWorker("environmentTask", params, timeout);
|
||||
const parsed = parseEnvironmentTaskResult(operation, taskId, result);
|
||||
if (operation.kind === "read_file") {
|
||||
const [current] = await db.select({ lease: environmentLeases, run: heartbeatRuns }).from(environmentLeases)
|
||||
@@ -316,7 +379,9 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
}
|
||||
if ((parsed.kind === "status" || parsed.kind === "copilot_metadata") && parsed.executionStopped === true) {
|
||||
await db.update(environmentLeases).set({
|
||||
metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ runnerTaskExecutionStopped: true, runnerTaskSessionRetained: parsed.kind === "status" && parsed.sessionStateAvailable === true })}::jsonb`,
|
||||
metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ runnerTaskExecutionStopped: true,
|
||||
runnerTaskSessionRetained: parsed.kind === "status" && parsed.sessionStateAvailable === true,
|
||||
...(parsed.kind === "status" && parsed.preparationStopped === true ? { runnerTaskPreparationStopped: true } : {}) })}::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, input.companyId)));
|
||||
}
|
||||
if (operation.kind === "complete") {
|
||||
@@ -353,16 +418,22 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor
|
||||
metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskRetiring":true}'::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId)))
|
||||
.returning({ metadata: environmentLeases.metadata, heartbeatRunId: environmentLeases.heartbeatRunId });
|
||||
if (current?.metadata?.runnerTaskDispatched !== true) {
|
||||
const startup = current?.metadata?.runnerTaskStartup;
|
||||
const startupCleanup = current?.metadata?.runnerTaskStartupCleanup as { requiresPreparationStop?: boolean } | undefined;
|
||||
if (current?.metadata?.runnerTaskDispatched !== true && !startup && !startupCleanup) {
|
||||
if (current?.metadata) await settlePluginTaskCredentialHandoff(db, { companyId: lease.companyId, leaseId: lease.id }, current.metadata,
|
||||
async () => { throw new Error("Undispatched task must not request credentials"); });
|
||||
return current?.metadata?.runnerTaskExecutionStopped === true;
|
||||
}
|
||||
const scope = { companyId: lease.companyId, leaseId: lease.id };
|
||||
try {
|
||||
const contextId = (startupCleanup as { contextId?: unknown } | undefined)?.contextId ?? (startup as { contextId?: unknown } | undefined)?.contextId;
|
||||
const draining = typeof contextId === "string" ? workers.drainStartup?.(contextId) : undefined;
|
||||
void draining?.catch(() => undefined);
|
||||
await executeEnvironmentTask(db, workers, { ...scope, operation: { kind: "stop" } });
|
||||
const state = await executeEnvironmentTask(db, workers, { ...scope, operation: { kind: "status" } });
|
||||
if (state.kind !== "status" || state.executionStopped !== true) throw new Error("Plugin Runner termination is unconfirmed");
|
||||
if (state.kind !== "status" || state.executionStopped !== true || ((startup || startupCleanup?.requiresPreparationStop) && state.preparationStopped !== true)) throw new Error("Plugin Runner termination is unconfirmed");
|
||||
await draining;
|
||||
await executeEnvironmentTask(db, workers, { ...scope, operation: { kind: "complete" } });
|
||||
return true;
|
||||
} catch {
|
||||
|
||||
@@ -16,7 +16,7 @@ import { hasVerifiedWorkspaceRepair, workspaceRecoveryDisplay } from "./workspac
|
||||
|
||||
const blockerContextKeys = ["issueId", "taskId", "source", "connectionIntentResolution",
|
||||
"interactionKind", "interactionStatus", "mutation", "interactionId"] as const;
|
||||
const blockerProfileKeys = ["adapterDispatch", "nativeExecutionInput", "nativeWorkspaceSync"] as const;
|
||||
const blockerProfileKeys = ["adapterDispatch", "nativeExecutionInput", "nativeWorkspaceSync", "remoteWorkspace"] as const;
|
||||
const blockerContextColumn = sql<Record<string, unknown> | null>`case
|
||||
when ${heartbeatRuns.contextSnapshot} is null then null else (
|
||||
select coalesce(jsonb_object_agg(field.key, field.value), '{}'::jsonb)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
|
||||
import { enableRunFailureReporting } from "../__tests__/helpers/run-failure-reporting.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, issueRecoveryActions, issueThreadInteractions, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db";
|
||||
import {
|
||||
@@ -78,6 +78,55 @@ describeEmbeddedPostgres("reconcileAbandonedExecutionControl reports a genuine f
|
||||
return { companyId, agentId, issueId, runId };
|
||||
}
|
||||
|
||||
it("recognizes pending cleanup selected only by the run deadline batch", async () => {
|
||||
const f = await seedAbandonedRunFixture(), now = new Date();
|
||||
const cleanupIds = Array.from({ length: 51 }, () => randomUUID());
|
||||
const hidden = cleanupIds[50]!;
|
||||
await db.insert(heartbeatRuns).values(cleanupIds.map(id => ({ id, companyId: f.companyId, agentId: f.agentId,
|
||||
nativeIssueId: f.issueId, runtimeMode: "native", status: "running",
|
||||
executionControlDeadlineAt: id === hidden ? new Date(now.getTime() - 120_000) : null })));
|
||||
await db.insert(nativeRunFinalizations).values(cleanupIds.map(runId => ({ runId, companyId: f.companyId,
|
||||
issueId: f.issueId, phase: "terminal_failure", failureCode: "native_startup_cleanup_pending",
|
||||
controlDeadlineAt: new Date(now.getTime() - (runId === hidden ? 60_000 : 120_000)) })));
|
||||
const attempted: string[] = [];
|
||||
try {
|
||||
await reconcileAbandonedExecutionControl(db, now, { startupCleanup: async ({ runId }) => { attempted.push(runId); throw new Error("Provider unavailable"); } });
|
||||
expect(attempted).toContain(hidden);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, hidden)))[0].status).toBe("running");
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, hidden)))[0].controlDeadlineAt! > now).toBe(true);
|
||||
} finally {
|
||||
await db.update(nativeRunFinalizations).set({ controlDeadlineAt: null }).where(inArray(nativeRunFinalizations.runId, cleanupIds));
|
||||
await db.update(heartbeatRuns).set({ executionControlDeadlineAt: null }).where(inArray(heartbeatRuns.id, cleanupIds));
|
||||
}
|
||||
});
|
||||
|
||||
it("moves failed startup cleanup retries behind later expired runs", async () => {
|
||||
const f = await seedAbandonedRunFixture();
|
||||
const now = new Date();
|
||||
const cleanupIds = Array.from({ length: 50 }, () => randomUUID());
|
||||
const firstDeadline = new Date(now.getTime() - 120_000);
|
||||
await db.insert(heartbeatRuns).values(cleanupIds.map(id => ({ id, companyId: f.companyId, agentId: f.agentId,
|
||||
nativeIssueId: f.issueId, runtimeMode: "native", status: "running", executionControlDeadlineAt: firstDeadline,
|
||||
contextSnapshot: { issueId: f.issueId } })));
|
||||
await db.insert(nativeRunFinalizations).values(cleanupIds.map(runId => ({ runId, companyId: f.companyId,
|
||||
issueId: f.issueId, phase: "terminal_failure", failureCode: "native_startup_cleanup_pending", controlDeadlineAt: firstDeadline })));
|
||||
const attempted: string[] = [];
|
||||
const startupCleanup = async ({ runId }: { runId: string }) => { attempted.push(runId); throw new Error("Provider unavailable"); };
|
||||
try {
|
||||
await reconcileAbandonedExecutionControl(db, now, { startupCleanup });
|
||||
await reconcileAbandonedExecutionControl(db, now, { startupCleanup });
|
||||
expect(new Set(attempted)).toEqual(new Set(cleanupIds));
|
||||
expect(attempted).toHaveLength(50);
|
||||
const [later] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(later?.status).toBe("failed");
|
||||
const retries = await db.select().from(nativeRunFinalizations).where(inArray(nativeRunFinalizations.runId, cleanupIds));
|
||||
expect(retries.every(row => row.controlDeadlineAt && row.controlDeadlineAt > now)).toBe(true);
|
||||
} finally {
|
||||
await db.update(nativeRunFinalizations).set({ controlDeadlineAt: null }).where(inArray(nativeRunFinalizations.runId, cleanupIds));
|
||||
await db.update(heartbeatRuns).set({ executionControlDeadlineAt: null }).where(inArray(heartbeatRuns.id, cleanupIds));
|
||||
}
|
||||
});
|
||||
|
||||
it("reports exactly one Sentry event for a genuine finalization-deadline failure", async () => {
|
||||
const { runId } = await seedAbandonedRunFixture();
|
||||
const captureCallsBefore = mockCaptureRunFailure.mock.calls.length;
|
||||
|
||||
@@ -8,7 +8,7 @@ import { notifyInstructionCollection } from "./instruction-collection-notificati
|
||||
import { notifyExecutionRunWork } from "./execution-control-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { and, eq, isNotNull, lte, sql } from "drizzle-orm";
|
||||
import { and, asc, eq, isNotNull, lte, sql } from "drizzle-orm";
|
||||
import {
|
||||
agents,
|
||||
heartbeatRuns,
|
||||
@@ -22,17 +22,25 @@ import { hasLiveLegacyController } from "./legacy-controller-lease.js";
|
||||
import { reportRunFailure } from "./run-failure-report.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
|
||||
import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js";
|
||||
import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "./execution-control-deadline.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { NATIVE_STARTUP_CLEANUP_PENDING, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-runtime/native-startup-cleanup.js";
|
||||
|
||||
/** Only newly recorded control deadlines are eligible. Upgrades never replay ambiguous historical runs. */
|
||||
export async function reconcileAbandonedExecutionControl(
|
||||
db: Db,
|
||||
now = new Date(),
|
||||
options: { startupCleanup?: (scope: { companyId: string; runId: string }) => Promise<unknown> } = {},
|
||||
) {
|
||||
const nativeDue = await db
|
||||
.select({
|
||||
runId: nativeRunFinalizations.runId,
|
||||
issueId: nativeRunFinalizations.issueId,
|
||||
companyId: nativeRunFinalizations.companyId,
|
||||
failureCode: nativeRunFinalizations.failureCode,
|
||||
leaseOwner: nativeRunFinalizations.leaseOwner,
|
||||
controllerGeneration: nativeRunFinalizations.controllerGeneration,
|
||||
attempt: nativeRunFinalizations.attempt,
|
||||
})
|
||||
.from(nativeRunFinalizations)
|
||||
.where(
|
||||
@@ -41,20 +49,28 @@ export async function reconcileAbandonedExecutionControl(
|
||||
lte(nativeRunFinalizations.controlDeadlineAt, now),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(nativeRunFinalizations.controlDeadlineAt), asc(nativeRunFinalizations.runId))
|
||||
.limit(50);
|
||||
const controlDue = await db
|
||||
.select({
|
||||
runId: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
context: heartbeatRuns.contextSnapshot,
|
||||
nativeIssueId: nativeRunFinalizations.issueId,
|
||||
failureCode: nativeRunFinalizations.failureCode,
|
||||
leaseOwner: nativeRunFinalizations.leaseOwner,
|
||||
controllerGeneration: nativeRunFinalizations.controllerGeneration,
|
||||
attempt: nativeRunFinalizations.attempt,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.leftJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
|
||||
.where(
|
||||
and(
|
||||
isNotNull(heartbeatRuns.executionControlDeadlineAt),
|
||||
lte(heartbeatRuns.executionControlDeadlineAt, now),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(heartbeatRuns.executionControlDeadlineAt), asc(heartbeatRuns.id))
|
||||
.limit(50);
|
||||
const due = [
|
||||
...new Map(
|
||||
@@ -62,10 +78,10 @@ export async function reconcileAbandonedExecutionControl(
|
||||
...nativeDue,
|
||||
...controlDue.map((row) => ({
|
||||
...row,
|
||||
issueId:
|
||||
issueId: row.nativeIssueId ?? (
|
||||
typeof row.context?.issueId === "string"
|
||||
? row.context.issueId
|
||||
: null,
|
||||
: null),
|
||||
})),
|
||||
].map((row) => [row.runId, row]),
|
||||
).values(),
|
||||
@@ -78,6 +94,25 @@ export async function reconcileAbandonedExecutionControl(
|
||||
while (nextCandidate < due.length) {
|
||||
const candidate = due[nextCandidate++]!;
|
||||
try {
|
||||
if (candidate.failureCode === NATIVE_STARTUP_CLEANUP_PENDING) {
|
||||
if (!options.startupCleanup) continue;
|
||||
try {
|
||||
await options.startupCleanup({ companyId: candidate.companyId, runId: candidate.runId });
|
||||
} finally {
|
||||
await db.transaction(async tx => {
|
||||
if (candidate.issueId) await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, candidate.issueId)).for("update");
|
||||
const [owner] = await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, candidate.runId)).for("update");
|
||||
if (owner?.failureCode !== NATIVE_STARTUP_CLEANUP_PENDING || !owner.controlDeadlineAt || owner.controlDeadlineAt > now ||
|
||||
owner.leaseOwner !== candidate.leaseOwner || owner.controllerGeneration !== candidate.controllerGeneration || owner.attempt !== candidate.attempt) return;
|
||||
const cleanupRetryAt = new Date(Math.max(now.getTime(), Date.now()) + EXECUTION_RECONCILIATION_INTERVAL_MS);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization);
|
||||
await tx.update(nativeRunFinalizations).set({ controlDeadlineAt: cleanupRetryAt }).where(eq(nativeRunFinalizations.runId, candidate.runId));
|
||||
await tx.update(heartbeatRuns).set({ executionControlDeadlineAt: cleanupRetryAt }).where(and(eq(heartbeatRuns.id, candidate.runId),
|
||||
isNotNull(heartbeatRuns.executionControlDeadlineAt), lte(heartbeatRuns.executionControlDeadlineAt, now)));
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Set inside the transaction only when the write below genuinely
|
||||
// transitions the run into "failed". Read after the transaction
|
||||
// commits, so a rolled-back write never reports a false failure.
|
||||
@@ -128,6 +163,18 @@ export async function reconcileAbandonedExecutionControl(
|
||||
].some((deadline) => deadline && deadline <= now)
|
||||
)
|
||||
return false;
|
||||
if (coordinator?.failureCode === NATIVE_STARTUP_CLEANUP_PENDING) return false;
|
||||
const startupContext = coordinator && readNativeStartupContext(run, coordinator);
|
||||
if (startupContext) {
|
||||
if (!await promoteNativeStartupCleanup(tx as unknown as Db, run, coordinator!, now)) {
|
||||
const retryAt = new Date(Math.max(now.getTime() + EXECUTION_RECONCILIATION_INTERVAL_MS,
|
||||
startupContext.deadlineAtMs, coordinator!.leaseExpiresAt?.getTime() ?? 0));
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization);
|
||||
await tx.update(nativeRunFinalizations).set({ controlDeadlineAt: retryAt }).where(eq(nativeRunFinalizations.runId, run.id));
|
||||
await tx.update(heartbeatRuns).set({ executionControlDeadlineAt: retryAt }).where(eq(heartbeatRuns.id, run.id));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
await tx
|
||||
.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: null })
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { withDatabaseOperation } from "@paperclipai/db";
|
||||
import { EXECUTION_PREPARATION_SPAN_ATTRS, getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
||||
import { createHash } from "node:crypto";
|
||||
import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context } from "@opentelemetry/api";
|
||||
@@ -156,7 +157,7 @@ function measure<T>(runId: string, name: string, work: () => T | Promise<T>, ind
|
||||
ended: false,
|
||||
details: !independent && previous?.runId === runId ? previous.details : new Map<string, number>(),
|
||||
};
|
||||
operation = Promise.resolve().then(() =>
|
||||
operation = withDatabaseOperation(name, () => Promise.resolve().then(() =>
|
||||
ownership.run(owned, () => runWithRuntimeParent(owned.parent, work))).then(value => {
|
||||
try { span?.end(); } catch {}
|
||||
owned.ended = true;
|
||||
@@ -166,10 +167,10 @@ function measure<T>(runId: string, name: string, work: () => T | Promise<T>, ind
|
||||
try { span?.end(); } catch {}
|
||||
owned.ended = true;
|
||||
throw error;
|
||||
});
|
||||
}));
|
||||
return operation;
|
||||
};
|
||||
if (!endpointEnabled()) return (async () => work())();
|
||||
if (!endpointEnabled()) return withDatabaseOperation(name, async () => work());
|
||||
try {
|
||||
const parent = independent ? undefined : activeParent ?? getRunExecutionTraceContext(runId);
|
||||
const trigger = parent ? undefined : trace.getActiveSpan()?.spanContext();
|
||||
@@ -202,20 +203,22 @@ export function withRunWorkspaceFinalizationTiming<T>(runId: string, work: () =>
|
||||
export function withRunResponsePresentationTiming<T>(runId: string, work: () => T | Promise<T>): Promise<T> {
|
||||
const owned = ownership.getStore();
|
||||
if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) {
|
||||
return (async () => work())();
|
||||
return withDatabaseOperation("task.response_presentation", async () => work());
|
||||
}
|
||||
return measure(runId, "task.response_presentation", work, true);
|
||||
}
|
||||
|
||||
export async function measureRunPhase<T>(phase: ExecutionPhase, work: () => T | Promise<T>, counts?: { fileCount?: number; bytes?: number }): Promise<T> {
|
||||
const owned = ownership.getStore();
|
||||
if (!endpointEnabled() || !owned || owned.ended) return work();
|
||||
const count = owned.details.get(phase) ?? 0;
|
||||
if (count >= 16) return work();
|
||||
owned.details.set(phase, count + 1);
|
||||
// A startup step can be more specific than the owning run phase. Keep that
|
||||
// causal parent while the child publishes its own context to plugin workers.
|
||||
return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined);
|
||||
export function measureRunPhase<T>(phase: ExecutionPhase, work: () => T | Promise<T>, counts?: { fileCount?: number; bytes?: number }): Promise<T> {
|
||||
return withDatabaseOperation(phase, async () => {
|
||||
const owned = ownership.getStore();
|
||||
if (!endpointEnabled() || !owned || owned.ended) return work();
|
||||
const count = owned.details.get(phase) ?? 0;
|
||||
if (count >= 16) return work();
|
||||
owned.details.set(phase, count + 1);
|
||||
// A startup step can be more specific than the owning run phase. Keep that
|
||||
// causal parent while the child publishes its own context to plugin workers.
|
||||
return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined);
|
||||
});
|
||||
}
|
||||
|
||||
export function beginRunPhase(phase: ExecutionPhase): (failed?: boolean) => void {
|
||||
|
||||
@@ -52,7 +52,7 @@ const preflightProfile = sql<Run["runnerProfileJson"]>`case
|
||||
then jsonb_build_object('adapterType', field.value->'adapterType') else field.value end), '{}'::jsonb)
|
||||
from jsonb_each(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object'
|
||||
then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) as field(key, value)
|
||||
where field.key in ('adapterDispatch', 'nativeWorkspaceSync')
|
||||
where field.key in ('adapterDispatch', 'nativeWorkspaceSync', 'remoteWorkspace')
|
||||
) end`;
|
||||
const continuationRunFields = {
|
||||
id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
|
||||
|
||||
@@ -398,6 +398,7 @@ import {
|
||||
isNativeRunnerOwnershipHeld,
|
||||
NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,
|
||||
} from "./native-runtime/native-runner-ownership.js";
|
||||
import { NativeStartupCleanupPendingError, requestNativeStartupCleanup, verifyNativeStartupCleanup, readNativeStartupCleanup } from "./native-runtime/native-startup-cleanup.js";
|
||||
import {
|
||||
emitAgentTaskRun,
|
||||
emitAgentTaskRunById,
|
||||
@@ -1436,6 +1437,14 @@ export function heartbeatService(
|
||||
}
|
||||
|
||||
const runControl = createHeartbeatRunControl(db, {
|
||||
verifyStartupCleanup: async run => {
|
||||
await verifyNativeStartupCleanup(db, run, () => envOrchestrator.releaseForRun({
|
||||
heartbeatRunId: run.id, companyId: run.companyId, agentId: run.agentId, status: "expired",
|
||||
cancelActiveWork: true, providerResourceDisposition: "destroy",
|
||||
}));
|
||||
const marker = readNativeStartupCleanup(run)!;
|
||||
await options.pluginWorkerManager?.retireStartup?.(marker.contextId);
|
||||
},
|
||||
getRun,
|
||||
getRunCleanupState,
|
||||
getRunExecutionWait,
|
||||
@@ -1759,7 +1768,7 @@ export function heartbeatService(
|
||||
},
|
||||
});
|
||||
})().catch((error) => {
|
||||
if (!nativeRecoveryDeadlines.has(runId)) {
|
||||
if (!nativeRecoveryDeadlines.has(runId) || nativeRecoveryDeadlines.get(runId) === pending) {
|
||||
scheduleNativeSessionResumeDispatch(runId, new Date(Date.now() + 30_000), mode);
|
||||
}
|
||||
logger.error(
|
||||
@@ -1956,6 +1965,19 @@ export function heartbeatService(
|
||||
Parameters<typeof cleanupGitHubOperationLaunchers>[0] | null = null;
|
||||
let nativeSessionResumeScheduled = false;
|
||||
let nativeOwnershipHeld = false;
|
||||
let nativeStartupCleanupScheduled = false;
|
||||
let nativeStartupCleanupError: NativeStartupCleanupPendingError | undefined;
|
||||
const requestStartupCleanup = async (error: NativeStartupCleanupPendingError) => {
|
||||
nativeStartupCleanupError = error;
|
||||
try {
|
||||
nativeStartupCleanupScheduled = await requestNativeStartupCleanup(db, { companyId: run.companyId, agentId: run.agentId, runId: run.id,
|
||||
contextId: error.contextId, deadlineAtMs: error.deadlineAtMs, leaseOwner: error.leaseOwner,
|
||||
leaseId: error.leaseId, requiresPreparationStop: error.requiresPreparationStop });
|
||||
} catch (writeError) {
|
||||
scheduleNativeSessionResumeDispatch(run.id, new Date(Date.now() + 30_000), "restart");
|
||||
throw writeError;
|
||||
}
|
||||
};
|
||||
let instructionWorkingCopy: ReturnType<typeof createHeartbeatInstructionWorkingCopy> | null = null;
|
||||
let nativeDispatchStarted = false;
|
||||
let nativeWorkspaceFinalizeScheduled = false;
|
||||
@@ -3527,6 +3549,10 @@ export function heartbeatService(
|
||||
}));
|
||||
} catch (err) {
|
||||
await persistUsageCaptureFailure?.();
|
||||
if (err instanceof NativeStartupCleanupPendingError) {
|
||||
await requestStartupCleanup(err);
|
||||
return;
|
||||
}
|
||||
if (err instanceof NativeControllerDetachedForRestartError) {
|
||||
nativeSessionResumeScheduled = true;
|
||||
return;
|
||||
@@ -3659,7 +3685,12 @@ export function heartbeatService(
|
||||
});
|
||||
}
|
||||
} catch (outerErr) {
|
||||
if (
|
||||
if (nativeStartupCleanupScheduled || nativeStartupCleanupError || outerErr instanceof NativeStartupCleanupPendingError) {
|
||||
const pending = outerErr instanceof NativeStartupCleanupPendingError ? outerErr : nativeStartupCleanupError;
|
||||
if (!nativeStartupCleanupScheduled && pending) await requestStartupCleanup(pending).catch(error => {
|
||||
logger.warn({ err: error, runId: run.id }, "Native startup cleanup intent will retry through owned recovery");
|
||||
});
|
||||
} else if (
|
||||
nativeOwnershipHeld ||
|
||||
outerErr instanceof NativeRunnerOwnershipUnverifiedError
|
||||
) {
|
||||
@@ -3696,7 +3727,7 @@ export function heartbeatService(
|
||||
);
|
||||
} else {
|
||||
await preserveNativeWorkspaceOnFailure();
|
||||
if (!nativeOwnershipHeld) await runCompletion.failRunSetup({
|
||||
if (!nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError) await runCompletion.failRunSetup({
|
||||
run, runId, outerErr, signal: executionControl.controller.signal,
|
||||
providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted,
|
||||
identityRedactor, readFailureReportSecrets,
|
||||
@@ -3746,7 +3777,7 @@ export function heartbeatService(
|
||||
latestRun &&
|
||||
!nativeSessionResumeScheduled &&
|
||||
!nativeWorkspaceFinalizeScheduled &&
|
||||
!nativeOwnershipHeld
|
||||
!nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError
|
||||
) {
|
||||
latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch(
|
||||
(terminalizeErr) => {
|
||||
@@ -3772,7 +3803,7 @@ export function heartbeatService(
|
||||
if (
|
||||
!nativeSessionResumeScheduled &&
|
||||
!nativeWorkspaceFinalizeScheduled &&
|
||||
!nativeOwnershipHeld
|
||||
!nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError
|
||||
) {
|
||||
// Keep launchers during same-run recovery. At a terminal boundary all
|
||||
// operations have settled; clean before the remote lease can be stopped.
|
||||
@@ -3895,7 +3926,7 @@ export function heartbeatService(
|
||||
}).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "cancelled")));
|
||||
}
|
||||
if (latestRun?.runtimeMode === "native" && isHeartbeatRunTerminalStatus(latestRun.status) &&
|
||||
!nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled && !nativeOwnershipHeld) {
|
||||
!nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled && !nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError) {
|
||||
await settleNativeRunProcess(db, run.id).catch(error => {
|
||||
logger.warn({ err: error, runId: run.id }, "failed to release settled native process identity");
|
||||
});
|
||||
@@ -3932,7 +3963,7 @@ export function heartbeatService(
|
||||
caseHandoffCleanupSettled = hasStoppedLocalProcessTree(latestRun);
|
||||
}
|
||||
if (caseHandoffCleanupSettled && latestRun && !nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled &&
|
||||
!nativeOwnershipHeld && isHeartbeatRunTerminalStatus(latestRun.status)) {
|
||||
!nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError && isHeartbeatRunTerminalStatus(latestRun.status)) {
|
||||
const handoff = await pendingCaseHandoffForRun(db, run.id);
|
||||
if (handoff) await recordCaseHandoffExecutionReleased(db, run.id, handoff.issueId);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
|
||||
@@ -31,7 +31,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read
|
||||
contextSnapshot: { savedInput: bulk }, runnerProfileJson: { savedInput: bulk }, stdoutExcerpt: bulk,
|
||||
stderrExcerpt: bulk, resultJson: result }).returning();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let evidence: Awaited<ReturnType<typeof readNativeStopExecutionEvidence>>;
|
||||
try { evidence = await readNativeStopExecutionEvidence(db, company.id, id); }
|
||||
@@ -68,7 +68,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read
|
||||
runtimeMode: "native", status: "cancelled" }).returning();
|
||||
await db.execute(sql`update ${heartbeatRuns} set result_json = ${JSON.stringify(saved)}::jsonb where id = ${run.id}`);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const evidence = await readNativeStopExecutionEvidence(db, company.id, run.id);
|
||||
|
||||
@@ -2299,7 +2299,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId ?? ""}`,
|
||||
));
|
||||
const retry = readTaskRetryIntent(receipt?.payload);
|
||||
if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId ||
|
||||
if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || retry.queuedChatRequestId ||
|
||||
retry.actorId !== receipt.requestedByActorId) return null;
|
||||
const chatRetry = retry.chatActionId && durableRequest?.id === retry.chatActionId &&
|
||||
durableRequest.failedRunRetry?.failedRunId === retry.sourceRunId;
|
||||
@@ -2518,6 +2518,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// retain distinct durable receipts even when the same gate blocks them.
|
||||
const coalesceExecutionWait =
|
||||
opts.requestedByActorType === "system" &&
|
||||
opts.allowRunCoalescing !== false &&
|
||||
!receiptRequest &&
|
||||
!wakeCommentId &&
|
||||
queuedCommentIdsFromRunContext(enrichedContextSnapshot).length === 0 &&
|
||||
@@ -4369,7 +4370,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)),
|
||||
)).then(rows => rows[0] ?? null)
|
||||
: null;
|
||||
const canCoalesceComments = !isConversation(issue) && opts.allowRunCoalescing !== false;
|
||||
// External input owns an exact authorized source batch. Board input
|
||||
// must retain its separate receipt, including during normal resume.
|
||||
const canCoalesceComments = !durableRequest && !isConversation(issue) && opts.allowRunCoalescing !== false;
|
||||
// A resumed receipt already passed admission under this issue lock.
|
||||
// Consume it with its successor even when chat keeps other messages
|
||||
// in separate turns, or finalization will deliver it a second time.
|
||||
|
||||
@@ -168,13 +168,13 @@ describe("heartbeat recovery module callbacks", () => {
|
||||
expect(database.access).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("persists an authentication ownership hold before its event without terminating or retrying", async () => {
|
||||
it.each(["adopted_runner_authentication_timeout"] as const)("persists a %s ownership hold before its event without terminating or retrying", async reason => {
|
||||
const database = guardedDatabase();
|
||||
const deps = callbacks(database.db);
|
||||
const run = { id: "run", runtimeMode: "native", status: "running" } as Run;
|
||||
const blocked = { ...run, nativePhase: "terminal_failure", errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE };
|
||||
deps.setRunStatusFromLive.mockResolvedValue({ updated: true, run: blocked });
|
||||
expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason: "adopted_runner_authentication_timeout" })).toBe(blocked);
|
||||
expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason })).toBe(blocked);
|
||||
expect(deps.setRunStatusFromLive).toHaveBeenCalledWith("run", "running", ["running"], expect.objectContaining({ nativePhase: "terminal_failure" }));
|
||||
expect(deps.appendRunEvent).toHaveBeenCalledAfter(deps.setRunStatusFromLive);
|
||||
expect(deps.cancelHeartbeatNativeRun).not.toHaveBeenCalled();
|
||||
@@ -207,6 +207,42 @@ describePostgres("heartbeat recovery module ownership", () => {
|
||||
return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0];
|
||||
}
|
||||
|
||||
it("routes an expired pre-capability startup to cleanup before generic orphan ownership holds", async () => {
|
||||
const run = await seedRun(), leaseId = randomUUID(), environmentId = randomUUID(), leaseOwner = randomUUID();
|
||||
const deadlineAtMs = Date.now() - 1_000, contextId = randomUUID();
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, run.agentId));
|
||||
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, run.issueId!));
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, nativePhase: "observed",
|
||||
runnerInstanceId: randomUUID(), nativeSessionId: randomUUID(), runnerProfileJson: { nativeStartupContext: {
|
||||
schema: "paperclip.native-startup-context.v1", companyId: run.companyId, agentId: run.agentId, runId: run.id,
|
||||
contextId, deadlineAtMs, leaseId, leaseOwner, controllerGeneration: 1, attempt: 1 } } }).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!,
|
||||
phase: "observed", attempt: 1, leaseOwner, controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() - 1_000),
|
||||
controllerBootId: randomUUID(), controllerPid: process.pid, controllerProcessStartedAt: new Date() });
|
||||
await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "plugin", config: {} });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId: run.companyId, environmentId, heartbeatRunId: run.id,
|
||||
issueId: run.issueId, provider: "plugin", providerLeaseId: "early-task", status: "active", metadata: { runnerTaskLease: true } });
|
||||
const deps = callbacks(db);
|
||||
deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null);
|
||||
native.claims.mockImplementationOnce(claimNativeRestartRecoveries);
|
||||
try {
|
||||
await createHeartbeatRecovery(db, deps).recoverActiveExecutions({ runId: run.id });
|
||||
expect(native.claims).toHaveBeenCalledWith(expect.objectContaining({ runIds: [run.id] }));
|
||||
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(current).toMatchObject({ status: "running", errorCode: "native_startup_cleanup_pending",
|
||||
resultJson: { nativeStartupCleanup: { contextId, state: "pending" } } });
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await db.delete(environmentLeases).where(eq(environmentLeases.id, leaseId));
|
||||
await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));
|
||||
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id));
|
||||
await db.delete(activityLog).where(eq(activityLog.runId, run.id));
|
||||
await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, run.issueId!));
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
}
|
||||
});
|
||||
|
||||
async function ownedProcess() {
|
||||
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", `paperclip-recovery-${randomUUID()}`], {
|
||||
stdio: "ignore", detached: true, env: { PATH: process.env.PATH },
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { readNativeStartupCleanup, readNativeStartupContext } from "../native-runtime/native-startup-cleanup.js";
|
||||
import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js";
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { registerLiveRecoveryWork } from "./live-recovery-work.js";
|
||||
@@ -1038,7 +1039,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
? "Native restart recovery is waiting for safe ownership evidence"
|
||||
: disposition.kind === "already_finalized"
|
||||
? "Native restart recovery found an already-finalized result"
|
||||
: "Native restart recovery blocked ambiguous or conflicting ownership",
|
||||
: disposition.reason === "native_startup_cleanup_pending"
|
||||
? "Native startup timed out; cancellation and verified cleanup are queued"
|
||||
: "Native restart recovery blocked ambiguous or conflicting ownership",
|
||||
payload: {
|
||||
restartKind: isClaim ? disposition.restartKind : restartKind,
|
||||
recoveryRequestId: isClaim
|
||||
@@ -2251,6 +2254,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
nativeControllerProcessStartedAt:
|
||||
nativeRunFinalizations.controllerProcessStartedAt,
|
||||
nativeControllerLeaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,
|
||||
nativeLeaseOwner: nativeRunFinalizations.leaseOwner,
|
||||
nativeControllerGeneration: nativeRunFinalizations.controllerGeneration,
|
||||
nativeAttempt: nativeRunFinalizations.attempt,
|
||||
nativeResultId: nativeRunFinalizations.resultId,
|
||||
nativeWorkspaceFinalizationPending: pendingNativeWorkspaceFinalizationCondition(db),
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
@@ -2271,7 +2278,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
&& !isNativeRunnerOwnershipHeld(run)
|
||||
&& !activeRunExecutions.has(run.id) && !runningProcesses.has(run.id),
|
||||
).map(({ run }) => run.id);
|
||||
const remoteRecoveryRunIds = new Set(await listExpiredNativeRunnerAuthorities(db, now, expiredNativeRunIds));
|
||||
const earlyStartupRunIds = activeRuns.filter(row => expiredNativeRunIds.includes(row.run.id) &&
|
||||
readNativeStartupContext(row.run, { leaseOwner: row.nativeLeaseOwner, controllerGeneration: row.nativeControllerGeneration ?? 0,
|
||||
attempt: row.nativeAttempt ?? 0, resultId: row.nativeResultId })).map(row => row.run.id);
|
||||
const remoteRecoveryRunIds = new Set([...earlyStartupRunIds, ...await listExpiredNativeRunnerAuthorities(db, now, expiredNativeRunIds)]);
|
||||
if (remoteRecoveryRunIds.size) {
|
||||
const recovery = await recoverNativeRunsAfterRestart(now, [...remoteRecoveryRunIds]);
|
||||
for (const claim of recovery.claims) resumedRunIds.add(claim.runId);
|
||||
@@ -2321,6 +2331,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
nativeControllerLeaseExpiresAt,
|
||||
nativeWorkspaceFinalizationPending,
|
||||
} of activeRuns) {
|
||||
if (run.runtimeMode === "native" && readNativeStartupCleanup(run)) continue;
|
||||
if (parseObject(run.contextSnapshot).nativeQuestionCancellation) continue;
|
||||
// Native recovery owns these dispositions. Its writes happened after this
|
||||
// inventory was read; do not reap or overwrite them using the old row.
|
||||
|
||||
@@ -123,7 +123,8 @@ import {
|
||||
} from "../native-runtime/index.js";
|
||||
import { runningProcesses } from "../../adapters/index.js";
|
||||
import { parseObject } from "../../adapters/utils.js";
|
||||
import { isNativeRunnerOwnershipHeld } from "../native-runtime/native-runner-ownership.js";
|
||||
import { isNativeRunnerOwnershipHeld, nativeRunnerOwnershipNotHeldCondition } from "../native-runtime/native-runner-ownership.js";
|
||||
import { readNativeStartupCleanup, acknowledgeNativeStartupCleanup } from "../native-runtime/native-startup-cleanup.js";
|
||||
import {
|
||||
withCurrentBudgetEnforcement,
|
||||
type BudgetEnforcementScope,
|
||||
@@ -177,6 +178,7 @@ export interface HeartbeatRunControlDependencies extends Pick<ReturnType<typeof
|
||||
Pick<ReturnType<typeof createHeartbeatLifecycle>,
|
||||
"mergeRunStopMetadataForAgent" | "setRunStatusFromLive" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent">,
|
||||
Pick<HeartbeatRetryDependencies, "finalizeAgentStatus"> {
|
||||
verifyStartupCleanup?: (run: typeof heartbeatRuns.$inferSelect) => Promise<void>;
|
||||
options: {
|
||||
closeWarmNativeSessionsForRun?: (input: {
|
||||
runId: string;
|
||||
@@ -763,6 +765,10 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
eq(agentWakeupRequests.requestedByActorType, "user"),
|
||||
));
|
||||
const intent = readTaskRetryIntent(wake?.payload);
|
||||
if (intent?.queuedChatRequestId) {
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId, requestId });
|
||||
return;
|
||||
}
|
||||
if (!wake || !intent || intent.chatActionId || intent.canonicalIntentId || intent.generationId !== wake.id ||
|
||||
intent.actorId !== wake.requestedByActorId) return;
|
||||
const issueId = readNonEmptyString(wake.payload?.issueId);
|
||||
@@ -788,6 +794,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
|
||||
sql`${agentWakeupRequests.payload}->'taskRetry'->>'queuedChatRequestId' is null`,
|
||||
notInArray(issues.status, ["done", "cancelled"])))
|
||||
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
|
||||
const chatEnabled = !retries.some(retry => retry.conversationAgentId)
|
||||
@@ -1394,13 +1401,19 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
}
|
||||
let terminationSettled = false;
|
||||
try {
|
||||
await cancelHeartbeatNativeRun({
|
||||
db,
|
||||
runId: run.id,
|
||||
reason,
|
||||
runtimeMode: run.runtimeMode,
|
||||
...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}),
|
||||
});
|
||||
const startupCleanupRun = run.runtimeMode === "native" ? await getRun(run.id) : null;
|
||||
if (startupCleanupRun && readNativeStartupCleanup(startupCleanupRun) && !control && !running && !activeRunExecutions.has(run.id)) {
|
||||
if (!dependencies.verifyStartupCleanup) throw conflict("Native startup cleanup worker unavailable");
|
||||
await dependencies.verifyStartupCleanup(startupCleanupRun);
|
||||
} else {
|
||||
await cancelHeartbeatNativeRun({
|
||||
db,
|
||||
runId: run.id,
|
||||
reason,
|
||||
runtimeMode: run.runtimeMode,
|
||||
...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}),
|
||||
});
|
||||
}
|
||||
if (running) {
|
||||
await terminateHeartbeatRunProcess({
|
||||
pid: running.child.pid,
|
||||
@@ -1461,12 +1474,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
}
|
||||
|
||||
const finishedAt = new Date();
|
||||
const persistedCancellationResult =
|
||||
run.runtimeMode === "native"
|
||||
? await getRun(run.id).then((current) =>
|
||||
parseObject(current?.resultJson),
|
||||
)
|
||||
: {};
|
||||
let persistedCancellationRun = run.runtimeMode === "native" ? await getRun(run.id) : null;
|
||||
if (persistedCancellationRun && readNativeStartupCleanup(persistedCancellationRun)) {
|
||||
if (!dependencies.verifyStartupCleanup) throw conflict("Native startup cleanup worker unavailable");
|
||||
await dependencies.verifyStartupCleanup(persistedCancellationRun);
|
||||
persistedCancellationRun = await getRun(run.id);
|
||||
}
|
||||
const persistedCancellationResult = parseObject(persistedCancellationRun?.resultJson);
|
||||
return await setRunStatusFromLive(
|
||||
run.id,
|
||||
"cancelled",
|
||||
@@ -1515,12 +1529,18 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
persistedCancellationResult.nativeCancellation,
|
||||
}
|
||||
: {}),
|
||||
...(Object.hasOwn(persistedCancellationResult, "nativeStartupCleanup")
|
||||
? { nativeStartupCleanup: persistedCancellationResult.nativeStartupCleanup } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
undefined,
|
||||
pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined,
|
||||
and(pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult)
|
||||
: persistedCancellationRun && readNativeStartupCleanup(persistedCancellationRun) ? and(nativeRunnerOwnershipNotHeldCondition(),
|
||||
sql`not exists (select 1 from ${nativeRunFinalizations} where ${nativeRunFinalizations.runId} = ${run.id} and ${nativeRunFinalizations.resultId} is not null)`)
|
||||
: undefined,
|
||||
run.runtimeMode === "native" ? sql`coalesce(${heartbeatRuns.resultJson}->'nativeStartupCleanup'->>'state', '') <> 'pending'` : undefined),
|
||||
);
|
||||
} catch (error) {
|
||||
if (processCancellationSettlement) {
|
||||
@@ -1552,6 +1572,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
const cancelled = cancellation.run;
|
||||
|
||||
if (cancellation.updated && cancelled) {
|
||||
if (readNativeStartupCleanup(cancelled)) await acknowledgeNativeStartupCleanup(db, { companyId: cancelled.companyId, runId: cancelled.id });
|
||||
await setWakeupStatus(run.wakeupRequestId, "cancelled", {
|
||||
finishedAt: cancelled.finishedAt ?? new Date(),
|
||||
error: reason,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { readNativeStartupCleanup } from "../native-runtime/native-startup-cleanup.js";
|
||||
import { notifyStrandedAgentStatus } from "../live-run-recovery-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
|
||||
@@ -1159,6 +1160,9 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const previousStatus = previous?.run ?? null;
|
||||
if (status === "failed" && patch?.errorCode === "process_lost" && previousStatus?.runtimeMode === "native" && readNativeStartupCleanup(previousStatus)) {
|
||||
return { updated: false, run: previousStatus };
|
||||
}
|
||||
|
||||
if (previousStatus && (status === "cancelled" || status === "interrupted")) {
|
||||
patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
@@ -186,13 +186,13 @@ describePostgres("heartbeat run state database wiring", () => {
|
||||
stdoutExcerpt: payload, stderrExcerpt: payload,
|
||||
}).returning();
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const state = extracted.createHeartbeatRunState(observed);
|
||||
const read = await state.getRunExecutionWait(companyId, run.id);
|
||||
capture.flush();
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(2);
|
||||
expect(summary.responseValueBytes).toBeLessThan(4096);
|
||||
expect(read).toMatchObject({ id: run.id, companyId, agentId: agent.id, status: "cancelled", contextSnapshot: context,
|
||||
@@ -228,7 +228,7 @@ describePostgres("heartbeat run state database wiring", () => {
|
||||
resultJson: { stdout: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
|
||||
}).returning();
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const state = extracted.createHeartbeatRunState(observed);
|
||||
@@ -236,7 +236,7 @@ describePostgres("heartbeat run state database wiring", () => {
|
||||
id: run.id, companyId, scopeKind: "issue", issueId: issue.id, executionWorkspaceId: workspaceId,
|
||||
});
|
||||
capture.flush();
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(1);
|
||||
expect(summary.responseValueBytes).toBeLessThan(1024);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
|
||||
@@ -260,7 +260,7 @@ describePostgres("heartbeat run state database wiring", () => {
|
||||
}).returning();
|
||||
const observed = createDb(database.connectionString);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const state = extracted.createHeartbeatRunState(observed);
|
||||
@@ -270,7 +270,7 @@ describePostgres("heartbeat run state database wiring", () => {
|
||||
runtimeModeResolvedAt: null, startedAt: run.startedAt, createdAt: run.createdAt,
|
||||
contextSnapshot: { issueId: issue.id }, runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } } });
|
||||
capture.flush();
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(2);
|
||||
expect(summary.responseValueBytes).toBeLessThan(2048);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
|
||||
@@ -321,7 +321,7 @@ describePostgres("heartbeat run state database wiring", () => {
|
||||
await state.getRunWorkspaceSettlement(run.id);
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
expect(await state.getRunCleanupState(run.id)).toEqual({
|
||||
id: run.id, companyId, agentId: agent.id, runtimeMode: run.runtimeMode,
|
||||
@@ -334,7 +334,7 @@ describePostgres("heartbeat run state database wiring", () => {
|
||||
contextSnapshot: { timerClaimWasFirstHeartbeat: true },
|
||||
});
|
||||
capture.flush();
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(2);
|
||||
expect(summary.responseValueBytes).toBeLessThan(2000);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { NativeStartupCleanupPendingError } from "../native-runtime/native-startup-cleanup.js";
|
||||
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
|
||||
|
||||
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
|
||||
@@ -1072,15 +1073,21 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
&& !(nativeExecution && "access" in nativeExecution.workspace && nativeExecution.workspace.access === "none")) {
|
||||
input.effects.onProviderResourceDisposition("stop_and_retain");
|
||||
}
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx
|
||||
.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.id, run.id),
|
||||
eq(heartbeatRuns.status, "running"),
|
||||
),
|
||||
), { companyId: run.companyId });
|
||||
try {
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx
|
||||
.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.id, run.id),
|
||||
eq(heartbeatRuns.status, "running"),
|
||||
),
|
||||
), { companyId: run.companyId });
|
||||
} catch (writeError) {
|
||||
if (adapterErr instanceof NativeStartupCleanupPendingError) throw adapterErr;
|
||||
throw writeError;
|
||||
}
|
||||
if (adapterErr instanceof NativeStartupCleanupPendingError) throw adapterErr;
|
||||
if (
|
||||
issueRef &&
|
||||
context.resumeSessionGoalHeartbeat === true &&
|
||||
|
||||
@@ -7,7 +7,7 @@ import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { captureDirectorySnapshot, directorySnapshotSha256, serializeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, nativeRunFinalizations, plugins, projects, projectWorkspaces, type Db } from "@paperclipai/db";
|
||||
import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, nativeRunFinalizations, plugins, projects, projectWorkspaces, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
@@ -170,15 +170,16 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ kind: "empty", repoUrl: null, branchMode: null, fresh: false, ephemeral: true },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: null, fresh: false },
|
||||
{ kind: "git", repoUrl: "ssh://git@github.com/example/remote.git", branchMode: null, fresh: false },
|
||||
{ kind: "git", repoUrl: "git@github.com:example/remote.git", branchMode: null, fresh: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "existing", fresh: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true },
|
||||
] as const)("reserves a $kind remote workspace ($repoUrl, branch: $branchMode, fresh: $fresh) without materializing a controller checkout", async (scenario) => {
|
||||
const { kind, repoUrl, branchMode, fresh } = scenario;
|
||||
{ kind: "empty", repoUrl: null, branchMode: null, fresh: false, strategyChange: false, ephemeral: true },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: null, fresh: false, strategyChange: false },
|
||||
{ kind: "git", repoUrl: "ssh://git@github.com/example/remote.git", branchMode: null, fresh: false, strategyChange: false },
|
||||
{ kind: "git", repoUrl: "git@github.com:example/remote.git", branchMode: null, fresh: false, strategyChange: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "existing", fresh: false, strategyChange: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: false, strategyChange: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true, strategyChange: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true, strategyChange: true },
|
||||
] as const)("reserves a $kind remote workspace ($repoUrl, branch: $branchMode, fresh: $fresh, strategy change: $strategyChange) without materializing a controller checkout", async (scenario) => {
|
||||
const { kind, repoUrl, branchMode, fresh, strategyChange } = scenario;
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
const [savedEnvironment] = await db.insert(environments).values({ name: `Remote ${randomUUID()}`,
|
||||
driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning();
|
||||
@@ -194,7 +195,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
if (branchMode) {
|
||||
input.policy.requestedExecutionWorkspaceMode = "isolated_workspace";
|
||||
input.policy.effectiveExecutionWorkspaceMode = "isolated_workspace";
|
||||
input.config.mergedConfig = { workspaceStrategy: { type: "git_worktree", baseRef: "main",
|
||||
input.config.mergedConfig = strategyChange ? { workspaceStrategy: { type: "project_primary" } } : { workspaceStrategy: { type: "git_worktree", baseRef: "main",
|
||||
...(branchMode === "existing" ? { existingBranch: "feature/diverged" } : { branchTemplate: "test-{{issue.identifier}}" }) } };
|
||||
}
|
||||
if (kind === "git") {
|
||||
@@ -243,9 +244,44 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
expect(workspace.mode).toBe(branchMode ? "isolated_workspace" : "shared_workspace");
|
||||
expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: { kind }, writerRunId: input.run.id });
|
||||
if (branchMode) expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: {
|
||||
ref: "main", branchMode, branchName: branchMode === "existing" ? "feature/diverged" : "test-WSP-1",
|
||||
ref: "main", branchMode,
|
||||
...(strategyChange ? {} : { branchName: branchMode === "existing" ? "feature/diverged" : "test-WSP-1" }),
|
||||
} });
|
||||
await expect(access(workspace.cwd!)).rejects.toThrow();
|
||||
if ((fresh || strategyChange) && branchMode === "create") {
|
||||
expect(workspace.strategyType).toBe("git_worktree");
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, input.run.id));
|
||||
const [saved] = await db.update(executionWorkspaces).set({ metadata: {
|
||||
...workspace.metadata,
|
||||
remoteWorkspace: { ...workspace.metadata!.remoteWorkspace as object, writerRunId: null, dirty: false },
|
||||
} }).where(eq(executionWorkspaces.id, workspace.id)).returning();
|
||||
const [nextRun] = await db.insert(heartbeatRuns).values({ companyId: issue.companyId,
|
||||
agentId: input.agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning();
|
||||
input.run = nextRun;
|
||||
input.task.freshWorkspace = null;
|
||||
input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: saved.id, executionWorkspacePreference: "reuse_existing" };
|
||||
input.reuse.existingExecutionWorkspace = toExecutionWorkspace(saved);
|
||||
input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(saved);
|
||||
input.reuse.requestedShouldReuseExisting = true;
|
||||
input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({
|
||||
issueExecutionWorkspaceId: saved.id, issueExecutionWorkspacePreference: "reuse_existing", existingExecutionWorkspaceStatus: "active",
|
||||
});
|
||||
if (strategyChange) input.config.mergedConfig = { workspaceStrategy: { type: "git_worktree" } };
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached remote acquisition");
|
||||
const operations = await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, nextRun.id));
|
||||
const freshness = operations.filter(operation => operation.phase === "workspace_config_freshness");
|
||||
if (strategyChange) {
|
||||
expect(freshness).toHaveLength(1);
|
||||
expect(freshness[0].metadata).toMatchObject({ action: "replace", changedCategories: ["strategy"] });
|
||||
} else expect(freshness).toEqual([]);
|
||||
const [retained] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, saved.id));
|
||||
expect(retained.id).toBe(workspace.id);
|
||||
expect(retained.metadata?.configFingerprint).toMatchObject({
|
||||
workspaceHash: (saved.metadata?.configFingerprint as { workspaceHash: string }).workspaceHash,
|
||||
});
|
||||
expect(retained.metadata?.remoteWorkspace).toMatchObject({ writerRunId: nextRun.id });
|
||||
}
|
||||
} finally {
|
||||
for (const [key, value] of previous) {
|
||||
if (value === undefined) delete process.env[key]; else process.env[key] = value;
|
||||
|
||||
@@ -543,7 +543,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
const skipHostProvision = copyOnlyProvisioning || hostProvisionDriver === "sandbox" || hostProvisionDriver === "ssh" || hostProvisionDriver === "plugin";
|
||||
const hostExecutionWorkspaceConfig = isDotRun ? {} :
|
||||
stripHostWorkspaceProvision({
|
||||
config: fresh ? { ...mergedConfig, workspaceStrategy: { type: "project_primary" } } : mergedConfig,
|
||||
config: fresh && !remoteAuthority ? { ...mergedConfig, workspaceStrategy: { type: "project_primary" } } : mergedConfig,
|
||||
skipHostProvision,
|
||||
});
|
||||
const executionWorkspaceBase = {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { sql, type SQL } from "drizzle-orm";
|
||||
import { issues } from "@paperclipai/db";
|
||||
import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME, type IssueDisplayStatus } from "@paperclipai/shared";
|
||||
|
||||
/** Read-only projection, shared by task reads, search, filters and counts.
|
||||
* The persisted deadline is the scheduler's source of truth. Locks and recovery
|
||||
* take precedence; a retained monitor alone must not disguise active work.
|
||||
*/
|
||||
export function issueDisplayStatusSql(): SQL<IssueDisplayStatus> {
|
||||
// Drizzle removes column qualifiers in a single-table SELECT. Keep these
|
||||
// outer references explicitly qualified so correlated reads cannot bind to
|
||||
// the run/recovery table's own id or company_id instead.
|
||||
const issueId = sql`"issues"."id"`;
|
||||
const companyId = sql`"issues"."company_id"`;
|
||||
return sql<IssueDisplayStatus>`CASE WHEN
|
||||
${issues.status} = 'in_progress'
|
||||
AND ${issues.assigneeAgentId} IS NOT NULL AND ${issues.assigneeUserId} IS NULL
|
||||
AND ${issues.monitorNextCheckAt} IS NOT NULL
|
||||
AND ${issues.executionRunId} IS NULL AND ${issues.checkoutRunId} IS NULL
|
||||
AND ${issues.executionState}->>'currentParticipant' IS NULL
|
||||
AND COALESCE(${issues.executionState}#>>'{monitor,status}', 'scheduled') = 'scheduled'
|
||||
AND ${issues.executionPolicy}->'monitor' IS NOT NULL
|
||||
AND COALESCE(${issues.executionPolicy}#>>'{monitor,serviceName}', '') <> ${PROVIDER_QUOTA_MONITOR_SERVICE_NAME}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM heartbeat_runs display_run
|
||||
WHERE display_run.company_id = ${companyId}
|
||||
AND display_run.issue_id = ${issueId}
|
||||
AND display_run.status IN ('queued', 'running')
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM issue_recovery_actions display_recovery
|
||||
WHERE display_recovery.company_id = ${companyId}
|
||||
AND display_recovery.source_issue_id = ${issueId}
|
||||
AND display_recovery.status IN ('active', 'escalated')
|
||||
)
|
||||
THEN 'monitoring' ELSE ${issues.status} END`;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { issueDisplayStatusSql } from "./issue-display-status.js";
|
||||
import { and, eq, inArray, isNull, type SQL } from "drizzle-orm";
|
||||
import { issues, type Db } from "@paperclipai/db";
|
||||
import { ISSUE_SUMMARIES_MAX_IDS } from "@paperclipai/shared";
|
||||
@@ -18,6 +19,7 @@ export async function listIssueSummaries(
|
||||
title: issues.title,
|
||||
identifier: issues.identifier,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
updatedAt: issues.updatedAt,
|
||||
externalConversationState: externalConversationStateSql(),
|
||||
projectId: issues.projectId,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { IssueDisplayStatus } from "@paperclipai/shared";
|
||||
import { issueDisplayStatusSql } from "./issue-display-status.js";
|
||||
import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js";
|
||||
import { finishedIdleWake } from "./idle-work-proof.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
@@ -1900,6 +1902,7 @@ export function parseStatusFilter(
|
||||
export interface IssueFilters {
|
||||
attention?: "blocked";
|
||||
status?: string | readonly string[];
|
||||
displayStatus?: string | readonly string[];
|
||||
/**
|
||||
* Filter by assignee agent ID.
|
||||
* - `string` (UUID): match issues assigned to that agent.
|
||||
@@ -1944,14 +1947,14 @@ export interface IssueFilters {
|
||||
q?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
sortField?: "updated" | "id";
|
||||
sortField?: "updated" | "created" | "id";
|
||||
afterId?: string;
|
||||
sortDir?: "asc" | "desc";
|
||||
/** ISO 8601 timestamp — only return issues with updatedAt strictly after this value. */
|
||||
updatedSince?: string;
|
||||
}
|
||||
|
||||
type IssueRow = typeof issues.$inferSelect & { externalConversationState?: "active" | "waiting" | null };
|
||||
type IssueRow = typeof issues.$inferSelect & { displayStatus?: IssueDisplayStatus; externalConversationState?: "active" | "waiting" | null };
|
||||
type IssueLabelRow = typeof labels.$inferSelect;
|
||||
type IssuePlanDecompositionRow = typeof issuePlanDecompositions.$inferSelect;
|
||||
type IssueActiveRunRow = {
|
||||
@@ -2559,12 +2562,13 @@ function blockerWorkspaceFacts(companyId: string, workspaceScope: SQL) {
|
||||
}
|
||||
|
||||
/** An issue's own operation takes precedence over the legacy NULL fallback.
|
||||
* Only a strictly later successful finalize clears either barrier. */
|
||||
* A later successful finalize clears either barrier. A successful operation
|
||||
* enclosed by its owning finalize is also saved, although it started later. */
|
||||
function pendingFinalizeBlockerCondition(companyId: string, issueId: SQL, workspaceId: SQL) {
|
||||
return sql`${workspaceId} is not null and exists (
|
||||
select 1 from (values (1)) seed(value)
|
||||
left join lateral (
|
||||
select phase, status, started_at from workspace_operations blocker_operation
|
||||
select phase, status, started_at, finished_at, heartbeat_run_id from workspace_operations blocker_operation
|
||||
where blocker_operation.company_id = ${companyId}::uuid
|
||||
and blocker_operation.execution_workspace_id = ${workspaceId}
|
||||
and blocker_operation.issue_id = ${issueId}
|
||||
@@ -2575,6 +2579,20 @@ function pendingFinalizeBlockerCondition(companyId: string, issueId: SQL, worksp
|
||||
and coalesce(latest_blocker_operation.status, blocker_workspace_facts.status) = 'succeeded')
|
||||
and (blocker_workspace_facts.finalized_at is null
|
||||
or blocker_workspace_facts.finalized_at <= coalesce(latest_blocker_operation.started_at, blocker_workspace_facts.started_at))
|
||||
and not (
|
||||
latest_blocker_operation.status = 'succeeded'
|
||||
and latest_blocker_operation.finished_at is not null
|
||||
and exists (
|
||||
select 1 from workspace_operations owning_finalize
|
||||
where owning_finalize.company_id = ${companyId}::uuid
|
||||
and owning_finalize.execution_workspace_id = ${workspaceId}
|
||||
and owning_finalize.issue_id = ${issueId}
|
||||
and owning_finalize.heartbeat_run_id = latest_blocker_operation.heartbeat_run_id
|
||||
and owning_finalize.phase = 'workspace_finalize' and owning_finalize.status = 'succeeded'
|
||||
and owning_finalize.started_at <= latest_blocker_operation.started_at
|
||||
and owning_finalize.finished_at >= latest_blocker_operation.finished_at
|
||||
)
|
||||
)
|
||||
)`;
|
||||
}
|
||||
|
||||
@@ -2929,7 +2947,7 @@ function myLastTouchAtExpr(companyId: string, userId: string) {
|
||||
COALESCE(${myLastCommentAt}, to_timestamp(0)),
|
||||
COALESCE(${myLastReadAt}, to_timestamp(0)),
|
||||
COALESCE(CASE WHEN ${issues.createdByUserId} = ${userId} AND ${humanCreatedIssueCondition()} THEN ${issues.createdAt} ELSE NULL END, to_timestamp(0)),
|
||||
COALESCE(CASE WHEN ${issues.assigneeUserId} = ${userId} THEN ${issues.updatedAt} ELSE NULL END, to_timestamp(0))
|
||||
COALESCE(CASE WHEN ${issues.assigneeUserId} = ${userId} THEN ${issues.createdAt} ELSE NULL END, to_timestamp(0))
|
||||
)
|
||||
`;
|
||||
}
|
||||
@@ -2996,6 +3014,7 @@ function unreadForUserCondition(companyId: string, userId: string) {
|
||||
FROM ${issueComments}
|
||||
WHERE ${issueComments.issueId} = ${issues.id}
|
||||
AND ${issueComments.companyId} = ${companyId}
|
||||
AND ${issueComments.deletedAt} IS NULL
|
||||
AND (
|
||||
${issueComments.authorUserId} IS NULL
|
||||
OR ${issueComments.authorUserId} <> ${userId}
|
||||
@@ -3118,8 +3137,10 @@ export function deriveIssueUserContext(
|
||||
issue.createdByUserId === userId && !isAgentCreatedIssue(issue)
|
||||
? normalizeDate(issue.createdAt)
|
||||
: null;
|
||||
// Assignment establishes inbox participation, not a read. Other people's
|
||||
// comments also advance updatedAt, so only creation is a stable fallback.
|
||||
const assignedTouchAt =
|
||||
issue.assigneeUserId === userId ? normalizeDate(issue.updatedAt) : null;
|
||||
issue.assigneeUserId === userId ? normalizeDate(issue.createdAt) : null;
|
||||
const myLastTouchAt =
|
||||
[myLastCommentAt, myLastReadAt, createdTouchAt, assignedTouchAt]
|
||||
.filter((value): value is Date => value instanceof Date)
|
||||
@@ -3218,6 +3239,11 @@ function issueListOrderBy(
|
||||
},
|
||||
) {
|
||||
if (sortField === "id") return [sortDir === "desc" ? desc(issues.id) : asc(issues.id)];
|
||||
if (sortField === "created") {
|
||||
return sortDir === "asc"
|
||||
? [asc(issues.createdAt), asc(issues.id)]
|
||||
: [desc(issues.createdAt), desc(issues.id)];
|
||||
}
|
||||
const canonicalLastActivityAt = issueCanonicalLastActivityAtExpr(companyId);
|
||||
if (sortField === "updated") {
|
||||
const activityOrder =
|
||||
@@ -3603,6 +3629,7 @@ function appendBlockerAttentionEdges(
|
||||
}
|
||||
|
||||
type IssueRelationSummaryRow = {
|
||||
displayStatus?: IssueDisplayStatus;
|
||||
relatedId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
@@ -3620,6 +3647,7 @@ function summarizeIssueRelationRow(
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status as IssueRelationIssueSummary["status"],
|
||||
displayStatus: row.displayStatus,
|
||||
priority: row.priority as IssueRelationIssueSummary["priority"],
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
@@ -3735,6 +3763,7 @@ async function terminalExplicitBlockersByRoot(
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
@@ -5039,6 +5068,7 @@ const issueListSelect = {
|
||||
END
|
||||
`,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
statusVersion: issues.statusVersion,
|
||||
lastStatusDecisionId: issues.lastStatusDecisionId,
|
||||
workMode: issues.workMode,
|
||||
@@ -5124,7 +5154,8 @@ async function userCommentStatsForIssues(
|
||||
lastExternalCommentAt: sql<Date | null>`
|
||||
MAX(
|
||||
CASE
|
||||
WHEN ${issueComments.authorUserId} IS NULL OR ${issueComments.authorUserId} <> ${userId}
|
||||
WHEN ${issueComments.deletedAt} IS NULL
|
||||
AND (${issueComments.authorUserId} IS NULL OR ${issueComments.authorUserId} <> ${userId})
|
||||
THEN ${issueComments.createdAt}
|
||||
END
|
||||
)
|
||||
@@ -5279,6 +5310,7 @@ async function blockedByMapForIssues(
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
@@ -5301,6 +5333,7 @@ async function blockedByMapForIssues(
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status as IssueRelationIssueSummary["status"],
|
||||
displayStatus: row.displayStatus,
|
||||
priority: row.priority as IssueRelationIssueSummary["priority"],
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
@@ -6420,6 +6453,8 @@ async function blockedInboxIssueConditions(
|
||||
filters?.lowTrustBoundary,
|
||||
);
|
||||
if (lowTrustCondition) conditions.push(lowTrustCondition);
|
||||
const displayStatuses = parseStatusFilter(filters?.displayStatus);
|
||||
if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses));
|
||||
const statuses = parseStatusFilter(filters?.status);
|
||||
if (statuses.length > 0) {
|
||||
conditions.push(
|
||||
@@ -6872,7 +6907,7 @@ export function issueService(db: Db) {
|
||||
|
||||
async function getIssueByUuid(id: string) {
|
||||
const row = await retryIdempotentDatabaseOperation(() => db
|
||||
.select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() })
|
||||
.select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() })
|
||||
.from(issues)
|
||||
.where(eq(issues.id, id))
|
||||
.then((rows) => rows[0] ?? null));
|
||||
@@ -6883,7 +6918,7 @@ export function issueService(db: Db) {
|
||||
|
||||
async function getIssueByIdentifier(identifier: string) {
|
||||
const row = await retryIdempotentDatabaseOperation(() => db
|
||||
.select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() })
|
||||
.select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() })
|
||||
.from(issues)
|
||||
.where(eq(issues.identifier, identifier.toUpperCase()))
|
||||
.then((rows) => rows[0] ?? null));
|
||||
@@ -7482,6 +7517,7 @@ export function issueService(db: Db) {
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
@@ -7502,6 +7538,7 @@ export function issueService(db: Db) {
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
@@ -8193,6 +8230,9 @@ export function issueService(db: Db) {
|
||||
if (filters?.sortField === "id" && filters.attention) {
|
||||
throw unprocessable("ID ordering is not supported for blocked attention lists");
|
||||
}
|
||||
if (filters?.sortField === "created" && filters.attention) {
|
||||
throw unprocessable("Creation-time ordering is not supported for blocked attention lists");
|
||||
}
|
||||
if (filters?.afterId !== undefined && (
|
||||
!isUuidLike(filters.afterId) || filters.sortField !== "id" ||
|
||||
filters.sortDir !== "asc" || (filters.offset ?? 0) !== 0
|
||||
@@ -8281,6 +8321,8 @@ export function issueService(db: Db) {
|
||||
filters?.lowTrustBoundary,
|
||||
);
|
||||
if (lowTrustCondition) conditions.push(lowTrustCondition);
|
||||
const displayStatuses = parseStatusFilter(filters?.displayStatus);
|
||||
if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses));
|
||||
const statuses = parseStatusFilter(filters?.status);
|
||||
if (statuses.length === 1) {
|
||||
conditions.push(eq(issues.status, statuses[0]));
|
||||
@@ -8574,6 +8616,8 @@ export function issueService(db: Db) {
|
||||
conditions.push(nonIdleSlackIssueCondition());
|
||||
}
|
||||
}
|
||||
const displayStatuses = parseStatusFilter(filters?.displayStatus);
|
||||
if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses));
|
||||
const statuses = parseStatusFilter(filters?.status);
|
||||
if (statuses.length === 1)
|
||||
conditions.push(eq(issues.status, statuses[0]!));
|
||||
@@ -8895,6 +8939,7 @@ export function issueService(db: Db) {
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
@@ -9522,6 +9567,7 @@ export function issueService(db: Db) {
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
displayStatus: issueDisplayStatusSql(),
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../../__tests__/helpers/full-database-response-capture.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { expect } from "vitest";
|
||||
|
||||
@@ -21,11 +21,11 @@ export async function verifyCommittedReplayPayloadReads(
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try { await replay(); }
|
||||
finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBeGreaterThan(0);
|
||||
expect(summary.responseValueBytes).toBeLessThan(100_000);
|
||||
const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues,
|
||||
nativeRunFinalizations, nativeRunResults, chatTaskHandoffs, chatCompletionDeliveries, agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
nativeRunFinalizations, nativeRunResults, issueThreadInteractions, chatTaskHandoffs, chatCompletionDeliveries, agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
|
||||
@@ -26,7 +26,7 @@ describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () =>
|
||||
}, 60_000);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
|
||||
async function fixture(conversation = true, options: { succeed?: boolean } = {}) {
|
||||
async function fixture(conversation = true, options: { succeed?: boolean; disposition?: "done" | "needs_review" } = {}) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const sessionId = randomUUID(), runnerId = randomUUID(), userId = `fixture-${randomUUID()}`, wakeId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Conversation owner", email: `${userId}@example.test`,
|
||||
@@ -53,10 +53,12 @@ describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () =>
|
||||
const prpEvent = { runId, normalizedSessionId: sessionId, turnId: "hire-turn", sourceEventId: "fixture-final-answer",
|
||||
payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: reply } };
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 170, eventType: "item.completed", payload: { prpEvent } });
|
||||
await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done",
|
||||
const disposition = options.disposition ?? "done";
|
||||
await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: disposition,
|
||||
summary: "A completed hire, with its final message preserved separately.", completionClaim: { contractRevision: "test",
|
||||
objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [] },
|
||||
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, turnId: "hire-turn" });
|
||||
objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [],
|
||||
attentionRequests: disposition === "needs_review" ? [{ kind: "review", ownerClass: "human", summary: "Review the completed work." }] : [], artifacts: [] },
|
||||
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: disposition }, turnId: "hire-turn" });
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: false });
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("committed");
|
||||
if (options.succeed !== false) await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date("2020-01-01T00:02:00Z") }).where(eq(heartbeatRuns.id, runId));
|
||||
@@ -641,6 +643,38 @@ describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () =>
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply })]);
|
||||
expect((await issue(f.issueId)).conversationAgentId).toBeNull();
|
||||
});
|
||||
it("recovers a deferred review reply once without resolving the review or rerunning work", async () => {
|
||||
const f = await fixture(false, { disposition: "needs_review" });
|
||||
const before = await issue(f.issueId);
|
||||
const reviews = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, f.issueId));
|
||||
expect(before.status).toBe("in_review");
|
||||
expect(reviews).toEqual([expect.objectContaining({ status: "pending" })]);
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
|
||||
expect(entries).toHaveLength(1);
|
||||
await expect(work.check(entries[0])).resolves.toBeNull();
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply, createdByRunId: f.runId })]);
|
||||
expect(await issue(f.issueId)).toEqual({ ...before, updatedAt: expect.any(Date) });
|
||||
expect(await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, f.issueId))).toEqual(reviews);
|
||||
expect(await work.load(f.input)).toEqual([]);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.nativeIssueId, f.issueId))).toHaveLength(1);
|
||||
});
|
||||
it.each(["missing-final", "wrong-turn", "digest", "skip-comment"])(
|
||||
"keeps deferred review response recovery closed for %s", async fault => {
|
||||
const f = await fixture(false, { disposition: "needs_review" });
|
||||
if (fault === "missing-final") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId));
|
||||
if (fault === "wrong-turn") await db.update(heartbeatRunEvents).set({ payload: { prpEvent: { ...f.prpEvent, turnId: "another-turn" } } })
|
||||
.where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.seq, 170)));
|
||||
if (fault === "digest") await db.update(nativeRunResults).set({ canonicalSha256: "invalid-digest" }).where(eq(nativeRunResults.runId, f.runId));
|
||||
if (fault === "skip-comment") await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, skipIssueComment: true } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const before = await issue(f.issueId);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
expect(await issue(f.issueId)).toEqual(before);
|
||||
},
|
||||
);
|
||||
it("rolls back a partial reply write without changing the waiting state or consuming recovery", async () => {
|
||||
const f = await fixture(), abort = new Error("fixture-commit-aborted");
|
||||
const failing = new Proxy(db, { get(target, key) { return key === "transaction"
|
||||
|
||||
@@ -19,6 +19,50 @@ describe("assigned MCP runner tools", () => {
|
||||
const searchName = "paperclip_search_assigned_tools";
|
||||
const callName = "paperclip_call_assigned_tool";
|
||||
|
||||
it.each(["", " \t\n"])("discovers and calls assigned tools through MCP with a blank schema reference %j", async schemaTool => {
|
||||
const github = descriptor("github_bot:read_pull_request");
|
||||
const f = fixture([github, descriptor("calendar.search")]);
|
||||
const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" });
|
||||
const bridge = await startOpenCodeMcpBridge({ tools: assigned.definitions(), handler: call => assigned.execute(call) });
|
||||
let id = 0;
|
||||
const call = async (name: string, args: Record<string, unknown>) => {
|
||||
const response = await fetch(bridge.url, {
|
||||
method: "POST", headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }),
|
||||
});
|
||||
const result = (await response.json()).result;
|
||||
expect(result.isError).not.toBe(true);
|
||||
return JSON.parse(result.content[0].text);
|
||||
};
|
||||
try {
|
||||
const page = await call(searchName, { query: "github", schemaTool });
|
||||
expect(page.tools).toHaveLength(1);
|
||||
expect(page.tools[0]).toMatchObject({ description: expect.stringContaining(github.name), inputSchema: github.parametersSchema });
|
||||
expect(await call(searchName, { query: "github", schemaTool, schemaOffset: 0 })).toEqual(page);
|
||||
expect((await call(searchName, { query: "", schemaTool })).tools).toHaveLength(2);
|
||||
await call(callName, { name: page.tools[0].name, arguments: { query: "fixture" } });
|
||||
expect(f.executeTool).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
|
||||
tool: github.name, parameters: { query: "fixture" }, gatewayPublicId: f.gatewayPublicId, sessionToken: f.bearerToken,
|
||||
}));
|
||||
f.listToolsForNamedGateway.mockResolvedValue([]);
|
||||
expect(await call(searchName, { query: "github", schemaTool })).toEqual({ tools: [], nextOffset: null });
|
||||
} finally {
|
||||
await bridge.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("explains how to recover from an unknown schema reference or an offset without a reference", async () => {
|
||||
const f = fixture([descriptor("github_bot:read_pull_request")]);
|
||||
const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" });
|
||||
await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool: "unknown" } }))
|
||||
.rejects.toThrow("Omit schemaTool to search assigned tools");
|
||||
for (const schemaTool of [undefined, "", " \t\n"]) {
|
||||
await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool, schemaOffset: 1 } }))
|
||||
.rejects.toThrow("A positive schemaOffset requires an exact inputSchemaRef in schemaTool");
|
||||
}
|
||||
expect(f.executeTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([undefined, "on_demand"] as const)("advertises usable page limits through MCP discovery (%s)", async delivery => {
|
||||
const f = fixture(Array.from({ length: 25 }, (_, i) => descriptor(`operation_${i}`)));
|
||||
const assigned = await createAssignedMcpTools({ ...f, delivery });
|
||||
|
||||
@@ -21,8 +21,8 @@ const ON_DEMAND_TOOLS: ToolDefinition[] = [
|
||||
query: { type: "string", maxLength: 200 },
|
||||
offset: { type: "integer", minimum: 0 },
|
||||
limit: { type: "integer", minimum: 1, maximum: 20, default: 5, description: "Maximum results per page (1 to 20; default 5)." },
|
||||
schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result." },
|
||||
schemaOffset: { type: "integer", minimum: 0 },
|
||||
schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result. Omit or leave blank for a normal tool search." },
|
||||
schemaOffset: { type: "integer", minimum: 0, description: "Schema chunk offset. A positive offset requires an exact inputSchemaRef in schemaTool." },
|
||||
},
|
||||
required: ["query"],
|
||||
},
|
||||
@@ -116,14 +116,17 @@ export async function createAssignedMcpTools(input: {
|
||||
const offset = args.offset ?? 0;
|
||||
const limit = args.limit ?? 5;
|
||||
const schemaOffset = args.schemaOffset ?? 0;
|
||||
const hasSchemaTool = typeof args.schemaTool === "string" && args.schemaTool.trim().length > 0;
|
||||
if (typeof args.query !== "string" || args.query.length > 200 ||
|
||||
typeof offset !== "number" || !Number.isSafeInteger(offset) || offset < 0 ||
|
||||
typeof limit !== "number" || !Number.isSafeInteger(limit) || limit < 1 || limit > 20 ||
|
||||
(args.schemaTool !== undefined && typeof args.schemaTool !== "string") ||
|
||||
typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0 ||
|
||||
(args.schemaOffset !== undefined && args.schemaTool === undefined)) {
|
||||
typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0) {
|
||||
throw new Error("assigned_mcp_tool_invalid_arguments");
|
||||
}
|
||||
if (schemaOffset > 0 && !hasSchemaTool) {
|
||||
throw new Error("assigned_mcp_tool_invalid_arguments: A positive schemaOffset requires an exact inputSchemaRef in schemaTool. Omit schemaTool and schemaOffset to search assigned tools.");
|
||||
}
|
||||
// Intersect fresh grants with this session's pinned catalog. Revocations
|
||||
// take effect immediately; newly assigned tools require a new session.
|
||||
const current = new Map(deduplicateToolDescriptors(await input.gateway.listToolsForNamedGateway({
|
||||
@@ -135,9 +138,9 @@ export async function createAssignedMcpTools(input: {
|
||||
return fresh && permits(tool) && permits(tool, currentWorkMode) &&
|
||||
permits(fresh) && permits(fresh, currentWorkMode);
|
||||
});
|
||||
if (typeof args.schemaTool === "string") {
|
||||
if (hasSchemaTool) {
|
||||
const selected = authorized.find(([name]) => name === args.schemaTool);
|
||||
if (!selected) throw new Error("assigned_mcp_tool_unknown");
|
||||
if (!selected) throw new Error("assigned_mcp_tool_unknown: schemaTool must match an authorized inputSchemaRef. Omit schemaTool to search assigned tools, then use an exact inputSchemaRef from the results.");
|
||||
const schema = JSON.stringify(selected[1].parametersSchema);
|
||||
const end = schemaOffset + schemaChunkCharacters;
|
||||
return {
|
||||
|
||||
@@ -29,7 +29,7 @@ import {
|
||||
toolApplications,
|
||||
toolConnections,
|
||||
} from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js";
|
||||
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js";
|
||||
@@ -322,7 +322,7 @@ describe("native same-conversation chat attachment reuse", () => {
|
||||
const deliveries = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId));
|
||||
const bulk = randomBytes(64 * 1024).toString("base64");
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
let capture: ReturnType<typeof createDatabaseResponseCapture> | undefined;
|
||||
let capture: ReturnType<typeof createFullDatabaseResponseCapture> | undefined;
|
||||
try {
|
||||
await db.update(chatEndpoints).set({ communicationInstructions: bulk,
|
||||
setup: { ...endpoint.setup, unusedContent: bulk } as typeof endpoint.setup,
|
||||
@@ -335,7 +335,7 @@ describe("native same-conversation chat attachment reuse", () => {
|
||||
.where(eq(chatDeliveries.id, delivery.id));
|
||||
}
|
||||
const now = Date.now();
|
||||
capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatConversations, chatEndpoints,
|
||||
chatEndpointResources, chatExternalPrincipals, chatIdentityLinks, companyMemberships }, capture);
|
||||
for (const lockMode of ["read", "blocking", "nonblocking"] as const) {
|
||||
|
||||
Loaded 100 of 314 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user