mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-28 12:04:41 +01:00
Merge synchronized native tracing into portable controller runtime
Include the safe evidence-reader module in portable inventory and preserve current master checkbox behavior plus release completion waits. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
ebe8910c29
313 files changed
+13760
-1123
No files matched your search
@@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
const mockActivityService = vi.hoisted(() => ({
|
||||
list: vi.fn(),
|
||||
forIssue: vi.fn(),
|
||||
inputFactsForIssue: vi.fn(),
|
||||
runsForIssue: vi.fn(),
|
||||
issuesForRun: vi.fn(),
|
||||
create: vi.fn(),
|
||||
@@ -152,6 +153,22 @@ describe("activity routes", () => {
|
||||
expect(res.body).toEqual([{ action: "issue.recovery_action_resolved", actorType: "system" }]);
|
||||
});
|
||||
|
||||
it("reads bounded input facts with the same issue access checks", async () => {
|
||||
const commentId = "11111111-1111-4111-8111-111111111111";
|
||||
mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-1", companyId: "company-1" });
|
||||
mockActivityService.inputFactsForIssue.mockResolvedValue([{ action: "issue.queued_comment_steered" }]);
|
||||
const app = await createApp({ type: "agent", companyId: "company-1", agentId: "agent-1" });
|
||||
const res = await requestApp(app, baseUrl => request(baseUrl).get(`/api/issues/TEST-1/activity?view=comment-inputs&commentIds=${commentId}`));
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockActivityService.inputFactsForIssue).toHaveBeenCalledWith("company-1", "issue-1", [commentId]);
|
||||
expect(mockAccessService.decide).toHaveBeenCalledWith(expect.objectContaining({ action: "issue:read" }));
|
||||
const invalid = await requestApp(app, baseUrl => request(baseUrl).get("/api/issues/TEST-1/activity?view=comment-inputs&commentIds=invalid"));
|
||||
expect(invalid.status).toBe(400);
|
||||
const oversized = await requestApp(app, baseUrl => request(baseUrl).get(`/api/issues/TEST-1/activity?view=comment-inputs&commentIds=${Array(101).fill(commentId).join(",")}`));
|
||||
expect(oversized.status).toBe(400);
|
||||
expect(mockActivityService.inputFactsForIssue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each(["boundary", "company"])("denies agent activity reads outside the %s scope", async (kind) => {
|
||||
const issue = { id: "issue-1", companyId: kind === "company" ? "company-2" : "company-1" };
|
||||
mockIssueService.getById.mockResolvedValue(issue);
|
||||
@@ -161,6 +178,9 @@ describe("activity routes", () => {
|
||||
const res = await requestApp(app, baseUrl => request(baseUrl).get("/api/issues/issue-1/activity"));
|
||||
expect(res.status).toBe(kind === "company" ? 404 : 403);
|
||||
expect(mockActivityService.forIssue).not.toHaveBeenCalled();
|
||||
const inputs = await requestApp(app, baseUrl => request(baseUrl).get("/api/issues/issue-1/activity?view=comment-inputs"));
|
||||
expect(inputs.status).toBe(kind === "company" ? 404 : 403);
|
||||
expect(mockActivityService.inputFactsForIssue).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns redacted all-actors rows to a basic company reader", async () => {
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
|
||||
import { actorMiddleware } from "../middleware/auth.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { captureRunIdentity, initializeRunIdentity, reserveSteeredIdentity } from "../services/run-identity.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("agent authentication identity transfer", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => {
|
||||
vi.stubEnv("PAPERCLIP_AGENT_JWT_SECRET", "identity-transfer-test-secret");
|
||||
vi.stubEnv("PAPERCLIP_INSTANCE_ID", "default");
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-auth-identity-transfer-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 30_000);
|
||||
afterAll(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await db?.$client.end({ timeout: 5 });
|
||||
await database?.cleanup();
|
||||
}, 60_000);
|
||||
|
||||
async function seed() {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Authentication transfer", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Reader", role: "engineer", adapterType: "codex_local", status: "idle" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Authentication task", responsibleUserId: "owner" });
|
||||
const padding = "retained input ".repeat(32_768);
|
||||
const payloads = {
|
||||
contextSnapshot: { issueId, launchInput: padding },
|
||||
runnerProfileJson: { nativeExecutionInput: { savedInput: padding } },
|
||||
resultJson: { savedResult: padding }, stdoutExcerpt: padding, stderrExcerpt: padding,
|
||||
};
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", ...payloads });
|
||||
const identity = await initializeRunIdentity(db, { companyId, runId, issueId, messageIds: [], responsibleUserId: "owner", cause: "instruction" });
|
||||
const app = express();
|
||||
app.use(actorMiddleware(db, { deploymentMode: "authenticated", resolveSession: async () => null }));
|
||||
app.get("/actor", (req, res) => res.json(req.actor));
|
||||
app.patch("/actor", (req, res) => res.json(req.actor));
|
||||
app.use(errorHandler);
|
||||
const token = createLocalAgentJwt(agentId, companyId, "codex_local", runId, "stale-user");
|
||||
expect(token).not.toBeNull();
|
||||
return { companyId, agentId, issueId, runId, app, token: token!, identity, payloads };
|
||||
}
|
||||
|
||||
it("bounds actual middleware responses and preserves the complete stored run", async () => {
|
||||
const fixture = await seed();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId,
|
||||
identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" });
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384);
|
||||
expect((await captureRunIdentity(db, fixture)).run).toMatchObject(fixture.payloads);
|
||||
});
|
||||
|
||||
it("blocks uncertain steering, accepts its durable acknowledgement, and rechecks cancellation", async () => {
|
||||
const fixture = await seed(), messageId = randomUUID();
|
||||
await db.insert(issueComments).values({ id: messageId, companyId: fixture.companyId, issueId: fixture.issueId,
|
||||
authorUserId: "next-user", body: "Next instruction" });
|
||||
const pending = await reserveSteeredIdentity(db, { ...fixture, messageId });
|
||||
expect(pending).not.toBeNull();
|
||||
const blocked = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
|
||||
expect(blocked.status).toBe(409);
|
||||
await db.insert(heartbeatRunEvents).values({ companyId: fixture.companyId, agentId: fixture.agentId, runId: fixture.runId,
|
||||
seq: 1, eventType: "item.completed", sourceEventId: randomUUID(),
|
||||
payload: { prpEvent: { turnId: "auth-turn", itemId: `auth-turn:steer:${messageId}`,
|
||||
payload: { kind: "steering_acknowledgement" } } } });
|
||||
const accepted = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
|
||||
expect(accepted.status).toBe(200);
|
||||
expect(accepted.body).toMatchObject({ identityContextId: pending!.id, onBehalfOfUserId: "next-user" });
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const revoked = await request(fixture.app).patch("/actor").set("Authorization", `Bearer ${fixture.token}`);
|
||||
expect(revoked.status).toBe(403);
|
||||
expect(revoked.body.code).toBe("agent_run_cancelled");
|
||||
const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
expect(stored).toMatchObject(fixture.payloads);
|
||||
});
|
||||
|
||||
it.each(["company", "agent", "missing", "finished"] as const)("rejects %s run identity for the compact scope", async (fault) => {
|
||||
const fixture = await seed();
|
||||
if (fault === "finished") await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const input = { companyId: fault === "company" ? randomUUID() : fixture.companyId,
|
||||
agentId: fault === "agent" ? randomUUID() : fixture.agentId,
|
||||
runId: fault === "missing" ? randomUUID() : fixture.runId };
|
||||
await expect(captureRunIdentity(db, input, { readScope: "request_authentication" })).rejects.toThrow(/active run/);
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,7 @@ const mockHeartbeatService = vi.hoisted(() => ({
|
||||
getActiveRunIssueSummaryForAgent: vi.fn(),
|
||||
getRunReadAccess: vi.fn(),
|
||||
getRunLogAccess: vi.fn(),
|
||||
getRunWorkspaceAccess: vi.fn(),
|
||||
readLog: vi.fn(),
|
||||
wakeup: vi.fn(),
|
||||
getRun: vi.fn(),
|
||||
@@ -378,6 +379,10 @@ describe("agent live run routes", () => {
|
||||
logStore: "local_file",
|
||||
logRef: "logs/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa.ndjson",
|
||||
});
|
||||
mockHeartbeatService.getRunWorkspaceAccess.mockResolvedValue({
|
||||
id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", companyId: "company-1", scopeKind: "company", issueId: null,
|
||||
executionWorkspaceId: null,
|
||||
});
|
||||
mockHeartbeatService.readLog.mockResolvedValue({
|
||||
runId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||
store: "local_file",
|
||||
@@ -609,10 +614,10 @@ describe("agent live run routes", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses narrow run log metadata lookups for log polling", async () => {
|
||||
it.each([false, true])("uses narrow run log metadata lookups (aligned tail: %s)", async (aligned) => {
|
||||
const res = await requestApp(await createApp(), (baseUrl) =>
|
||||
request(baseUrl).get(
|
||||
"/api/heartbeat-runs/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/log?offset=12&limitBytes=64",
|
||||
`/api/heartbeat-runs/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/log?offset=12&limitBytes=64${aligned ? "&alignStartToRecord=true" : ""}`,
|
||||
),
|
||||
);
|
||||
|
||||
@@ -630,6 +635,8 @@ describe("agent live run routes", () => {
|
||||
{
|
||||
offset: 12,
|
||||
limitBytes: 64,
|
||||
signal: expect.any(AbortSignal),
|
||||
...(aligned ? { alignStartToRecord: true } : {}),
|
||||
},
|
||||
);
|
||||
expect(res.body).toEqual({
|
||||
@@ -687,6 +694,21 @@ describe("agent live run routes", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("lists workspace operations using narrow current run access", async () => {
|
||||
mockHeartbeatService.getRunWorkspaceAccess.mockResolvedValue({
|
||||
id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", companyId: "company-1", scopeKind: "company", issueId: null,
|
||||
executionWorkspaceId: " workspace-one ",
|
||||
});
|
||||
mockWorkspaceOperationService.listForRun.mockResolvedValue([]);
|
||||
const response = await requestApp(await createApp(), baseUrl => request(baseUrl)
|
||||
.get("/api/heartbeat-runs/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/workspace-operations"));
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body).toEqual([]);
|
||||
expect(mockHeartbeatService.getRunWorkspaceAccess).toHaveBeenCalledWith("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa");
|
||||
expect(mockHeartbeatService.getRun).not.toHaveBeenCalled();
|
||||
expect(mockWorkspaceOperationService.listForRun).toHaveBeenCalledWith("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "workspace-one");
|
||||
});
|
||||
|
||||
it("caps company live run polling by default", async () => {
|
||||
const rows = Array.from({ length: 75 }, (_, index) => ({
|
||||
id: `run-${index}`,
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, agentWakeupRequests, activityLog, authUsers, companies, companyMemberships, createDb,
|
||||
executionWorkspaces, heartbeatRuns, issues, projects, projectWorkspaces, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { registerAgentProjectAccessRoutes } from "../routes/agent-project-access.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { assertTaskWorkspaceExecutionAccess, assertTaskWorkspaceSourceProjectAccess } from "../services/task-workspace-access.js";
|
||||
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
|
||||
import { ProjectAccessDeniedError } from "../services/project-access-denial.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { executionWorkspaceService } from "../services/execution-workspaces.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
describe("additive project access", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => {
|
||||
const external = process.env.PAPERCLIP_TEST_DATABASE_URL;
|
||||
if (external) db = createDb(external);
|
||||
else { database = await startEmbeddedPostgresTestDatabase("agent-project-access-"); db = createDb(database.connectionString); }
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
|
||||
async function seed() {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Project access", issuePrefix: `A${companyId.slice(0, 6)}` });
|
||||
const selected = await db.insert(projects).values(["Existing", "Needed", "Third"].map(name => ({ companyId, name }))).returning();
|
||||
const permissions = { canCreateAgents: false, canCreateSkills: false, trustPreset: "low_trust_review",
|
||||
authorizationPolicy: { reviewPreset: { id: "low_trust_review", version: 1, rawOutputDisposition: "quarantine" },
|
||||
customPolicy: { preserve: true }, trustBoundary: { mode: "low_trust_review", companyId,
|
||||
projectIds: [selected[0]!.id], issueIds: [issueId], allowedToolClasses: ["git.read"], allowedSecretBindingIds: [] } } };
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Reviewer", adapterType: "process", status: "idle", permissions });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "agent", principalId: agentId, membershipRole: "member", status: "active" });
|
||||
const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Saved message", status: "blocked", assigneeAgentId: agentId, projectId: selected[1]!.id }).returning();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "failed", errorCode: "setup_failed",
|
||||
error: "Project is outside this low-trust boundary.", contextSnapshot: { issueId } });
|
||||
return { companyId, agentId, issue: issue!, runId, selected, permissions };
|
||||
}
|
||||
function app(actor: express.Request["actor"]) {
|
||||
const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.actor = actor; next(); });
|
||||
registerAgentProjectAccessRoutes(app, db); app.use(errorHandler); return app;
|
||||
}
|
||||
const board = { type: "board" as const, source: "local_implicit" as const };
|
||||
|
||||
it("keeps a historical denial blocked after the task moves to an allowed project but retains its source", async () => {
|
||||
const f = await seed();
|
||||
const [source] = await db.insert(projectWorkspaces).values({ companyId: f.companyId, projectId: f.selected[1]!.id,
|
||||
name: "Original repository", cwd: "/tmp/project-access-original" }).returning();
|
||||
await db.update(issues).set({ projectWorkspaceId: source!.id,
|
||||
workspaceSelection: { version: 1, source: "project", selection: {
|
||||
kind: "configured_source", projectWorkspaceId: source!.id, mode: "managed_isolated",
|
||||
} } }).where(eq(issues.id, f.issue.id));
|
||||
const moved = await issueService(db).update(f.issue.id, { projectId: f.selected[0]!.id });
|
||||
expect(moved).toMatchObject({ projectId: f.selected[0]!.id, projectWorkspaceId: source!.id });
|
||||
const actor = { type: "agent" as const, source: "agent_key" as const, agentId: f.agentId, companyId: f.companyId };
|
||||
await expect(assertTaskWorkspaceExecutionAccess(db, actor, f.companyId, { issueId: f.issue.id }))
|
||||
.rejects.toBeInstanceOf(ProjectAccessDeniedError);
|
||||
expect(await taskInterruptionPresentation(db, moved!)).toMatchObject({
|
||||
canRetry: false, userAction: "project_access", projectAccess: { ready: false, projectId: null },
|
||||
});
|
||||
expect((await request(app(board)).post(`/agents/${f.agentId}/project-access`)
|
||||
.send({ projectIds: [f.selected[1]!.id] })).status).toBe(200);
|
||||
expect(await taskInterruptionPresentation(db, moved!)).toMatchObject({ canRetry: true, projectAccess: { ready: true } });
|
||||
await expect(assertTaskWorkspaceExecutionAccess(db, actor, f.companyId, { issueId: f.issue.id })).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it.each(["applied", "pending"] as const)("rechecks the selected source after an %s repair instead of the old denied project", async repair => {
|
||||
const f = await seed();
|
||||
const [source] = await db.insert(projectWorkspaces).values({ companyId: f.companyId, projectId: f.selected[0]!.id,
|
||||
name: "Allowed repository", cwd: "/tmp/project-access-allowed" }).returning();
|
||||
await db.update(heartbeatRuns).set({ errorCode: "project_access_denied", error: null,
|
||||
resultJson: { projectAccessDenied: { projectId: f.selected[1]!.id } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const moved = await issueService(db).update(f.issue.id, { projectId: f.selected[0]!.id });
|
||||
if (repair === "applied") {
|
||||
await issueService(db).update(f.issue.id, { projectWorkspaceId: source!.id, workspaceSelectionActor: board });
|
||||
} else {
|
||||
await executionWorkspaceService(db).selectTaskWorkspace({ companyId: f.companyId, issueId: f.issue.id, actor: board,
|
||||
selection: { kind: "configured_source", projectWorkspaceId: source!.id, mode: "managed_isolated" },
|
||||
expectedBindingRevision: 0, requestKey: randomUUID() });
|
||||
}
|
||||
expect(await taskInterruptionPresentation(db, moved!)).toMatchObject({ canRetry: true, projectAccess: { ready: true } });
|
||||
// The original denied source was not granted, and presentation creates no execution.
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0]!.permissions).toEqual(f.permissions);
|
||||
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
|
||||
if (repair === "pending") {
|
||||
await db.update(issues).set({ workspaceBindingRevision: 2 }).where(eq(issues.id, f.issue.id));
|
||||
const stale = await taskInterruptionPresentation(db, moved!);
|
||||
expect(stale).toMatchObject({ state: "failed", canRetry: false, userAction: null });
|
||||
expect(stale).not.toHaveProperty("projectAccess");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps a pending existing selection blocked if its workspace is archived before retry", async () => {
|
||||
const f = await seed();
|
||||
const task = await issueService(db).update(f.issue.id, { projectId: f.selected[0]!.id });
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: f.companyId,
|
||||
projectId: f.selected[0]!.id, name: "Selected files", cwd: "/tmp/project-access-selected",
|
||||
mode: "isolated_workspace", strategyType: "git_worktree" }).returning();
|
||||
const service = executionWorkspaceService(db);
|
||||
await service.selectTaskWorkspace({ companyId: f.companyId, issueId: f.issue.id, actor: board,
|
||||
selection: { kind: "existing", workspaceId: workspace!.id }, expectedBindingRevision: 0, requestKey: randomUUID() });
|
||||
expect(await taskInterruptionPresentation(db, task!)).toMatchObject({ canRetry: true, projectAccess: { ready: true } });
|
||||
expect(await service.archiveWorkspaceUnderLifecycleLock({ id: workspace!.id, patch: {}, closedAt: new Date() }))
|
||||
.toMatchObject({ outcome: "archived" });
|
||||
await expect(service.applyPendingTaskWorkspaceSelection({ companyId: f.companyId, issueId: f.issue.id,
|
||||
runId: f.runId, actor: { type: "agent", source: "agent_key", agentId: f.agentId, companyId: f.companyId } }))
|
||||
.rejects.toThrow("Workspace is unavailable or inaccessible");
|
||||
const archived = await taskInterruptionPresentation(db, task!);
|
||||
expect(archived).toMatchObject({ state: "failed", canRetry: false, userAction: null });
|
||||
expect(archived).not.toHaveProperty("projectAccess");
|
||||
});
|
||||
|
||||
it.each(["denied_project", "foreign_company"] as const)("retains the %s gate on a bound workspace even when its source is allowed", async boundary => {
|
||||
const f = await seed(), foreign = boundary === "foreign_company" ? await seed() : null;
|
||||
const [source] = await db.insert(projectWorkspaces).values({ companyId: f.companyId, projectId: f.selected[0]!.id,
|
||||
name: "Allowed source", cwd: "/tmp/project-access-source" }).returning();
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: foreign?.companyId ?? f.companyId,
|
||||
projectId: foreign?.selected[0]!.id ?? f.selected[1]!.id, projectWorkspaceId: foreign ? null : source!.id,
|
||||
name: "Retained files", cwd: "/tmp/project-access-bound", mode: "isolated_workspace", strategyType: "git_worktree" }).returning();
|
||||
const [task] = await db.update(issues).set({ projectId: f.selected[0]!.id, projectWorkspaceId: source!.id,
|
||||
executionWorkspaceId: workspace!.id }).where(eq(issues.id, f.issue.id)).returning();
|
||||
const blocked = await taskInterruptionPresentation(db, task!);
|
||||
if (boundary === "denied_project") {
|
||||
expect(blocked).toMatchObject({ canRetry: false,
|
||||
userAction: "project_access", projectAccess: { ready: false, projectId: null } });
|
||||
} else {
|
||||
expect(blocked).toMatchObject({ state: "failed", canRetry: false, userAction: null });
|
||||
expect(blocked).not.toHaveProperty("projectAccess");
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves containment, audits concurrent additions, and changes the persisted denial into an explicit retry", async () => {
|
||||
const f = await seed();
|
||||
const actor = { type: "agent" as const, source: "agent_key" as const, agentId: f.agentId, companyId: f.companyId };
|
||||
await expect(assertTaskWorkspaceSourceProjectAccess(db, actor, f.companyId, f.selected[1]!.id)).rejects.toBeInstanceOf(ProjectAccessDeniedError);
|
||||
expect(await taskInterruptionPresentation(db, f.issue)).toMatchObject({ userAction: "project_access", canRetry: false });
|
||||
const api = app(board);
|
||||
const results = await Promise.all(f.selected.slice(1).map(project => request(api).post(`/agents/${f.agentId}/project-access`).send({ projectIds: [project.id] })));
|
||||
expect(results.map(result => result.status)).toEqual([200, 200]);
|
||||
const [saved] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
expect(saved!.permissions).toEqual({ ...f.permissions, authorizationPolicy: { ...f.permissions.authorizationPolicy,
|
||||
trustBoundary: { ...f.permissions.authorizationPolicy.trustBoundary, projectIds: expect.arrayContaining(f.selected.map(project => project.id)) } } });
|
||||
expect((saved!.permissions as typeof f.permissions).authorizationPolicy.trustBoundary.projectIds).toHaveLength(3);
|
||||
await expect(assertTaskWorkspaceSourceProjectAccess(db, actor, f.companyId, f.selected[1]!.id)).resolves.toBeUndefined();
|
||||
expect(await taskInterruptionPresentation(db, f.issue)).toMatchObject({ canRetry: true, projectAccess: { ready: true } });
|
||||
const again = await request(api).post(`/agents/${f.agentId}/project-access`).send({ projectIds: [f.selected[1]!.id] });
|
||||
expect(again.status).toBe(200);
|
||||
expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.companyId), eq(activityLog.action, "agent.project_access_added")))).toHaveLength(2);
|
||||
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rejects foreign projects, agent self-grants, unknown policy fields, and pending-approval agents", async () => {
|
||||
const f = await seed(), other = await seed(), api = app(board);
|
||||
expect((await request(api).post(`/agents/${f.agentId}/project-access`).send({ projectIds: [other.selected[1]!.id] })).status).toBe(404);
|
||||
expect((await request(api).post(`/agents/${f.agentId}/project-access`).send({ projectIds: [f.selected[1]!.id], trustPreset: "standard" })).status).toBe(400);
|
||||
expect((await request(app({ type: "agent", agentId: f.agentId, companyId: f.companyId, source: "agent_key" }))
|
||||
.post(`/agents/${f.agentId}/project-access`).send({ projectIds: [f.selected[1]!.id] })).status).toBe(403);
|
||||
await db.update(agents).set({ status: "pending_approval" }).where(eq(agents.id, f.agentId));
|
||||
expect((await request(api).post(`/agents/${f.agentId}/project-access`).send({ projectIds: [f.selected[1]!.id] })).status).toBe(409);
|
||||
const [saved] = await db.select().from(agents).where(eq(agents.id, f.agentId)); expect(saved!.permissions).toEqual(f.permissions);
|
||||
});
|
||||
|
||||
it.each(["standard", "root_issue"])("rechecks actual project authority after a %s repair and acknowledges an accepted retry", async repair => {
|
||||
const f = await seed();
|
||||
const permissions = repair === "standard" ? { canCreateAgents: false, trustPreset: "standard", authorizationPolicy: {} }
|
||||
: { ...f.permissions, authorizationPolicy: { ...f.permissions.authorizationPolicy,
|
||||
trustBoundary: { ...f.permissions.authorizationPolicy.trustBoundary, rootIssueId: f.issue.id } } };
|
||||
await db.update(agents).set({ permissions }).where(eq(agents.id, f.agentId));
|
||||
expect(await taskInterruptionPresentation(db, f.issue)).toMatchObject({ canRetry: true, projectAccess: { ready: true } });
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
source: "on_demand", status: "deferred_issue_execution", requestedByActorType: "user", payload: { issueId: f.issue.id } });
|
||||
const accepted = await taskInterruptionPresentation(db, f.issue);
|
||||
expect(accepted).toMatchObject({ state: "working", canRetry: false });
|
||||
expect(accepted).not.toHaveProperty("projectAccess");
|
||||
// Current project policy remains authoritative even after agent access changes.
|
||||
await db.update(projects).set({ executionWorkspacePolicy: { trustPreset: "low_trust_review", authorizationPolicy: {
|
||||
trustBoundary: { mode: "low_trust_review", companyId: f.companyId, projectIds: [f.selected[0]!.id] },
|
||||
} } }).where(eq(projects.id, f.selected[1]!.id));
|
||||
if (repair === "standard") expect(await taskInterruptionPresentation(db, f.issue))
|
||||
.toMatchObject({ canRetry: false, userAction: "project_access" });
|
||||
});
|
||||
|
||||
it("shows read-only access to viewers and denies their saves and cross-company reads", async () => {
|
||||
const f = await seed(), other = await seed(), userId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Viewer", email: `${userId}@example.test`, emailVerified: true, createdAt: new Date(), updatedAt: new Date() });
|
||||
await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: userId, membershipRole: "viewer", status: "active" });
|
||||
const api = app({ type: "board", source: "session", userId, companyIds: [f.companyId], memberships: [{ companyId: f.companyId, membershipRole: "viewer", status: "active" }] });
|
||||
expect((await request(api).get(`/agents/${f.agentId}/project-access`)).body).toMatchObject({ canManage: false });
|
||||
expect((await request(api).post(`/agents/${f.agentId}/project-access`).send({ projectIds: [f.selected[1]!.id] })).status).toBe(403);
|
||||
expect((await request(api).get(`/agents/${other.agentId}/project-access`)).status).toBe(404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { agents, companies, costEvents, createDb } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { agentRecordQueries } from "../lib/agent-records.js";
|
||||
import { agentService } from "../services/agents.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("agent roster read projections", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("agent-roster-read-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
vi.setSystemTime(new Date("2020-06-15T12:00:00Z"));
|
||||
});
|
||||
|
||||
afterEach(() => { vi.useRealTimers(); });
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), foreignCompanyId = randomUUID();
|
||||
const managerId = randomUUID(), workerId = randomUUID(), unrelatedId = randomUUID(), archivedId = randomUUID(), foreignId = randomUUID();
|
||||
await db.insert(companies).values([
|
||||
{ id: companyId, name: "Roster fixture", issuePrefix: `R${companyId.slice(0, 7).toUpperCase()}` },
|
||||
{ id: foreignCompanyId, name: "Foreign roster", issuePrefix: `F${foreignCompanyId.slice(0, 7).toUpperCase()}` },
|
||||
]);
|
||||
const bulk = "retained-other-agent-configuration🙂".repeat(32_768);
|
||||
await db.insert(agents).values([
|
||||
{ id: managerId, companyId, name: "Manager", status: "terminated", adapterType: "process", adapterConfig: { instructions: bulk } },
|
||||
{ id: unrelatedId, companyId, name: "Other worker", status: "idle", adapterType: "process", adapterConfig: { instructions: bulk } },
|
||||
{ id: archivedId, companyId, name: "Archived worker", status: "terminated", adapterType: "process", runtimeConfig: { retained: bulk } },
|
||||
{ id: foreignId, companyId: foreignCompanyId, name: "Foreign manager", status: "idle", adapterType: "process", adapterConfig: { instructions: bulk } },
|
||||
]);
|
||||
const [worker] = await db.insert(agents).values({ id: workerId, companyId, name: "Selected worker", status: "idle",
|
||||
reportsTo: managerId, adapterType: "process", adapterConfig: { instructions: "selected instructions", model: "fixture-model" },
|
||||
runtimeConfig: { heartbeat: { enabled: false } }, metadata: { retained: "selected metadata" },
|
||||
permissions: { canCreateAgents: true }, spentMonthlyCents: 999,
|
||||
}).returning();
|
||||
const now = new Date();
|
||||
await db.insert(costEvents).values([
|
||||
{ companyId, agentId: workerId, provider: "fixture", model: "fixture-model", costCents: 19.5, occurredAt: now },
|
||||
{ companyId, agentId: workerId, provider: "fixture", model: "fixture-model", costCents: 50,
|
||||
occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 1, 15)) },
|
||||
{ companyId: foreignCompanyId, agentId: foreignId, provider: "fixture", model: "fixture-model", costCents: 100, occurredAt: now },
|
||||
]);
|
||||
return { companyId, foreignCompanyId, managerId, workerId, unrelatedId, archivedId, foreignId, worker, bulk };
|
||||
}
|
||||
|
||||
async function observe<T>(run: (observed: ReturnType<typeof createDb>) => Promise<T>) {
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agents, costEvents }, capture);
|
||||
try {
|
||||
const result = await run(observed);
|
||||
capture.flush();
|
||||
const reads = events.filter(event => event.table === "agents" && event.operation === "select");
|
||||
return { result, calls: reads.reduce((n, event) => n + Number(event.calls), 0),
|
||||
bytes: reads.reduce((n, event) => n + Number(event.responseValueBytes), 0), events };
|
||||
} finally {
|
||||
capture.flush();
|
||||
await observed.$client.end();
|
||||
}
|
||||
}
|
||||
|
||||
it.each([false, true])("reads one full roster for list, preserving terminated-manager evidence and configurations (includeTerminated=%s)", async includeTerminated => {
|
||||
const f = await fixture();
|
||||
const captured = await observe(observed => agentService(observed).list(f.companyId, { includeTerminated }));
|
||||
expect(captured.calls).toBe(1);
|
||||
expect(captured.result.map(row => row.id).sort()).toEqual((includeTerminated
|
||||
? [f.workerId, f.managerId, f.archivedId, f.unrelatedId] : [f.workerId, f.unrelatedId]).sort());
|
||||
const selected = captured.result.find(row => row.id === f.workerId)!;
|
||||
expect(selected).toMatchObject({ adapterConfig: f.worker.adapterConfig, runtimeConfig: f.worker.runtimeConfig,
|
||||
metadata: f.worker.metadata, permissions: { canCreateAgents: true }, spentMonthlyCents: 19.5,
|
||||
orgChainHealth: { status: "invalid_org_chain", reason: "terminated_ancestor", firstInvalidAncestor: { id: f.managerId } } });
|
||||
expect(captured.result.find(row => row.id === f.unrelatedId)?.adapterConfig.instructions).toBe(f.bulk);
|
||||
if (includeTerminated) expect(captured.result.find(row => row.id === f.archivedId)?.runtimeConfig.retained).toBe(f.bulk);
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.workerId)))[0]).toEqual(f.worker);
|
||||
expect(JSON.stringify(captured.events)).not.toContain("retained-other-agent-configuration");
|
||||
});
|
||||
|
||||
it("keeps detail reads small while returning the selected agent's complete current configuration", async () => {
|
||||
const f = await fixture();
|
||||
const captured = await observe(observed => agentRecordQueries(observed).getById(f.workerId));
|
||||
expect(captured.calls).toBe(2);
|
||||
expect(captured.bytes).toBeLessThan(8192);
|
||||
expect(captured.result).toMatchObject({ id: f.workerId, companyId: f.companyId, adapterConfig: f.worker.adapterConfig,
|
||||
runtimeConfig: f.worker.runtimeConfig, metadata: f.worker.metadata, permissions: { canCreateAgents: true }, spentMonthlyCents: 19.5,
|
||||
orgChainHealth: { reason: "terminated_ancestor", firstInvalidAncestor: { id: f.managerId, name: "Manager" } } });
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.workerId)))[0]).toEqual(f.worker);
|
||||
});
|
||||
|
||||
it("projects only current company chain fields for plugin normalization", async () => {
|
||||
const f = await fixture();
|
||||
const captured = await observe(observed => agentRecordQueries(observed).listCompanyAgentEligibilityRows(f.companyId));
|
||||
expect(captured.calls).toBe(1);
|
||||
expect(captured.bytes).toBeLessThan(4096);
|
||||
expect(captured.result.map(row => row.id).sort()).toEqual([f.workerId, f.managerId, f.unrelatedId, f.archivedId].sort());
|
||||
for (const row of captured.result) expect(Object.keys(row).sort()).toEqual(["companyId", "id", "name", "reportsTo", "status"]);
|
||||
const normalized = agentRecordQueries(db).normalizeAgentRow(f.worker, captured.result);
|
||||
expect(normalized).toMatchObject({ adapterConfig: f.worker.adapterConfig, metadata: f.worker.metadata,
|
||||
orgChainHealth: { reason: "terminated_ancestor", firstInvalidAncestor: { id: f.managerId } } });
|
||||
expect(await agentRecordQueries(db).listCompanyAgentEligibilityRows(randomUUID())).toEqual([]);
|
||||
expect(await agentRecordQueries(db).getById(randomUUID())).toBeNull();
|
||||
});
|
||||
|
||||
it("rechecks manager status, name, selected configuration, missing managers, and cycles on later reads", async () => {
|
||||
const f = await fixture(), records = agentRecordQueries(db);
|
||||
expect((await records.getById(f.workerId))?.orgChainHealth.reason).toBe("terminated_ancestor");
|
||||
await db.update(agents).set({ status: "paused", name: "Current manager" }).where(eq(agents.id, f.managerId));
|
||||
await db.update(agents).set({ adapterConfig: { instructions: "changed instructions" } }).where(eq(agents.id, f.workerId));
|
||||
expect(await records.getById(f.workerId)).toMatchObject({ adapterConfig: { instructions: "changed instructions" },
|
||||
orgChainHealth: { status: "healthy", pausedAncestors: [{ id: f.managerId, name: "Current manager", status: "paused" }] } });
|
||||
await db.update(agents).set({ reportsTo: f.foreignId }).where(eq(agents.id, f.workerId));
|
||||
expect((await records.getById(f.workerId))?.orgChainHealth.reason).toBe("missing_manager");
|
||||
await db.update(agents).set({ reportsTo: f.managerId }).where(eq(agents.id, f.workerId));
|
||||
await db.update(agents).set({ reportsTo: f.workerId }).where(eq(agents.id, f.managerId));
|
||||
expect((await records.getById(f.workerId))?.orgChainHealth.reason).toBe("cycle");
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,8 @@ import { createAgentLifecycle } from "../services/agent-lifecycle.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
@@ -1034,6 +1035,177 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
expect(feed.items.filter((item) => item.sourceKind === "failed_run")).toEqual([]);
|
||||
});
|
||||
|
||||
it("hides a failed run only after a newer run of the same agent on the same issue or task", async () => {
|
||||
const { companyId, workerId, reviewerId } = await seedCompany("ATK");
|
||||
const taskIssueId = await insertIssue({ companyId, identifier: "ATK-1", title: "Task-keyed", status: "in_progress" });
|
||||
const issueKeyedId = await insertIssue({ companyId, identifier: "ATK-2", title: "Issue-keyed", status: "in_progress" });
|
||||
const otherIssueId = await insertIssue({ companyId, identifier: "ATK-3", title: "Other work", status: "in_progress" });
|
||||
const failedAt = new Date("2026-07-09T12:00:00.000Z");
|
||||
const laterAt = new Date("2026-07-09T12:01:00.000Z");
|
||||
const taskKeyedFailureId = randomUUID();
|
||||
const issueKeyedFailureId = randomUUID();
|
||||
const unkeyedHiddenFailureId = randomUUID();
|
||||
const unkeyedKeptFailureId = randomUUID();
|
||||
const failureIds = [taskKeyedFailureId, issueKeyedFailureId, unkeyedHiddenFailureId, unkeyedKeptFailureId];
|
||||
const sameMillisecondRunId = randomUUID();
|
||||
const run = (id: string, agentId: string, status: string, contextSnapshot: Record<string, unknown>, at: Date) => ({
|
||||
id, companyId, agentId, invocationSource: "automation", status, contextSnapshot,
|
||||
createdAt: at, updatedAt: at, finishedAt: at, ...(status === "failed" ? { error: "adapter failed" } : {}),
|
||||
});
|
||||
|
||||
await db.insert(heartbeatRuns).values([
|
||||
run(taskKeyedFailureId, workerId, "failed", { taskId: taskIssueId }, failedAt),
|
||||
run(issueKeyedFailureId, workerId, "failed", { issueId: issueKeyedId }, failedAt),
|
||||
run(unkeyedHiddenFailureId, workerId, "failed", {}, failedAt),
|
||||
run(unkeyedKeptFailureId, reviewerId, "failed", {}, failedAt),
|
||||
run(randomUUID(), workerId, "succeeded", { taskId: taskIssueId }, laterAt),
|
||||
run(randomUUID(), workerId, "succeeded", { wakeReason: "timer" }, laterAt),
|
||||
run(randomUUID(), reviewerId, "succeeded", { issueId: issueKeyedId }, laterAt),
|
||||
run(randomUUID(), reviewerId, "succeeded", { issueId: otherIssueId }, laterAt),
|
||||
run(randomUUID(), workerId, "succeeded", { issueId: otherIssueId }, laterAt),
|
||||
run(randomUUID(), workerId, "succeeded", { issueId: otherIssueId, taskId: issueKeyedId }, laterAt),
|
||||
run(randomUUID(), workerId, "succeeded", { issueId: "", taskId: issueKeyedId }, laterAt),
|
||||
run(sameMillisecondRunId, workerId, "succeeded", { issueId: issueKeyedId }, failedAt),
|
||||
run(randomUUID(), workerId, "succeeded", { issueId: issueKeyedId }, new Date("2026-07-09T11:59:00.000Z")),
|
||||
]);
|
||||
await db.execute(sql`update heartbeat_runs set created_at = created_at + interval '400 microseconds'
|
||||
where id in (${sql.join(failureIds.map((id) => sql`${id}::uuid`), sql`, `)})`);
|
||||
await db.execute(sql`update heartbeat_runs set created_at = created_at + interval '600 microseconds'
|
||||
where id = ${sameMillisecondRunId}::uuid`);
|
||||
await db.insert(heartbeatRunEvents).values(
|
||||
failureIds.map((runId) => ({
|
||||
companyId,
|
||||
runId,
|
||||
agentId: runId === unkeyedKeptFailureId ? reviewerId : workerId,
|
||||
seq: 1,
|
||||
eventType: "lifecycle",
|
||||
message: "Bounded retry exhausted after 4 scheduled attempts; no further automatic retry will be queued",
|
||||
createdAt: new Date("2026-07-09T12:00:01.000Z"),
|
||||
})),
|
||||
);
|
||||
|
||||
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
|
||||
|
||||
expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort())
|
||||
.toEqual([issueKeyedFailureId, unkeyedKeptFailureId].sort());
|
||||
});
|
||||
|
||||
it("keeps newer-run response size bounded despite unrelated retained history", async () => {
|
||||
const { companyId, workerId } = await seedCompany("ATW");
|
||||
const issueId = await insertIssue({ companyId, identifier: "ATW-1", title: "Exhausted task", status: "in_progress" });
|
||||
const otherIssueId = await insertIssue({ companyId, identifier: "ATW-2", title: "Other task", status: "done" });
|
||||
const failedId = randomUUID(), failedAt = new Date("2026-07-09T12:00:00.000Z");
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: failedId, companyId, agentId: workerId, status: "failed", contextSnapshot: { issueId },
|
||||
createdAt: failedAt, updatedAt: failedAt, finishedAt: failedAt,
|
||||
});
|
||||
await db.insert(heartbeatRunEvents).values({
|
||||
companyId, agentId: workerId, runId: failedId, seq: 1, eventType: "lifecycle",
|
||||
message: "Bounded retry exhausted after scheduled attempts",
|
||||
});
|
||||
for (let batch = 0; batch < 4; batch += 1) {
|
||||
await db.insert(heartbeatRuns).values(Array.from({ length: 500 }, (_, index) => ({
|
||||
id: randomUUID(), companyId, agentId: workerId, status: "succeeded",
|
||||
contextSnapshot: { issueId: otherIssueId, prompt: "x".repeat(8_000) },
|
||||
createdAt: new Date(failedAt.getTime() + batch * 500 + index + 1),
|
||||
updatedAt: failedAt, finishedAt: failedAt,
|
||||
})));
|
||||
}
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
|
||||
expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]);
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
expect(reads.filter(event => event.operation === "execute")
|
||||
.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
const [remaining] = await db.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.companyId, companyId));
|
||||
expect(remaining?.count).toBe(2001);
|
||||
});
|
||||
|
||||
it("reads keyless history once per agent despite thousands of failures", async () => {
|
||||
const { companyId, workerId, reviewerId } = await seedCompany("ATB");
|
||||
const failedAt = new Date("2026-07-09T12:00:00.000Z");
|
||||
const laterAt = new Date("2026-07-09T12:01:00.000Z");
|
||||
const hiddenIds = Array.from({ length: 16_390 }, () => randomUUID());
|
||||
const keptIds = Array.from({ length: 10 }, () => randomUUID());
|
||||
const failure = (id: string, agentId: string) => ({
|
||||
id, companyId, agentId, invocationSource: "automation", status: "failed", error: "adapter failed",
|
||||
contextSnapshot: {}, createdAt: failedAt, updatedAt: failedAt, finishedAt: failedAt,
|
||||
});
|
||||
const failures = hiddenIds.map((id) => failure(id, workerId));
|
||||
keptIds.forEach((id, index) => failures.splice(index * 1_700, 0, failure(id, reviewerId)));
|
||||
for (let start = 0; start < failures.length; start += 1_000) {
|
||||
const chunk = failures.slice(start, start + 1_000);
|
||||
await db.insert(heartbeatRuns).values(chunk);
|
||||
await db.insert(heartbeatRunEvents).values(chunk.map((run) => ({
|
||||
companyId, runId: run.id, agentId: run.agentId, seq: 1, eventType: "lifecycle",
|
||||
message: "Bounded retry exhausted after 4 scheduled attempts; no further automatic retry will be queued",
|
||||
createdAt: new Date("2026-07-09T12:00:01.000Z"),
|
||||
})));
|
||||
}
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: randomUUID(), companyId, agentId: workerId, invocationSource: "automation", status: "succeeded",
|
||||
contextSnapshot: { wakeReason: "timer" }, createdAt: laterAt, updatedAt: laterAt, finishedAt: laterAt,
|
||||
});
|
||||
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const feed = await attentionService(db).list(companyId, {
|
||||
includeDismissed: true, all: true, allowUnscopedAll: true,
|
||||
});
|
||||
expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort())
|
||||
.toEqual([...keptIds].sort());
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4);
|
||||
}, 180_000);
|
||||
|
||||
it("checks more failed runs for a newer run than one statement can bind", async () => {
|
||||
const { companyId, workerId, reviewerId } = await seedCompany("ATQ");
|
||||
const issueId = await insertIssue({ companyId, identifier: "ATQ-1", title: "Keyed retained failures", status: "in_progress" });
|
||||
const failedAt = new Date("2026-07-09T12:00:00.000Z");
|
||||
const laterAt = new Date("2026-07-09T12:01:00.000Z");
|
||||
const hiddenIds = Array.from({ length: 16_390 }, () => randomUUID());
|
||||
const keptIds = Array.from({ length: 10 }, () => randomUUID());
|
||||
const failure = (id: string, agentId: string) => ({
|
||||
id, companyId, agentId, invocationSource: "automation", status: "failed", error: "adapter failed",
|
||||
contextSnapshot: { issueId }, createdAt: failedAt, updatedAt: failedAt, finishedAt: failedAt,
|
||||
});
|
||||
const failures = hiddenIds.map((id) => failure(id, workerId));
|
||||
keptIds.forEach((id, index) => failures.splice(index * 1_700, 0, failure(id, reviewerId)));
|
||||
for (let start = 0; start < failures.length; start += 1_000) {
|
||||
const chunk = failures.slice(start, start + 1_000);
|
||||
await db.insert(heartbeatRuns).values(chunk);
|
||||
await db.insert(heartbeatRunEvents).values(chunk.map((run) => ({
|
||||
companyId, runId: run.id, agentId: run.agentId, seq: 1, eventType: "lifecycle",
|
||||
message: "Bounded retry exhausted after 4 scheduled attempts; no further automatic retry will be queued",
|
||||
createdAt: new Date("2026-07-09T12:00:01.000Z"),
|
||||
})));
|
||||
}
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: randomUUID(), companyId, agentId: workerId, invocationSource: "automation", status: "succeeded",
|
||||
contextSnapshot: { issueId }, createdAt: laterAt, updatedAt: laterAt, finishedAt: laterAt,
|
||||
});
|
||||
|
||||
const feed = await attentionService(db).list(companyId, {
|
||||
includeDismissed: true, all: true, allowUnscopedAll: true,
|
||||
});
|
||||
expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id).sort())
|
||||
.toEqual([...keptIds].sort());
|
||||
}, 180_000);
|
||||
|
||||
it("enriches interaction details with project, workspace, plan metadata, and images", async () => {
|
||||
const { companyId, workerId } = await seedCompany("ATE");
|
||||
const projectId = randomUUID();
|
||||
|
||||
@@ -5,6 +5,8 @@ import { createSystemRecovery } from "../services/system-recovery-state.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js";
|
||||
import * as activityLogModule from "../services/activity-log.js";
|
||||
import * as connectionCredentialBindings from "../services/connection-credential-bindings.js";
|
||||
import { chatSlackManagerGrants, chatSlackRegistrations, toolOauthStates } from "@paperclipai/db";
|
||||
@@ -24,7 +26,7 @@ import { toolActionRequests, toolInvocations, toolPolicies } from "@paperclipai/
|
||||
import { GitHubPublicationLeaseLost, withGitHubPublicationLease } from "../services/chat-github-publication-lease.js";
|
||||
import { secretService } from "../services/secrets.js";
|
||||
import { githubChatWizardService } from "../services/chat-github-wizard.js";
|
||||
import { dispatchGitHubBotCloudEvent } from "../services/chat-github-cloud-ingress.js";
|
||||
import { dispatchGitHubBotCloudEvent, registerGitHubBotCloudSetupPoller } from "../services/chat-github-cloud-ingress.js";
|
||||
import { githubChatManagementService } from "../services/chat-github-management.js";
|
||||
import { githubChatReviewService } from "../services/chat-github-reviews.js";
|
||||
import { resolveGitHubReviewInstructions } from "../services/chat-channels/github/review-instructions.js";
|
||||
@@ -2585,6 +2587,81 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
.rejects.toMatchObject({ status: 403 });
|
||||
});
|
||||
|
||||
it("bounds GitHub response authorization reads with large saved chat content", async () => {
|
||||
const f = await toolReplyFixture();
|
||||
const [delivery] = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, f.endpoint.id));
|
||||
const [resource] = await db.select().from(chatEndpointResources).where(eq(chatEndpointResources.id, f.conversation.resourceId!));
|
||||
const [configuration] = await db.select().from(chatGitHubConfigurations).where(eq(chatGitHubConfigurations.endpointId, f.endpoint.id));
|
||||
const bulk = "unused-content-for-response-authorization".repeat(8_192);
|
||||
const setup = { ...f.endpoint.setup, unusedContent: bulk };
|
||||
await db.update(chatEndpoints).set({ communicationInstructions: bulk, setup }).where(eq(chatEndpoints.id, f.endpoint.id));
|
||||
await db.update(chatConversations).set({ communicationGuidance: bulk }).where(eq(chatConversations.id, f.conversation.id));
|
||||
await db.update(chatEndpointResources).set({ metadata: { ...resource.metadata, unusedContent: bulk } })
|
||||
.where(eq(chatEndpointResources.id, resource.id));
|
||||
await db.update(chatGitHubConfigurations).set({ configuration: { ...configuration.configuration,
|
||||
defaults: { ...configuration.configuration.defaults, instructions: bulk } } })
|
||||
.where(eq(chatGitHubConfigurations.endpointId, f.endpoint.id));
|
||||
const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk };
|
||||
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations,
|
||||
chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks,
|
||||
companyMemberships, chatActions, heartbeatRuns }, capture);
|
||||
try {
|
||||
const receipt = await githubResponseCommentService(db, f.providerFetch).tryAcknowledge(delivery.id);
|
||||
expect(receipt?.id).toBeTruthy();
|
||||
} finally { capture.flush(); }
|
||||
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(24_000);
|
||||
expect(events.filter(event => event.table === "chat_message_links").length).toBeGreaterThan(0);
|
||||
expect((await db.select({ normalizedEvent: chatDeliveries.normalizedEvent }).from(chatDeliveries)
|
||||
.where(eq(chatDeliveries.id, delivery.id)))[0].normalizedEvent).toEqual(normalizedEvent);
|
||||
expect((await db.select({ setup: chatEndpoints.setup }).from(chatEndpoints)
|
||||
.where(eq(chatEndpoints.id, f.endpoint.id)))[0].setup).toEqual(setup);
|
||||
});
|
||||
|
||||
it.each(["githubAutomatic", "githubIssue"])("preserves %s policy context while omitting body content", async key => {
|
||||
const f = await toolReplyFixture();
|
||||
const [delivery] = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, f.endpoint.id));
|
||||
const context = { event: key === "githubIssue" ? "issue_opened" : "opened", repositoryId: "418", repository: "acme/repo",
|
||||
author: { id: "42", login: "octocat", isBot: false }, sender: { id: "42", login: "octocat" },
|
||||
labels: ["review"], draft: false, baseBranch: "master", headSha: "a".repeat(40),
|
||||
body: "unused-body".repeat(32_768), title: "unused-title".repeat(8_192) };
|
||||
const normalizedEvent = { text: "unrelated message text", [key]: { context, revision: 42,
|
||||
policy: { instructions: "unused-policy-instructions".repeat(32_768),
|
||||
prompts: { opened: "unused-policy-prompt".repeat(32_768) } },
|
||||
} };
|
||||
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatDeliveries }, capture);
|
||||
let projected: { event: Record<string, unknown> };
|
||||
try {
|
||||
[projected] = await db.select({ event: githubResponseAuthorityFields.delivery.normalizedEvent })
|
||||
.from(chatDeliveries).where(eq(chatDeliveries.id, delivery.id));
|
||||
} finally { capture.flush(); }
|
||||
const { body: _body, title: _title, ...policy } = context;
|
||||
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(2_048);
|
||||
expect(projected.event).toEqual({ [key]: { context: policy } });
|
||||
expect((await db.select({ event: chatDeliveries.normalizedEvent }).from(chatDeliveries)
|
||||
.where(eq(chatDeliveries.id, delivery.id)))[0].event).toEqual(normalizedEvent);
|
||||
});
|
||||
|
||||
it.each([null, false, 7, "invalid-event", []])("preserves malformed GitHub authority input %j", async value => {
|
||||
const f = await toolReplyFixture();
|
||||
const [delivery] = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, f.endpoint.id));
|
||||
await db.update(chatDeliveries).set({ normalizedEvent: sql`${JSON.stringify(value)}::jsonb` })
|
||||
.where(eq(chatDeliveries.id, delivery.id));
|
||||
await db.update(chatEndpoints).set({ setup: sql`${JSON.stringify(value)}::jsonb` })
|
||||
.where(eq(chatEndpoints.id, f.endpoint.id));
|
||||
const [projected] = await db.select({ event: githubResponseAuthorityFields.delivery.normalizedEvent })
|
||||
.from(chatDeliveries).where(eq(chatDeliveries.id, delivery.id));
|
||||
const [endpoint] = await db.select({ setup: githubResponseAuthorityFields.endpoint.setup })
|
||||
.from(chatEndpoints).where(eq(chatEndpoints.id, f.endpoint.id));
|
||||
expect(projected.event).toEqual(value);
|
||||
expect(endpoint.setup).toEqual(value);
|
||||
});
|
||||
|
||||
async function publicationRetryFixture() {
|
||||
const f = await toolReplyFixture("github:paperclipai/paperclip:418", true);
|
||||
const head = "b".repeat(40);
|
||||
@@ -5135,13 +5212,22 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
(await f.service.get(f.endpoint.id)).setup.webhookVerifiedAt,
|
||||
).toBeNull();
|
||||
expect(f.wakeup).not.toHaveBeenCalled();
|
||||
event.payload.githubApp.headers["x-hub-signature-256"] =
|
||||
`sha256=${createHmac("sha256", f.webhookSecret).update(body).digest("hex")}`;
|
||||
expect(await dispatchGitHubBotCloudEvent(db, event)).toBe(true);
|
||||
expect(
|
||||
(await f.service.get(f.endpoint.id)).setup.webhookVerifiedAt,
|
||||
).toBeTruthy();
|
||||
expect(f.wakeup).not.toHaveBeenCalled();
|
||||
const poll = vi.fn(async () => { await dispatchGitHubBotCloudEvent(db, event); });
|
||||
const unregister = registerGitHubBotCloudSetupPoller(db, poll);
|
||||
try {
|
||||
const waiting = await f.service.githubWizard.advance(f.endpoint.id, "owner-user");
|
||||
expect(waiting.state).toBe("verify");
|
||||
expect(poll).toHaveBeenCalledOnce();
|
||||
expect((await f.service.get(f.endpoint.id)).setup.webhookVerifiedAt).toBeNull();
|
||||
event.payload.githubApp.headers["x-hub-signature-256"] =
|
||||
`sha256=${createHmac("sha256", f.webhookSecret).update(body).digest("hex")}`;
|
||||
const ready = await f.service.githubWizard.advance(f.endpoint.id, "owner-user");
|
||||
expect(ready.state).not.toBe("verify");
|
||||
expect((await f.service.get(f.endpoint.id)).setup.webhookVerifiedAt).toBeTruthy();
|
||||
expect(f.wakeup).not.toHaveBeenCalled();
|
||||
await f.service.githubWizard.advance(f.endpoint.id, "owner-user");
|
||||
expect(poll).toHaveBeenCalledTimes(2);
|
||||
} finally { unregister(); }
|
||||
});
|
||||
it("legacy name-only registration preserves an explicit empty repository selection", async () => {
|
||||
const fixture = await seedCompany();
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
projectWorkspaces,
|
||||
} from "@paperclipai/db";
|
||||
import { parseFrontmatterMarkdown } from "@paperclipai/shared";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
@@ -722,6 +723,73 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
expect(refreshed.filter((skill) => skill.key === key)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("validates warm bundled releases without returning their stored file bodies", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Release read checks", issuePrefix: "RRC" });
|
||||
await svc.list(companyId);
|
||||
const expected = await svc.list(companyId);
|
||||
const before = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId))
|
||||
.orderBy(companySkillVersions.id);
|
||||
expect(before.length).toBeGreaterThan(0);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
|
||||
try { expect(await companySkillService(db).list(companyId)).toEqual(expected); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& event.table === "company_skill_versions" && event.operation === "select");
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
expect(await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId))
|
||||
.orderBy(companySkillVersions.id)).toEqual(before);
|
||||
});
|
||||
|
||||
it.each(["agent", "board"] as const)("checks %s company access before using a warm release validation", async type => {
|
||||
const companyId = randomUUID(), otherCompanyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Release access checks", issuePrefix: "RAC" });
|
||||
await svc.list(companyId);
|
||||
await svc.list(companyId);
|
||||
const actor = type === "agent"
|
||||
? { type, companyId: otherCompanyId, agentId: randomUUID() }
|
||||
: { type, companyIds: [otherCompanyId] };
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
|
||||
try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); }
|
||||
finally { capture.flush(); }
|
||||
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0);
|
||||
});
|
||||
|
||||
it("restores a deleted release after warm validation", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Release deletion checks", issuePrefix: "RDC" });
|
||||
await svc.list(companyId);
|
||||
await svc.list(companyId);
|
||||
const before = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId));
|
||||
const original = before[0]!;
|
||||
await db.delete(companySkillVersions).where(eq(companySkillVersions.id, original.id));
|
||||
await companySkillService(db).list(companyId);
|
||||
const after = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId));
|
||||
expect(after).toHaveLength(before.length);
|
||||
expect(after.find(version => version.releaseId === original.releaseId)).toMatchObject({
|
||||
companyId, companySkillId: original.companySkillId, fileInventory: original.fileInventory,
|
||||
});
|
||||
expect(after.some(version => version.id === original.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects a changed bundled snapshot after warm validation without overwriting it", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Release integrity checks", issuePrefix: "RIC" });
|
||||
await svc.list(companyId);
|
||||
await svc.list(companyId);
|
||||
const [version] = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId));
|
||||
const changed = [{ path: "SKILL.md", kind: "skill" as const, content: "Changed stored snapshot" }];
|
||||
await db.update(companySkillVersions).set({ fileInventory: changed }).where(eq(companySkillVersions.id, version!.id));
|
||||
await expect(companySkillService(db).list(companyId)).rejects.toThrow("does not match its seeded snapshot");
|
||||
expect((await db.select().from(companySkillVersions).where(eq(companySkillVersions.id, version!.id)))[0]!.fileInventory)
|
||||
.toEqual(changed);
|
||||
});
|
||||
|
||||
it("seeds bundled skill releases idempotently and materializes the frozen champion snapshot", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { settleConversationTurnForRun } from "../services/agent-conversations.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("conversation settlement transfer", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-conversation-settlement-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 30_000);
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 5 }); await database?.cleanup(); }, 60_000);
|
||||
|
||||
async function seed(reply: "response" | "interaction" | "reset" | "none" = "response") {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), wakeId = randomUUID();
|
||||
const padding = "saved conversation ".repeat(32_768);
|
||||
await db.insert(companies).values({ id: companyId, name: "Settlement transfer", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Conversation agent", role: "engineer" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Conversation", description: padding,
|
||||
status: "in_progress", conversationState: "active", conversationUserId: "conversation-owner",
|
||||
conversationAgentId: agentId, assigneeAgentId: agentId, conversationSessionGeneration: 3 });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded",
|
||||
contextSnapshot: { issueId, conversationSessionGeneration: 3, wakeCommentId: wakeId,
|
||||
conversationReset: reply === "reset", savedInput: padding },
|
||||
runnerProfileJson: { savedProfile: padding }, resultJson: { savedResult: padding },
|
||||
stdoutExcerpt: padding, stderrExcerpt: padding });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const wakeTime = new Date(Date.now() - 10_000);
|
||||
await db.insert(issueComments).values({ id: wakeId, companyId, issueId, authorUserId: "conversation-owner",
|
||||
body: padding, createdAt: wakeTime });
|
||||
if (reply === "response") await db.insert(issueComments).values({ companyId, issueId,
|
||||
createdByRunId: runId, authorAgentId: agentId, origin: "comment", body: padding });
|
||||
if (reply === "interaction") await db.insert(issueThreadInteractions).values({ companyId, issueId,
|
||||
sourceRunId: runId, createdByAgentId: agentId, status: "pending", kind: "request_confirmation", payload: { version: 1 } });
|
||||
return { companyId, agentId, issueId, runId, wakeId, wakeTime };
|
||||
}
|
||||
|
||||
async function sourceDigests(input: Awaited<ReturnType<typeof seed>>) {
|
||||
return db.execute(sql`select
|
||||
(select encode(sha256(convert_to(jsonb_build_array(context_snapshot,runner_profile_json,result_json,stdout_excerpt,stderr_excerpt)::text,'UTF8')),'hex')
|
||||
from heartbeat_runs where id=${input.runId}::uuid) as run_digest,
|
||||
(select encode(sha256(convert_to(description,'UTF8')),'hex') from issues where id=${input.issueId}::uuid) as description_digest,
|
||||
(select encode(sha256(convert_to(jsonb_agg(jsonb_build_array(id,body) order by id)::text,'UTF8')),'hex')
|
||||
from issue_comments where issue_id=${input.issueId}::uuid) as comments_digest`);
|
||||
}
|
||||
|
||||
it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => {
|
||||
const input = await seed(reply), before = await sourceDigests(input);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture);
|
||||
try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table))
|
||||
&& (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384);
|
||||
const [issue] = await db.select({ status: issues.status, state: issues.conversationState }).from(issues).where(eq(issues.id, input.issueId));
|
||||
expect(issue).toEqual({ status: "in_review", state: "waiting" });
|
||||
expect(await sourceDigests(input)).toEqual(before);
|
||||
});
|
||||
|
||||
it.each(["conversationReplyBoundaryCommentId", "wakeCommentId", "commentId"] as const)("keeps later user messages active through %s", async key => {
|
||||
const input = await seed();
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: input.issueId,
|
||||
conversationSessionGeneration: 3, [key]: input.wakeId } }).where(eq(heartbeatRuns.id, input.runId));
|
||||
await db.insert(issueComments).values({ companyId: input.companyId, issueId: input.issueId,
|
||||
authorUserId: "conversation-owner", body: "A new instruction", createdAt: new Date(input.wakeTime.getTime() + 1000) });
|
||||
expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true);
|
||||
const [issue] = await db.select({ status: issues.status, state: issues.conversationState }).from(issues).where(eq(issues.id, input.issueId));
|
||||
expect(issue).toEqual({ status: "in_progress", state: "active" });
|
||||
});
|
||||
|
||||
it.each(["missing", "company", "issue-company", "generation", "owner", "failed", "no-reply"] as const)("refuses to settle a %s mismatch", async fault => {
|
||||
const input = await seed(fault === "no-reply" ? "none" : "response");
|
||||
if (fault === "generation") await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: input.issueId,
|
||||
conversationSessionGeneration: 2, wakeCommentId: input.wakeId } }).where(eq(heartbeatRuns.id, input.runId));
|
||||
if (fault === "owner") {
|
||||
const otherRun = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: otherRun, companyId: input.companyId, agentId: input.agentId });
|
||||
await db.update(issues).set({ executionRunId: otherRun }).where(eq(issues.id, input.issueId));
|
||||
}
|
||||
if (fault === "failed") await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, input.runId));
|
||||
if (fault === "issue-company") {
|
||||
const otherCompany = randomUUID(), otherIssue = randomUUID(), otherAgent = randomUUID();
|
||||
await db.insert(companies).values({ id: otherCompany, name: "Other company", issuePrefix: otherCompany.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: otherAgent, companyId: otherCompany, name: "Other agent" });
|
||||
await db.insert(issues).values({ id: otherIssue, companyId: otherCompany, title: "Other conversation",
|
||||
conversationUserId: "other-owner", conversationAgentId: otherAgent, assigneeAgentId: otherAgent,
|
||||
conversationState: "active", conversationSessionGeneration: 3 });
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: otherIssue, conversationSessionGeneration: 3,
|
||||
conversationReset: true } }).where(eq(heartbeatRuns.id, input.runId));
|
||||
}
|
||||
expect(await settleConversationTurnForRun(db, fault === "company" ? randomUUID() : input.companyId,
|
||||
fault === "missing" ? randomUUID() : input.runId)).toBe(false);
|
||||
const [issue] = await db.select({ status: issues.status }).from(issues).where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId)));
|
||||
expect(issue.status).toBe("in_progress");
|
||||
});
|
||||
});
|
||||
@@ -92,6 +92,7 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
|
||||
REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
|
||||
listReadyPluginEnvironmentDrivers: vi.fn(async () => []),
|
||||
resolvePluginSandboxProviderDriverByKey: vi.fn(async () => null),
|
||||
resolvePluginSandboxProviderConfigSchema: vi.fn(async () => null),
|
||||
validatePluginEnvironmentDriverConfig: vi.fn(async ({ config }) => config),
|
||||
validatePluginSandboxProviderConfig: vi.fn(async ({ provider, config }) => ({
|
||||
normalizedConfig: config,
|
||||
|
||||
@@ -79,6 +79,7 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
|
||||
REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
|
||||
listReadyPluginEnvironmentDrivers: vi.fn(async () => []),
|
||||
resolvePluginSandboxProviderDriverByKey: vi.fn(async () => null),
|
||||
resolvePluginSandboxProviderConfigSchema: vi.fn(async () => null),
|
||||
validatePluginEnvironmentDriverConfig: vi.fn(async ({ config }) => config),
|
||||
validatePluginSandboxProviderConfig: vi.fn(async ({ provider, config }) => ({
|
||||
normalizedConfig: config,
|
||||
|
||||
@@ -25,6 +25,8 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
|
||||
probePluginEnvironmentDriver: mockProbePluginEnvironmentDriver,
|
||||
probePluginSandboxProviderDriver: mockProbePluginSandboxProviderDriver,
|
||||
resolvePluginSandboxProviderDriverByKey: mockResolvePluginSandboxProviderDriverByKey,
|
||||
resolvePluginSandboxProviderConfigSchema: async (input: { db: unknown; driverKey: string }) =>
|
||||
(await mockResolvePluginSandboxProviderDriverByKey(input))?.driver?.configSchema ?? null,
|
||||
}));
|
||||
|
||||
vi.mock("../services/environment-runtime.js", () => ({
|
||||
|
||||
@@ -129,6 +129,8 @@ vi.mock("../services/plugin-environment-driver.js", () => ({
|
||||
REUSABLE_LEASE_WORKER_METHODS: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
|
||||
listReadyPluginEnvironmentDrivers: mockListReadyPluginEnvironmentDrivers,
|
||||
resolvePluginSandboxProviderDriverByKey: mockResolvePluginSandboxProviderDriverByKey,
|
||||
resolvePluginSandboxProviderConfigSchema: async (input: { db: unknown; driverKey: string }) =>
|
||||
(await mockResolvePluginSandboxProviderDriverByKey(input))?.driver?.configSchema ?? null,
|
||||
startPluginEnvironmentInteractiveSetup: mockStartPluginEnvironmentInteractiveSetup,
|
||||
getPluginEnvironmentInteractiveSetup: mockGetPluginEnvironmentInteractiveSetup,
|
||||
capturePluginEnvironmentTemplate: mockCapturePluginEnvironmentTemplate,
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
GIT_CREDENTIAL_TOKEN_ENV_KEY,
|
||||
buildGitAuthInvocation,
|
||||
createGitRemoteAuthProvider,
|
||||
createIssueGitRemoteAuthProvider,
|
||||
describeGitAuthFailure,
|
||||
isGitHubHttpsRemoteUrl,
|
||||
scrubGitCredentialText,
|
||||
@@ -379,3 +380,39 @@ describe("DEFAULT_GITHUB_TOKEN_SECRET_NAMES", () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createIssueGitRemoteAuthProvider", () => {
|
||||
const githubUrl = "https://github.com/paperclipai/private-repo.git";
|
||||
function issueDb(rows: unknown[]) {
|
||||
const where = vi.fn(async () => rows);
|
||||
return { db: { select: vi.fn(() => ({ from: () => ({ where }) })) } as unknown as Db, where };
|
||||
}
|
||||
|
||||
it("resolves credentials with the issue's responsible user and assigned agent", async () => {
|
||||
const { db } = issueDb([{ responsibleUserId: "user-1", assigneeAgentId: "agent-1" }]);
|
||||
const inner = vi.fn(async () => ({ env: {}, configArgs: [], source: "managed_connection" }));
|
||||
const create = vi.fn(() => inner) as unknown as typeof createGitRemoteAuthProvider;
|
||||
const provider = createIssueGitRemoteAuthProvider(db, "company-1", "issue-1", create);
|
||||
await expect(provider(githubUrl)).resolves.toMatchObject({ source: "managed_connection" });
|
||||
expect(create).toHaveBeenCalledWith(db, "company-1", { issueId: "issue-1", responsibleUserId: "user-1", agentId: "agent-1" });
|
||||
expect(inner).toHaveBeenCalledWith(githubUrl);
|
||||
});
|
||||
|
||||
it("looks the issue up once across calls", async () => {
|
||||
const { db } = issueDb([{ responsibleUserId: "user-1", assigneeAgentId: null }]);
|
||||
const inner = vi.fn(async () => null);
|
||||
const create = vi.fn(() => inner) as unknown as typeof createGitRemoteAuthProvider;
|
||||
const provider = createIssueGitRemoteAuthProvider(db, "company-1", "issue-1", create);
|
||||
await provider(githubUrl); await provider(githubUrl);
|
||||
expect(db.select).toHaveBeenCalledTimes(1);
|
||||
expect(create).toHaveBeenCalledTimes(1);
|
||||
expect(inner).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("falls back to the issue-only context when the issue is missing", async () => {
|
||||
const { db } = issueDb([]);
|
||||
const create = vi.fn(() => vi.fn(async () => null)) as unknown as typeof createGitRemoteAuthProvider;
|
||||
await createIssueGitRemoteAuthProvider(db, "company-1", "issue-1", create)(githubUrl);
|
||||
expect(create).toHaveBeenCalledWith(db, "company-1", { issueId: "issue-1", responsibleUserId: null, agentId: null });
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
connectionGrants,
|
||||
createDb,
|
||||
externalObjects,
|
||||
EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
|
||||
toolApplications,
|
||||
toolConnections,
|
||||
} from "@paperclipai/db";
|
||||
@@ -30,7 +31,7 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => {
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-github-events-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(activityLog);
|
||||
@@ -46,7 +47,7 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
it("applies a normalized merged PR once while retaining undecryptable siblings for retry", async () => {
|
||||
it.each([false, true])("applies a normalized merged PR once while retaining undecryptable siblings for retry (active setup=%s)", async activeSetup => {
|
||||
const companyId = randomUUID();
|
||||
const userId = `github-owner-${randomUUID()}`;
|
||||
const applicationId = randomUUID();
|
||||
@@ -151,8 +152,21 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => {
|
||||
let currentTime = new Date("2026-09-04T12:00:05.000Z");
|
||||
const service = githubConnectionEventService(db, { connector, now: () => currentTime });
|
||||
|
||||
if (activeSetup) {
|
||||
// An existing personal connection can leave a long empty-queue backoff
|
||||
// before a new dedicated App starts its signed-ping verification.
|
||||
for (const delay of [0, 10_000, 20_000, 40_000, 80_000]) {
|
||||
currentTime = new Date(currentTime.getTime() + delay);
|
||||
vi.mocked(connector.leaseEvents).mockResolvedValueOnce(null);
|
||||
await service.pollOnce();
|
||||
}
|
||||
currentTime = new Date(currentTime.getTime() + 5_000);
|
||||
await expect(service.pollOnce()).resolves.toMatchObject({ leased: 0 });
|
||||
expect(connector.leaseEvents).toHaveBeenCalledTimes(5);
|
||||
}
|
||||
|
||||
const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
|
||||
await expect(service.pollOnce()).resolves.toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 });
|
||||
await expect(service.pollOnce({ forSetup: activeSetup })).resolves.toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 });
|
||||
expect(warning.mock.calls).toEqual([[{
|
||||
deliveryId: "undecryptable-delivery", reason: "inner_event_authentication_failed",
|
||||
}, "GitHub connector event retained for retry"]]);
|
||||
|
||||
@@ -6,6 +6,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
@@ -31,6 +32,7 @@ import {
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import {
|
||||
initializeRunIdentity,
|
||||
captureRunIdentity,
|
||||
reserveSteeredIdentity,
|
||||
acceptSteeredIdentity,
|
||||
} from "../services/run-identity.js";
|
||||
@@ -192,6 +194,72 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
});
|
||||
return { id, connectionId, secretId, definitionId };
|
||||
}
|
||||
it.each(["config", "transportConfig"] as const)("reads only current GitHub identity connections via %s", async location => {
|
||||
const input = await seed(), selected = await grant(input, "A");
|
||||
const [connection] = await db.select().from(toolConnections).where(eq(toolConnections.id, selected.connectionId));
|
||||
await db.update(toolConnections).set({ config: {}, transportConfig: {}, [location]: { sourceTemplateKey: "github" } })
|
||||
.where(eq(toolConnections.id, selected.connectionId));
|
||||
const payload = "unrelated-connection-config🙂".repeat(10_000);
|
||||
const irrelevant = [null, true, 42, ["github"], { name: "github" }, "GitHub", " github ", "other"].map(sourceTemplateKey => {
|
||||
const id = randomUUID();
|
||||
return { id, companyId: input.companyId, applicationId: connection.applicationId, name: id, uid: id,
|
||||
transport: "mcp_remote" as const, config: { sourceTemplateKey, payload }, transportConfig: { sourceTemplateKey } };
|
||||
});
|
||||
await db.insert(toolConnections).values(irrelevant);
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { toolConnections }, capture);
|
||||
try {
|
||||
expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" }))
|
||||
.toMatchObject({ configured: true, grant: { id: selected.id } });
|
||||
capture.flush();
|
||||
const reads = events.filter(event => event.table === "tool_connections" && event.operation === "select");
|
||||
expect(reads).toHaveLength(1);
|
||||
expect(reads[0].rows).toBe(1);
|
||||
expect(reads[0].responseValueBytes).toBeLessThan(8192);
|
||||
await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, selected.connectionId));
|
||||
expect((await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" })).grant)
|
||||
.toBeUndefined();
|
||||
await db.update(toolConnections).set({ [location]: { sourceTemplateKey: "other" } }).where(eq(toolConnections.id, selected.connectionId));
|
||||
expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" }))
|
||||
.toEqual({ configured: false });
|
||||
expect(await resolveManagedGitHubIdentitySelection(observed, randomUUID(), { agentId: input.agentId, responsibleUserId: "A" }))
|
||||
.toEqual({ configured: false });
|
||||
const saved = await db.select().from(toolConnections).where(eq(toolConnections.companyId, input.companyId));
|
||||
for (const row of irrelevant) expect(saved.find(candidate => candidate.id === row.id)?.config).toEqual(row.config);
|
||||
} finally { capture.flush(); await observed.$client.end(); }
|
||||
});
|
||||
|
||||
it("resolves current GitHub credentials without transferring execution snapshots", async () => {
|
||||
const input = await seed(), selected = await grant(input, "A");
|
||||
const payload = "github-credential-saved-input🙂".repeat(25_000);
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: input.issueId, prompt: payload },
|
||||
runnerProfileJson: { nativeExecutionInput: { savedInput: payload }, savedInput: payload },
|
||||
resultJson: { stdout: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
|
||||
}).where(eq(heartbeatRuns.id, input.runId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({
|
||||
status: "available", login: "A", grantId: selected.id,
|
||||
});
|
||||
capture.flush();
|
||||
const reads = events.filter(event => event.table === "heartbeat_runs" && event.operation === "select");
|
||||
expect(reads.length).toBeGreaterThan(0);
|
||||
expect(reads.reduce((bytes, event) => bytes + Number(event.responseValueBytes), 0)).toBeLessThan(8192);
|
||||
expect(JSON.stringify(events)).not.toContain("github-credential-saved-input");
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)))[0]).toEqual(before);
|
||||
const full = await captureRunIdentity(observed, input);
|
||||
expect(full.run.runnerProfileJson).toEqual(before.runnerProfileJson);
|
||||
expect(full.run.resultJson).toEqual(before.resultJson);
|
||||
expect(full.run.contextSnapshot).toEqual(before.contextSnapshot);
|
||||
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, selected.id));
|
||||
expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({ status: "unavailable" });
|
||||
} finally { capture.flush(); await observed.$client.end(); }
|
||||
});
|
||||
|
||||
async function switchTo(
|
||||
input: Awaited<ReturnType<typeof seed>>,
|
||||
user: string,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
@@ -15,6 +16,62 @@ import {
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
describe("P6-11..13 / P6-17 canonical event allocator", () => {
|
||||
it("keeps native cursor responses bounded with long histories and a late gap fill", async () => {
|
||||
const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-cursor-budget-");
|
||||
const db = createDb(temporary.connectionString);
|
||||
const companyId = "20000000-0000-4000-8000-000000000001";
|
||||
const agentId = "20000000-0000-4000-8000-000000000002";
|
||||
const runId = "20000000-0000-4000-8000-000000000003";
|
||||
const otherRunId = "20000000-0000-4000-8000-000000000004";
|
||||
try {
|
||||
await db.insert(companies).values({ id: companyId, name: "Cursor budget", issuePrefix: "CBG" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Cursor agent" });
|
||||
await db.insert(heartbeatRuns).values([
|
||||
{ id: runId, companyId, agentId, status: "running", nextEventSeq: 5005 },
|
||||
{ id: otherRunId, companyId, agentId, status: "running" },
|
||||
]);
|
||||
for (let batch = 0; batch < 10; batch += 1) {
|
||||
await db.insert(heartbeatRunEvents).values(Array.from({ length: 500 }, (_, index) => {
|
||||
const seq = batch * 500 + index + 1;
|
||||
return { companyId, agentId, runId, seq, eventType: "item.delta", sourceInstanceId: "runner-a",
|
||||
sourceSeq: seq, sourceEventId: `seed:${seq}` };
|
||||
}));
|
||||
}
|
||||
await db.insert(heartbeatRunEvents).values([
|
||||
...[-1, 0, null].map((sourceSeq, index) => ({
|
||||
companyId, agentId, runId, seq: 5001 + index, eventType: "item.delta",
|
||||
sourceInstanceId: "runner-a", sourceSeq, sourceEventId: `legacy:${index}`,
|
||||
})),
|
||||
{ companyId, agentId, runId, seq: 5004, eventType: "item.delta", sourceInstanceId: "runner-b",
|
||||
sourceSeq: 5001, sourceEventId: "other-source:5001" },
|
||||
{ companyId, agentId, runId: otherRunId, seq: 1, eventType: "item.delta", sourceInstanceId: "runner-a",
|
||||
sourceSeq: 5001, sourceEventId: "other-run:5001" },
|
||||
]);
|
||||
const append = (sourceSeq: number) => appendHeartbeatRunEvent(db, {
|
||||
companyId, agentId, runId, eventType: "item.delta", payload: { sourceSeq },
|
||||
nativeSource: { sourceInstanceId: "runner-a", sourceSeq, sourceEventId: `runner-a:${sourceSeq}`,
|
||||
protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } },
|
||||
});
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture);
|
||||
try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(2048);
|
||||
expect((await append(5001)).highestContiguousSourceSeq).toBe(5002);
|
||||
const replay = await append(5002);
|
||||
expect(replay.disposition).toBe("duplicate");
|
||||
expect(replay.highestContiguousSourceSeq).toBe(5002);
|
||||
const [remaining] = await db.select({ count: sql<number>`count(*)::int` }).from(heartbeatRunEvents)
|
||||
.where(eq(heartbeatRunEvents.runId, runId));
|
||||
expect(remaining?.count).toBe(5006);
|
||||
} finally { await temporary.cleanup(); }
|
||||
}, 60_000);
|
||||
|
||||
it.each([false, true])("deduplicates concurrent retry exhaustion (historical receipt: %s)", async (historical) => {
|
||||
const temporary = await startEmbeddedPostgresTestDatabase("paperclip-exhaustion-events-");
|
||||
const db = createDb(temporary.connectionString);
|
||||
@@ -165,4 +222,46 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => {
|
||||
await temporary.cleanup();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it("reports the highest contiguous source seq per source instance across gaps and late arrivals", async () => {
|
||||
const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-cursor-");
|
||||
const db = createDb(temporary.connectionString);
|
||||
const companyId = "20000000-0000-4000-8000-000000000001";
|
||||
const agentId = "20000000-0000-4000-8000-000000000002";
|
||||
const runId = "20000000-0000-4000-8000-000000000003";
|
||||
try {
|
||||
await db.insert(companies).values({ id: companyId, name: "Cursor fixture", issuePrefix: "CUR" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Cursor agent" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" });
|
||||
const append = async (sourceInstanceId: string, sourceSeq: number) => (await appendHeartbeatRunEvent(db, {
|
||||
companyId,
|
||||
runId,
|
||||
agentId,
|
||||
eventType: "item.delta",
|
||||
payload: { sourceInstanceId, sourceSeq },
|
||||
nativeSource: {
|
||||
sourceInstanceId,
|
||||
sourceEventId: `${sourceInstanceId}:${sourceSeq}`,
|
||||
sourceSeq,
|
||||
protocolSchemaVersion: 1,
|
||||
canonicalPayload: { sourceInstanceId, sourceSeq },
|
||||
},
|
||||
})).highestContiguousSourceSeq;
|
||||
|
||||
expect(await append("runner-a", 2)).toBe(0);
|
||||
expect(await append("runner-a", 1)).toBe(2);
|
||||
expect(await append("runner-a", 3)).toBe(3);
|
||||
expect(await append("runner-a", 5)).toBe(3);
|
||||
expect(await append("runner-a", 6)).toBe(3);
|
||||
expect(await append("runner-b", 1)).toBe(1);
|
||||
expect(await append("runner-a", 4)).toBe(6);
|
||||
for (let sourceSeq = 7; sourceSeq <= 200; sourceSeq += 1) {
|
||||
expect(await append("runner-a", sourceSeq)).toBe(sourceSeq);
|
||||
}
|
||||
expect(await append("runner-b", 3)).toBe(1);
|
||||
expect(await append("runner-b", 2)).toBe(3);
|
||||
} finally {
|
||||
await temporary.cleanup();
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
@@ -257,6 +258,29 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
return app;
|
||||
}
|
||||
|
||||
it("lists workspace operations with small run reads and checks a revoked issue grant again", async () => {
|
||||
const f = await seedFixture(), payload = "workspace-operation-saved-input🙂".repeat(25_000);
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, prompt: payload },
|
||||
runnerProfileJson: { instructions: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
|
||||
}).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const app = createApp(f.companyId, f.grantedAgentId, observed);
|
||||
const response = await request(app).get(`/api/heartbeat-runs/${f.privateRunId}/workspace-operations`).expect(200);
|
||||
expect(response.body).toEqual([expect.objectContaining({ id: f.operationId })]);
|
||||
capture.flush();
|
||||
const reads = events.filter(event => event.table === "heartbeat_runs" && event.operation === "select");
|
||||
expect(reads.length).toBeGreaterThan(0);
|
||||
expect(reads.reduce((bytes, event) => bytes + Number(event.responseValueBytes), 0)).toBeLessThan(2048);
|
||||
await db.delete(issueAccessGrants).where(eq(issueAccessGrants.subjectId, f.grantedAgentId));
|
||||
await request(app).get(`/api/heartbeat-runs/${f.privateRunId}/workspace-operations`).expect(404);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId)))[0]).toEqual(before);
|
||||
} finally { capture.flush(); await observed.$client.end(); }
|
||||
});
|
||||
|
||||
it("returns 404 for direct private-run content and metadata-only list stubs to a non-member", async () => {
|
||||
const fixture = await seedFixture();
|
||||
const app = createApp(fixture.companyId, fixture.otherAgentId);
|
||||
@@ -459,6 +483,105 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce";
|
||||
}
|
||||
});
|
||||
it("batches distinct issue bindings without changing current authorization decisions", async () => {
|
||||
const fixture = await seedFixture();
|
||||
const ids = Array.from({ length: 64 }, () => randomUUID());
|
||||
await db.insert(issues).values(ids.map((id, index) => ({
|
||||
id, companyId: fixture.companyId, title: `History ${index}`, status: "in_progress",
|
||||
visibility: index % 2 ? "private" : "open", privacyRootIssueId: index % 2 ? id : null,
|
||||
assigneeAgentId: fixture.ownerAgentId, responsibleUserId: fixture.ownerUserId,
|
||||
})));
|
||||
const queries: string[] = [];
|
||||
const observedDb = drizzle(db.$client, { schema, logger: { logQuery(query) { queries.push(query); } } }) as typeof db;
|
||||
const access = authorizationService(observedDb);
|
||||
const actor = { type: "agent" as const, companyId: fixture.companyId, agentId: fixture.grantedAgentId };
|
||||
const bindings = [...ids, fixture.issueId].map(issueId => ({ companyId: fixture.companyId, scopeKind: "issue" as const, issueId }));
|
||||
const runs = [...bindings, ...bindings, { ...bindings[0], companyId: randomUUID() },
|
||||
{ ...bindings[0], issueId: randomUUID() }, { companyId: fixture.companyId, scopeKind: "company" as const },
|
||||
{ companyId: fixture.companyId, scopeKind: "issue" as const }];
|
||||
const individual = await getHeartbeatRunReadDecisions(observedDb, { decide: access.decide }, actor, runs);
|
||||
const individualCount = queries.length;
|
||||
queries.length = 0;
|
||||
const batched = await getHeartbeatRunReadDecisions(observedDb, access, actor, runs);
|
||||
expect(batched).toEqual(individual);
|
||||
expect(batched.slice(0, 64)).toEqual(ids.map((_, index) => index % 2 === 0));
|
||||
expect(batched.at(64)).toBe(true);
|
||||
expect(batched.slice(-4)).toEqual([false, false, true, false]);
|
||||
expect(queries.length).toBeLessThan(25);
|
||||
expect(queries.length).toBeLessThan(individualCount / 4);
|
||||
await db.update(issueAccessGrants).set({ revokedAt: new Date() }).where(eq(issueAccessGrants.subjectId, fixture.grantedAgentId));
|
||||
expect((await getHeartbeatRunReadDecisions(observedDb, access, actor, runs)).at(64)).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["enforce", "shadow", "off"])("keeps missing and foreign issue bindings closed in %s mode", async mode => {
|
||||
const fixture = await seedFixture();
|
||||
process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = mode;
|
||||
try {
|
||||
const access = authorizationService(db);
|
||||
const actor = { type: "board" as const, userId: fixture.ownerUserId };
|
||||
const binding = { companyId: fixture.companyId, scopeKind: "issue" as const, issueId: fixture.issueId };
|
||||
const runs = [binding, { ...binding, issueId: randomUUID() }, { ...binding, companyId: randomUUID() }];
|
||||
const expected = await getHeartbeatRunReadDecisions(db, { decide: access.decide }, actor, runs);
|
||||
expect(await getHeartbeatRunReadDecisions(db, access, actor, runs)).toEqual(expected);
|
||||
expect(expected).toEqual([true, false, false]);
|
||||
await db.delete(companyMemberships).where(eq(companyMemberships.principalId, fixture.ownerUserId));
|
||||
expect(await getHeartbeatRunReadDecisions(db, access, actor, [binding])).toEqual(
|
||||
await getHeartbeatRunReadDecisions(db, { decide: access.decide }, actor, [binding]),
|
||||
);
|
||||
} finally { process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce"; }
|
||||
});
|
||||
|
||||
it("checks more than 500 task bindings with bounded company-scoped queries", async () => {
|
||||
const fixture = await seedFixture();
|
||||
const ids = Array.from({ length: 501 }, () => randomUUID());
|
||||
await db.insert(issues).values(ids.map((id, index) => ({
|
||||
id, companyId: fixture.companyId, title: `Batch ${index}`, visibility: "open", status: "in_progress",
|
||||
})));
|
||||
const queries: { query: string; params: unknown[] }[] = [];
|
||||
const observedDb = drizzle(db.$client, { schema, logger: { logQuery(query, params) { queries.push({ query, params }); } } }) as typeof db;
|
||||
const access = authorizationService(observedDb);
|
||||
const actor = { type: "agent" as const, companyId: fixture.companyId, agentId: fixture.ownerAgentId };
|
||||
const runs = ids.map(issueId => ({ companyId: fixture.companyId, scopeKind: "issue" as const, issueId }));
|
||||
expect(await getHeartbeatRunReadDecisions(observedDb, access, actor, runs)).toEqual(ids.map(() => true));
|
||||
const existence = queries.filter(({ query }) => query.startsWith('select "id" from "issues"'));
|
||||
expect(existence.map(({ params }) => params.length).sort((a, b) => a - b)).toEqual([2, 501]);
|
||||
expect(existence.every(({ query, params }) => query.includes('"company_id"') && params[0] === fixture.companyId)).toBe(true);
|
||||
});
|
||||
|
||||
it("captures batch inputs before awaiting reads and rejects a failed batch", async () => {
|
||||
const fixture = await seedFixture();
|
||||
const access = authorizationService(db);
|
||||
const actor = { type: "agent" as const, companyId: fixture.companyId, agentId: fixture.ownerAgentId };
|
||||
const binding = { companyId: fixture.companyId, scopeKind: "issue" as const, issueId: fixture.issueId };
|
||||
const decideReads = vi.spyOn(access, "decideReads");
|
||||
const pending = getHeartbeatRunReadDecisions(db, access, actor, [binding, binding]);
|
||||
binding.issueId = randomUUID();
|
||||
expect(await pending).toEqual([true, true]);
|
||||
expect(decideReads).toHaveBeenCalledExactlyOnceWith(actor, [expect.objectContaining({ issueId: fixture.issueId })]);
|
||||
const error = new Error("authorization unavailable");
|
||||
decideReads.mockRejectedValueOnce(error);
|
||||
const current = { ...binding, issueId: fixture.issueId };
|
||||
await expect(getHeartbeatRunReadDecisions(db, access, actor, [current])).rejects.toBe(error);
|
||||
const transient = Object.assign(new Error("temporary disconnect"), { code: "CONNECTION_CLOSED" });
|
||||
decideReads.mockRejectedValueOnce(transient);
|
||||
expect(await getHeartbeatRunReadDecisions(db, access, actor, [current])).toEqual([true]);
|
||||
});
|
||||
|
||||
it("does no database or authorization work for company-only or empty batches", async () => {
|
||||
const fixture = await seedFixture();
|
||||
const queries: string[] = [];
|
||||
const observedDb = drizzle(db.$client, { schema, logger: { logQuery(query) { queries.push(query); } } }) as typeof db;
|
||||
const access = authorizationService(observedDb);
|
||||
const decideReads = vi.spyOn(access, "decideReads");
|
||||
const actor = { type: "agent" as const, companyId: fixture.companyId, agentId: fixture.ownerAgentId };
|
||||
expect(await getHeartbeatRunReadDecisions(observedDb, access, actor, [])).toEqual([]);
|
||||
expect(await getHeartbeatRunReadDecisions(observedDb, access, actor, [
|
||||
{ companyId: fixture.companyId, scopeKind: "company" }, { companyId: fixture.companyId, scopeKind: "issue" },
|
||||
])).toEqual([true, false]);
|
||||
expect(queries).toEqual([]);
|
||||
expect(decideReads).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["owner", "outsider"])("checks concurrent duplicate issue runs once for the %s", async (viewer) => {
|
||||
const fixture = await seedFixture();
|
||||
const access = authorizationService(db);
|
||||
@@ -469,7 +592,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
agentId: viewer === "owner" ? fixture.ownerAgentId : fixture.otherAgentId,
|
||||
};
|
||||
const binding = { companyId: fixture.companyId, scopeKind: "issue" as const, issueId: fixture.issueId };
|
||||
const decisions = await getHeartbeatRunReadDecisions(db, access, actor, Array.from({ length: 200 }, () => ({ ...binding })));
|
||||
const decisions = await getHeartbeatRunReadDecisions(db, { decide: access.decide }, actor, Array.from({ length: 200 }, () => ({ ...binding })));
|
||||
expect(decisions).toEqual(Array(200).fill(viewer === "owner"));
|
||||
expect(decide).toHaveBeenCalledTimes(1);
|
||||
decide.mockRestore();
|
||||
@@ -493,7 +616,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
const decide = vi.spyOn(access, "decide");
|
||||
const actor = { type: "agent" as const, companyId: fixture.companyId, agentId: fixture.ownerAgentId };
|
||||
const binding = { companyId: fixture.companyId, scopeKind: "issue" as const, issueId: fixture.issueId };
|
||||
const pending = getHeartbeatRunReadDecisions(db, access, actor, [binding, binding]);
|
||||
const pending = getHeartbeatRunReadDecisions(db, { decide: access.decide }, actor, [binding, binding]);
|
||||
binding.issueId = randomUUID();
|
||||
expect(await pending).toEqual([true, true]);
|
||||
expect(decide).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
|
||||
@@ -567,7 +690,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
const decide = vi.spyOn(access, "decide").mockRejectedValue(error);
|
||||
const actor = { type: "agent" as const, companyId: fixture.companyId, agentId: fixture.ownerAgentId };
|
||||
const binding = { companyId: fixture.companyId, scopeKind: "issue" as const, issueId: fixture.issueId };
|
||||
await expect(getHeartbeatRunReadDecisions(db, access, actor, Array(20).fill(binding))).rejects.toBe(error);
|
||||
await expect(getHeartbeatRunReadDecisions(db, { decide: access.decide }, actor, Array(20).fill(binding))).rejects.toBe(error);
|
||||
expect(decide).toHaveBeenCalledTimes(1);
|
||||
decide.mockRestore();
|
||||
expect(await getHeartbeatRunReadDecisions(db, access, actor, [binding])).toEqual([true]);
|
||||
|
||||
@@ -25,10 +25,10 @@ describe("model-facing run errors", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("scheduled continuation presentation", () => {
|
||||
describe.each(["scheduled_monitor_waiting", "scheduled_monitor_dispatched", "scheduled_monitor_preserved"])("scheduled continuation presentation (%s)", finalizationReasonCode => {
|
||||
const resultJson = {
|
||||
finalizationPhase: "committed",
|
||||
finalizationReasonCode: "scheduled_monitor_waiting",
|
||||
finalizationReasonCode,
|
||||
nativeResult: {
|
||||
schema: "paperclip.run_result.v1",
|
||||
reportedWorkDisposition: "yielded",
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { activityLog, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveIssueCreationSource, runBoundIssueReference } from "../services/issue-creation-source.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("creation source run projections", () => {
|
||||
const companyId = randomUUID(), otherCompanyId = randomUUID(), agentId = randomUUID();
|
||||
const sourceId = randomUUID(), secondId = randomUUID(), foreignId = randomUUID();
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let childNumber = 100;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("creation-source-read-");
|
||||
db = createDb(database.connectionString);
|
||||
await db.insert(companies).values([
|
||||
{ id: companyId, name: "Creation source fixture", issuePrefix: "CSR" },
|
||||
{ id: otherCompanyId, name: "Foreign source fixture", issuePrefix: "FSR" },
|
||||
]);
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Source agent", adapterType: "process" });
|
||||
await db.insert(issues).values([
|
||||
{ id: sourceId, companyId, title: "Source task", identifier: "CSR-1", issueNumber: 1 },
|
||||
{ id: secondId, companyId, title: "Second source task", identifier: "CSR-2", issueNumber: 2 },
|
||||
{ id: foreignId, companyId: otherCompanyId, title: "Foreign task", identifier: "FSR-1", issueNumber: 1 },
|
||||
]);
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
async function fixture(values: Partial<typeof heartbeatRuns.$inferInsert>) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", ...values }).returning();
|
||||
const number = childNumber++;
|
||||
const [child] = await db.insert(issues).values({ companyId, title: "Created task", identifier: `CSR-${number}`,
|
||||
issueNumber: number, originRunId: run.id }).returning();
|
||||
return { run, child };
|
||||
}
|
||||
|
||||
it.each(["scoped", "legacy task key", "legacy nested identifier"])("omits saved run payloads for %s bindings while preserving provenance and storage", async (binding) => {
|
||||
const bulk = "retained-source-context🙂".repeat(32_768);
|
||||
const contextSnapshot = binding === "scoped"
|
||||
? { issueId: sourceId, paperclipIssue: { id: sourceId, retainedDescription: bulk }, retainedLaunchInput: bulk }
|
||||
: binding === "legacy task key"
|
||||
? { taskKey: "CSR-1", paperclipIssue: { retainedDescription: bulk }, retainedLaunchInput: bulk }
|
||||
: { paperclipIssue: { id: "CSR-1", retainedDescription: bulk }, retainedLaunchInput: bulk };
|
||||
const { run, child } = await fixture({ contextSnapshot,
|
||||
runnerProfileJson: { retainedInput: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk, resultJson: { retainedReceipt: bulk } });
|
||||
expect(run.issueId).toBe(binding === "scoped" ? sourceId : null);
|
||||
expect(run.nativeIssueId).toBeNull();
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns, issues }, capture);
|
||||
try {
|
||||
const result = await resolveIssueCreationSource(observed, child);
|
||||
capture.flush();
|
||||
const reads = events.filter(event => event.table === "heartbeat_runs" && event.operation === "select");
|
||||
expect(reads.reduce((count, event) => count + Number(event.calls), 0)).toBe(1);
|
||||
expect(reads.reduce((bytes, event) => bytes + Number(event.responseValueBytes), 0)).toBeLessThan(8192);
|
||||
expect(result).toMatchObject({ run: { id: run.id, companyId, agentId }, sourceIssue: { id: sourceId } });
|
||||
expect(runBoundIssueReference(result!.run)).toBe(binding === "scoped" ? sourceId : "CSR-1");
|
||||
for (const key of ["runnerProfileJson", "stdoutExcerpt", "stderrExcerpt", "resultJson"]) expect(result!.run).not.toHaveProperty(key);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
|
||||
expect(JSON.stringify(events)).not.toContain("retained-source-context");
|
||||
} finally {
|
||||
capture.flush();
|
||||
await observed.$client.end();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "native identity", nativeIssueId: sourceId, issueId: secondId, contextSnapshot: { issueId: secondId }, expected: sourceId },
|
||||
{ label: "issue scope", issueId: sourceId, contextSnapshot: { issueId: secondId }, expected: sourceId },
|
||||
{ label: "legacy issue id", contextSnapshot: { issueId: sourceId, taskId: secondId }, expected: sourceId },
|
||||
{ label: "typed task id fallback", contextSnapshot: { issueId: 42, taskId: sourceId }, expected: sourceId },
|
||||
{ label: "nested issue identifier", contextSnapshot: { issueId: " ", paperclipIssue: { id: "CSR-1", ignored: "payload" }, taskKey: "CSR-2" }, expected: sourceId },
|
||||
{ label: "typed nested id fallback", contextSnapshot: { paperclipIssue: { id: 42 }, taskKey: "CSR-1" }, expected: sourceId },
|
||||
{ label: "foreign context", contextSnapshot: { taskId: foreignId }, expected: null },
|
||||
{ label: "null context", contextSnapshot: null, expected: null },
|
||||
])("preserves binding precedence for $label", async ({ label: _label, expected, ...values }) => {
|
||||
const { child } = await fixture(values);
|
||||
expect((await resolveIssueCreationSource(db, child))?.sourceIssue.id ?? null).toBe(expected);
|
||||
});
|
||||
|
||||
it.each([42, "scalar", ["issueId", sourceId]])("does not invent a task binding from malformed context %j", async (context) => {
|
||||
const { child } = await fixture({ contextSnapshot: context as unknown as Record<string, unknown> });
|
||||
expect(await resolveIssueCreationSource(db, child)).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves historical activity lookup, current bindings, and missing or foreign source denial", async () => {
|
||||
const { run, child } = await fixture({ contextSnapshot: { taskKey: "CSR-1" } });
|
||||
await db.update(issues).set({ originRunId: null }).where(eq(issues.id, child.id));
|
||||
await db.insert(activityLog).values({ companyId, actorType: "system", actorId: "fixture", action: "issue.created",
|
||||
entityType: "issue", entityId: child.id, runId: run.id });
|
||||
expect((await resolveIssueCreationSource(db, { ...child, originRunId: null }))?.sourceIssue.id).toBe(sourceId);
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { taskKey: "CSR-2" } }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect((await resolveIssueCreationSource(db, child))?.sourceIssue.id).toBe(secondId);
|
||||
expect(await resolveIssueCreationSource(db, { ...child, companyId: otherCompanyId })).toBeNull();
|
||||
expect(await resolveIssueCreationSource(db, { ...child, originRunId: randomUUID() })).toBeNull();
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { taskId: child.id } }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await resolveIssueCreationSource(db, child)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agentRuntimeState,
|
||||
@@ -79,7 +80,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
function app(companyId: string, userId = "queue-owner", agentActor?: { agentId: string; runId: string }) {
|
||||
function app(companyId: string, userId = "queue-owner", agentActor?: { agentId: string; runId: string }, database = db) {
|
||||
const testApp = express();
|
||||
testApp.use(express.json());
|
||||
testApp.use((req, _res, next) => {
|
||||
@@ -94,7 +95,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
};
|
||||
next();
|
||||
});
|
||||
testApp.use("/api", issueRoutes(db, {} as any, {}));
|
||||
testApp.use("/api", issueRoutes(database, {} as any, {}));
|
||||
testApp.use(errorHandler);
|
||||
return testApp;
|
||||
}
|
||||
@@ -192,6 +193,104 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
return { companyId, agentId, issueId, runId, wakeId, commentIds };
|
||||
}
|
||||
|
||||
it.each([false, true])("polls queued comments without transferring run snapshots (agent fallback: %s)", async fallback => {
|
||||
const f = await seedQueue(), payload = "saved-execution-input🙂".repeat(25_000);
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, prompt: payload },
|
||||
runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner", adapterConfig: { instructions: payload } } },
|
||||
stdoutExcerpt: payload, stderrExcerpt: payload,
|
||||
}).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (fallback) await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed))
|
||||
.get(`/api/issues/${f.issueId}/queued-comments`).expect(200);
|
||||
expect(response.body.queueId).toBe(f.wakeId);
|
||||
expect(response.body.targetRunId).toBe(f.runId);
|
||||
expect(response.body.entries.map((entry: { comment: { id: string } }) => entry.comment.id)).toEqual(f.commentIds);
|
||||
capture.flush();
|
||||
const reads = events.filter(event => event.table === "heartbeat_runs" && event.operation === "select");
|
||||
expect(reads.length).toBeGreaterThan(0);
|
||||
expect(reads.reduce((bytes, event) => bytes + Number(event.responseValueBytes), 0)).toBeLessThan(2048);
|
||||
await request(app(randomUUID(), "queue-owner", undefined, observed))
|
||||
.get(`/api/issues/${f.issueId}/queued-comments`).expect(404);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(before);
|
||||
} finally { capture.flush(); await observed.$client.end(); }
|
||||
});
|
||||
|
||||
it.each(["deferred", "queued"])("reads only queue display inputs from large %s wake payloads", async state => {
|
||||
const f = await seedQueue();
|
||||
if (state === "queued") await promoteQueue(f);
|
||||
const client = app(f.companyId);
|
||||
const path = `/api/issues/${f.issueId}/queued-comments`;
|
||||
const expected = await request(client).get(path).expect(200);
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId));
|
||||
const payload = "saved-wake-input🙂".repeat(30_000);
|
||||
const context = wake.payload?._paperclipWakeContext as Record<string, unknown>;
|
||||
await db.update(agentWakeupRequests).set({ payload: {
|
||||
...wake.payload, prompt: payload,
|
||||
_paperclipWakeContext: { ...context, prompt: payload, nativeExecutionInput: { instructions: payload } },
|
||||
} }).where(eq(agentWakeupRequests.id, f.wakeId));
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, prompt: payload },
|
||||
runnerProfileJson: { instructions: payload }, resultJson: { output: payload },
|
||||
}).where(eq(heartbeatRuns.companyId, f.companyId));
|
||||
const [before] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agentWakeupRequests, heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
|
||||
expect(response.body).toEqual(expected.body);
|
||||
capture.flush();
|
||||
const reads = events.filter(event => ["agent_wakeup_requests", "heartbeat_runs"].includes(String(event.table)) && event.operation === "select");
|
||||
expect(reads.some(event => event.table === "agent_wakeup_requests")).toBe(true);
|
||||
expect(reads.reduce((bytes, event) => bytes + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId)))[0]).toEqual(before);
|
||||
await request(app(randomUUID(), "queue-owner", undefined, observed)).get(path).expect(404);
|
||||
} finally { capture.flush(); await observed.$client.end(); }
|
||||
});
|
||||
|
||||
it.each(["deferred", "queued"])("filters unrelated task wakes before reading the %s comment queue", async state => {
|
||||
const f = await seedQueue();
|
||||
if (state === "queued") await promoteQueue(f);
|
||||
const path = `/api/issues/${f.issueId}/queued-comments`;
|
||||
const expected = await request(app(f.companyId)).get(path).expect(200);
|
||||
const unrelated = Array.from({ length: 80 }, (_, index) => ({
|
||||
companyId: f.companyId, agentId: f.agentId, source: "issue_comment",
|
||||
status: index % 2 ? "queued" : "deferred_issue_execution",
|
||||
requestedAt: new Date("2020-01-01T00:00:00Z"),
|
||||
payload: { issueId: randomUUID(), _paperclipWakeContext: {
|
||||
wakeCommentIds: Array.from({ length: 60 }, () => randomUUID()),
|
||||
} },
|
||||
}));
|
||||
const malformed = [null, [f.issueId], { id: f.issueId }, f.issueId.toUpperCase()];
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
...unrelated,
|
||||
...malformed.map(issueId => ({ companyId: f.companyId, agentId: f.agentId,
|
||||
source: "issue_comment", status: "deferred_issue_execution",
|
||||
requestedAt: new Date("2020-01-01T00:00:00Z"),
|
||||
payload: { issueId, _paperclipWakeContext: { wakeCommentIds: f.commentIds } },
|
||||
})),
|
||||
]);
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agentWakeupRequests }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
|
||||
expect(response.body).toEqual(expected.body);
|
||||
capture.flush();
|
||||
const reads = events.filter(event => event.table === "agent_wakeup_requests" && event.operation === "select");
|
||||
expect(reads.length).toBeGreaterThan(0);
|
||||
expect(reads.reduce((bytes, event) => bytes + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
expect(reads.reduce((rows, event) => rows + Number(event.rows), 0)).toBe(1);
|
||||
expect(await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(85);
|
||||
await request(app(randomUUID(), "queue-owner", undefined, observed)).get(path).expect(404);
|
||||
} finally { capture.flush(); await observed.$client.end(); }
|
||||
});
|
||||
|
||||
it.each(["cancelled", "running"])("rejects a late Done from an interrupted %s task run at the write boundary", async status => {
|
||||
const seeded = await seedQueue();
|
||||
await db.update(heartbeatRuns).set({ status, resultJson: { executionCancellation: { state: "requested" } } })
|
||||
|
||||
@@ -14,6 +14,7 @@ const mockTreeControlService = vi.hoisted(() => ({
|
||||
cancelIssueStatusesForHold: vi.fn(),
|
||||
restoreIssueStatusesForHold: vi.fn(),
|
||||
getHold: vi.fn(),
|
||||
getActivePauseHoldGate: vi.fn(),
|
||||
releaseHold: vi.fn(),
|
||||
cancelUnclaimedWakeupsForTree: vi.fn(),
|
||||
}));
|
||||
@@ -72,6 +73,7 @@ describe("issue tree control routes", () => {
|
||||
mockReplayBlocks.mockResolvedValue([]);
|
||||
mockExecutionBlocker.mockResolvedValue(null);
|
||||
mockTreeControlService.getHold.mockResolvedValue(null);
|
||||
mockTreeControlService.getActivePauseHoldGate.mockResolvedValue(null);
|
||||
mockIssueService.getById.mockResolvedValue({
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
companyId: "company-2",
|
||||
@@ -213,6 +215,36 @@ describe("issue tree control routes", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ skipReason: "terminal_status", stillPaused: false },
|
||||
{ skipReason: "already_held", stillPaused: false },
|
||||
{ skipReason: "already_held", stillPaused: true },
|
||||
])("rechecks skipped pause members with $skipReason and remaining pause=$stillPaused", async ({ skipReason, stillPaused }) => {
|
||||
const rootId = "11111111-1111-4111-8111-111111111111";
|
||||
const holdId = "33333333-3333-4333-8333-333333333333";
|
||||
const childId = "44444444-4444-4444-8444-444444444444";
|
||||
const root = { id: rootId, companyId: "company-2", status: "todo", assigneeAgentId: null };
|
||||
const child = { id: childId, companyId: "company-2", status: "todo", assigneeAgentId: "current-agent" };
|
||||
mockIssueService.getById.mockImplementation(async (id: string) => id === rootId ? root : child);
|
||||
const hold = { id: holdId, rootIssueId: rootId, mode: "pause", members: [
|
||||
{ issueId: childId, skipped: true, skipReason, issueStatus: "done" },
|
||||
] };
|
||||
mockTreeControlService.getHold.mockResolvedValue(hold);
|
||||
mockTreeControlService.releaseHold.mockResolvedValue({ ...hold, status: "released" });
|
||||
mockTreeControlService.getActivePauseHoldGate.mockResolvedValue(stillPaused ? { holdId: "remaining-hold" } : null);
|
||||
mockReplayBlocks.mockResolvedValue([{ id: childId, identifier: "TEST-2" }]);
|
||||
const app = await createApp({ type: "board", userId: "user-1", companyIds: ["company-2"], source: "session" });
|
||||
const response = await request(app)
|
||||
.post(`/api/issues/${rootId}/tree-holds/${holdId}/release`)
|
||||
.send({ metadata: { wakeAgents: true } });
|
||||
expect(response.status).toBe(200);
|
||||
expect(mockExecutionBlocker).toHaveBeenCalledWith(expect.anything(), "company-2", childId);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledTimes(stillPaused ? 0 : 1);
|
||||
if (!stillPaused) expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith("current-agent", expect.objectContaining({
|
||||
payload: expect.objectContaining({ issueId: childId }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("resumes sandbox-backed work through the shared plugin worker manager", async () => {
|
||||
const rootId = "11111111-1111-4111-8111-111111111111";
|
||||
const holdId = "33333333-3333-4333-8333-333333333333";
|
||||
@@ -284,14 +316,14 @@ describe("issue tree control routes", () => {
|
||||
).toEqual([rootId, "child"]);
|
||||
});
|
||||
|
||||
it("keeps the hold active when resume requests an unsafe execution replay", async () => {
|
||||
it.each([false, true])("keeps the hold active when resume requests an unsafe execution replay (previously skipped=%s)", async (skipped) => {
|
||||
const rootId = "11111111-1111-4111-8111-111111111111";
|
||||
const holdId = "33333333-3333-4333-8333-333333333333";
|
||||
mockTreeControlService.getHold.mockResolvedValue({
|
||||
id: holdId,
|
||||
rootIssueId: rootId,
|
||||
mode: "pause",
|
||||
members: [{ issueId: rootId }],
|
||||
members: [{ issueId: rootId, skipped, skipReason: skipped ? "terminal_status" : null }],
|
||||
});
|
||||
mockReplayBlocks.mockResolvedValue([{ id: rootId, identifier: "TEST-1" }]);
|
||||
mockExecutionBlocker.mockResolvedValue({ nextAction: "Resume without waking agents until the previous execution stops.",
|
||||
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
applyPendingMigrations,
|
||||
closeRegisteredClients,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
@@ -20,8 +22,10 @@ import {
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_TREE_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
@@ -35,6 +39,11 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
if (externalUrl) {
|
||||
await applyPendingMigrations(externalUrl);
|
||||
db = createDb(externalUrl);
|
||||
return;
|
||||
}
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
@@ -53,6 +62,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (externalUrl) await closeRegisteredClients(externalUrl);
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
@@ -623,6 +633,58 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
||||
expect(checkedOutLegacyFullPauseRoot.checkoutRunId).toBe(rootRunId);
|
||||
});
|
||||
|
||||
it("previews only executable current assignees as resume wake targets", async () => {
|
||||
const companyId = randomUUID();
|
||||
const rootId = randomUUID();
|
||||
const readyAgentId = randomUUID();
|
||||
const parkedAgentId = randomUUID();
|
||||
const previousAgentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId, name: "Resume preview", issuePrefix: "RESUME",
|
||||
});
|
||||
await db.insert(agents).values([readyAgentId, parkedAgentId, previousAgentId].map((id) => ({
|
||||
id, companyId, name: id, role: "engineer", status: "idle", adapterType: "process",
|
||||
})));
|
||||
const children = ["todo", "in_progress", "in_review", "blocked", "backlog", "done"].map((status) => ({
|
||||
id: randomUUID(), companyId, parentId: rootId, title: status, status,
|
||||
assigneeAgentId: status === "blocked" || status === "backlog" ? parkedAgentId : readyAgentId,
|
||||
}));
|
||||
await db.insert(issues).values([
|
||||
{ id: rootId, companyId, title: "Unassigned root", status: "todo" },
|
||||
...children,
|
||||
]);
|
||||
const svc = issueTreeControlService(db);
|
||||
const pause = await svc.createHold(companyId, rootId, {
|
||||
mode: "pause",
|
||||
actor: { actorType: "user", actorId: "board-user", userId: "board-user" },
|
||||
});
|
||||
expect(pause.hold.members?.find((member) => member.issueId === children[5].id)).toMatchObject({
|
||||
skipped: true, skipReason: "terminal_status",
|
||||
});
|
||||
await db.update(issues).set({ status: "todo" }).where(eq(issues.id, children[5].id));
|
||||
// A previous executor is observable while stopping, but is not the current assignee to wake.
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId, companyId, agentId: previousAgentId, invocationSource: "assignment",
|
||||
status: "running", contextSnapshot: { issueId: children[0].id },
|
||||
});
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, children[0].id));
|
||||
|
||||
const preview = await svc.preview(companyId, rootId, { mode: "resume" });
|
||||
expect(preview.totals.affectedIssues).toBe(7);
|
||||
expect(preview.affectedAgents).toEqual([{ agentId: readyAgentId, issueCount: 4, activeRunCount: 0 }]);
|
||||
expect(preview.totals.affectedAgents).toBe(1);
|
||||
expect(preview.warnings).toContainEqual({
|
||||
code: "resume_tasks_remain_idle",
|
||||
message: "Blocked, backlog, and unassigned tasks will be unpaused but will not start.",
|
||||
issueIds: [rootId, children[3].id, children[4].id].sort(),
|
||||
});
|
||||
const rows = await db.select().from(issues).where(eq(issues.companyId, companyId));
|
||||
expect(rows.find((row) => row.id === children[3].id)?.status).toBe("blocked");
|
||||
expect(rows.find((row) => row.id === children[4].id)?.status).toBe("backlog");
|
||||
expect((await svc.getHold(companyId, pause.hold.id))?.status).toBe("active");
|
||||
});
|
||||
|
||||
it("resumes subtree pauses by releasing matching pause holds", async () => {
|
||||
const companyId = randomUUID();
|
||||
const rootIssueId = randomUUID();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { recordLegacyWorkspaceRestoreFailure, legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
|
||||
@@ -8,6 +9,7 @@ import { hasConversationContinuationPolicy, conversationRecoveryActionPredicate
|
||||
import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../services/legacy-workspace-restore-recovery.js";
|
||||
import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { deliverExecutionStatuses } from "../services/execution-status-delivery.js";
|
||||
import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js";
|
||||
|
||||
const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
|
||||
@@ -45,6 +47,127 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
} };
|
||||
const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0];
|
||||
|
||||
it("locks only terminal control state while returning and retaining the complete run", async () => {
|
||||
const f = await seed();
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const [saved] = await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.run.contextSnapshot, savedInput: bulk },
|
||||
runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk,
|
||||
}).where(eq(heartbeatRuns.id, f.runId)).returning();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
|
||||
try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); }
|
||||
finally { capture.flush(); }
|
||||
const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
|
||||
const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select");
|
||||
expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1);
|
||||
expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
expect(returned).toMatchObject({ id: f.runId, status: "failed", contextSnapshot: saved!.contextSnapshot,
|
||||
runnerProfileJson: saved!.runnerProfileJson, stdoutExcerpt: bulk, stderrExcerpt: bulk });
|
||||
expect(hasRequiredWorkspaceRecovery(returned!.resultJson)).toBe(true);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(returned);
|
||||
expect(await readLease(f.lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" });
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)))
|
||||
.toEqual([expect.objectContaining({ evidence: expect.objectContaining({ runId: f.runId }), ownerType: "board" })]);
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
});
|
||||
|
||||
it.each(["legacy", "native"])("returns one complete run when retained workspace evidence is unchanged for %s", async (runtimeMode) => {
|
||||
const f = await seed();
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const [saved] = await db.update(heartbeatRuns).set({ runtimeMode, status: "failed", finishedAt: new Date(),
|
||||
contextSnapshot: { ...f.run.contextSnapshot, savedInput: bulk }, runnerProfileJson: { savedProfile: bulk },
|
||||
stdoutExcerpt: bulk, stderrExcerpt: bulk,
|
||||
}).where(eq(heartbeatRuns.id, f.runId)).returning();
|
||||
const recordRestoreFailureOnly = runtimeMode === "native";
|
||||
const failed = (await terminalizeLegacyExecution({ db, run: saved!, status: "failed", recordRestoreFailureOnly,
|
||||
workspaceRestoreSource: f.lease,
|
||||
patch: { resultJson: { workspaceRestoreFailure: "restore_failed", futureResult: bulk } },
|
||||
}))!;
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
|
||||
try { returned = await terminalizeLegacyExecution({ db, run: failed, status: "failed", recordRestoreFailureOnly,
|
||||
workspaceRestoreSource: f.lease,
|
||||
...(recordRestoreFailureOnly ? { patch: { resultJson: failed.resultJson } } : {}),
|
||||
}); }
|
||||
finally { capture.flush(); }
|
||||
const writes = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && event.operation === "update");
|
||||
expect(writes.reduce((sum, group) => sum + Number(group.rows), 0)).toBe(1);
|
||||
expect(returned).toMatchObject({ id: f.runId, status: "failed", finishedAt: failed.finishedAt,
|
||||
contextSnapshot: failed.contextSnapshot, runnerProfileJson: failed.runnerProfileJson, resultJson: failed.resultJson,
|
||||
stdoutExcerpt: bulk, stderrExcerpt: bulk });
|
||||
if (!recordRestoreFailureOnly) expect(returned!.executionStatusDeliveryId).toBe(failed.executionStatusDeliveryId);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(returned);
|
||||
expect(await readLease(f.lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" });
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)))
|
||||
.toEqual([expect.objectContaining({ evidence: expect.objectContaining({ runId: f.runId }), ownerType: "board" })]);
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
});
|
||||
|
||||
it("preserves pending and acknowledged status delivery on unpatched legacy recovery retries", async () => {
|
||||
await deliverExecutionStatuses(db, { publish: vi.fn() });
|
||||
const f = await seed();
|
||||
const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
|
||||
const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(holds).toHaveLength(1);
|
||||
expect(hasRequiredWorkspaceRecovery(failed.resultJson)).toBe(true);
|
||||
const repeated = (await terminalizeLegacyExecution({ db, run: failed, status: "failed" }))!;
|
||||
expect(repeated.executionStatusDeliveryId).toBe(failed.executionStatusDeliveryId);
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
await terminalizeLegacyExecution({ db, run: failed, status: "failed" });
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(1);
|
||||
const after = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(after).toHaveLength(1);
|
||||
expect(after[0]).toMatchObject({ id: holds[0].id, status: holds[0].status, outcome: holds[0].outcome,
|
||||
ownerType: holds[0].ownerType, wakePolicy: null, evidence: { ...holds[0].evidence,
|
||||
automaticRecovery: expect.objectContaining({ policy: "preserve_without_replay_v1", replay: "blocked",
|
||||
runId: f.runId, actionOutcome: "unknown" }),
|
||||
} });
|
||||
expect(await readLease(f.lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toEqual(failed.resultJson);
|
||||
});
|
||||
|
||||
it("delivers new retained workspace evidence even when legacy run status is unchanged", async () => {
|
||||
await deliverExecutionStatuses(db, { publish: vi.fn() });
|
||||
const f = await seed();
|
||||
const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
const [added] = await db.insert(environmentLeases).values({ ...f.lease, id: randomUUID(), providerLeaseId: randomUUID() }).returning();
|
||||
const updated = (await terminalizeLegacyExecution({ db, run: failed, status: "failed" }))!;
|
||||
expect(updated.executionStatusDeliveryId).toBeTruthy();
|
||||
expect(updated.executionStatusDeliveryId).not.toBe(failed.executionStatusDeliveryId);
|
||||
expect(updated.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: expect.arrayContaining([f.lease.id, added.id]) });
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
|
||||
await terminalizeLegacyExecution({ db, run: updated, status: "failed" });
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("retains write conditions and delivers an explicit patch on unchanged legacy status", async () => {
|
||||
await deliverExecutionStatuses(db, { publish: vi.fn() });
|
||||
const f = await seed();
|
||||
const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(await terminalizeLegacyExecution({ db, run: failed, status: "failed", writeConditions: [sql`false`] })).toBeNull();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(1);
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual(holds);
|
||||
const patched = (await terminalizeLegacyExecution({ db, run: failed, status: "failed", patch: { error: "Retained workspace still needs repair" } }))!;
|
||||
expect(patched.error).toBe("Retained workspace still needs repair");
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
|
||||
});
|
||||
|
||||
async function stopAndRetain(f: Awaited<ReturnType<typeof seed>>, retry = false) {
|
||||
let stopping = (await prepareSandboxStopAndRetain(db, f.lease))!;
|
||||
expect(stopping.status).toBe("pending_cleanup");
|
||||
|
||||
@@ -58,9 +58,19 @@ describe("live-event run read boundaries", () => {
|
||||
}).returning({ id: heartbeatRuns.id });
|
||||
const [maintenance] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, scopeKind: "company" }).returning({ id: heartbeatRuns.id });
|
||||
const [foreign] = await db.insert(heartbeatRuns).values({ companyId: foreignCompanyId, agentId: foreignAgent.id, scopeKind: "company" }).returning({ id: heartbeatRuns.id });
|
||||
const hiddenIssueId = randomUUID();
|
||||
await db.insert(issues).values({ id: hiddenIssueId, companyId, title: "Other private task", visibility: "private", privacyRootIssueId: hiddenIssueId, responsibleUserId: owner });
|
||||
const [hiddenRun] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, scopeKind: "issue", issueId: hiddenIssueId }).returning({ id: heartbeatRuns.id });
|
||||
const reads: string[] = [];
|
||||
const queries: string[] = [];
|
||||
let failNextRunRead = false;
|
||||
let failNextIssueRead = false;
|
||||
const observedDb = drizzle(db.$client, { schema, logger: { logQuery(query) {
|
||||
queries.push(query);
|
||||
if (query.startsWith("select ") && query.includes('from "issues"') && failNextIssueRead) {
|
||||
failNextIssueRead = false;
|
||||
throw new Error("fixture issue lookup unavailable");
|
||||
}
|
||||
if (query.startsWith("select ") && query.includes('from "heartbeat_runs"')) {
|
||||
reads.push(query);
|
||||
if (failNextRunRead) { failNextRunRead = false; throw new Error("fixture run lookup unavailable"); }
|
||||
@@ -78,17 +88,32 @@ describe("live-event run read boundaries", () => {
|
||||
socket.on("message", bytes => messages.push(JSON.parse(bytes.toString())));
|
||||
try {
|
||||
await once(socket, "open");
|
||||
async function deliver(runId: string) {
|
||||
let deliveries = 0;
|
||||
async function deliver(runId: string, payloadIssueId?: string) {
|
||||
deliveries += 1;
|
||||
const start = messages.length;
|
||||
const barrier = randomUUID();
|
||||
publishLiveEvent({ companyId, type: "heartbeat.run.status", payload: { runId, status: "running" } });
|
||||
publishLiveEvent({ companyId, type: "heartbeat.run.status", payload: {
|
||||
runId, status: "running", ...(payloadIssueId ? { issueId: payloadIssueId } : {}),
|
||||
} });
|
||||
publishLiveEvent({ companyId, type: "agent.status", payload: { agentId: barrier, status: "idle" } });
|
||||
await expect.poll(() => messages.some(message => message.payload.agentId === barrier)).toBe(true);
|
||||
return messages.slice(start).filter(message => message.type === "heartbeat.run.status");
|
||||
}
|
||||
|
||||
expect(await deliver(run.id)).toMatchObject([{ payload: { runId: run.id, status: "running" } }]);
|
||||
const batchStart = queries.length;
|
||||
expect(await deliver(run.id, issueId)).toMatchObject([{ payload: { runId: run.id, issueId, status: "running" } }]);
|
||||
expect(queries.length - batchStart).toBeLessThanOrEqual(6);
|
||||
expect(queries.slice(batchStart).filter(query => query.startsWith("select ") && query.includes('from "issues"')).length).toBeLessThanOrEqual(2);
|
||||
expect(await deliver(hiddenRun.id, issueId)).toEqual([]);
|
||||
expect(await deliver(run.id, hiddenIssueId)).toEqual([]);
|
||||
expect(await deliver(run.id, randomUUID())).toEqual([]);
|
||||
failNextIssueRead = true;
|
||||
expect(await deliver(run.id, issueId)).toEqual([]);
|
||||
expect(await deliver(run.id, issueId)).toHaveLength(1);
|
||||
await db.update(issueAccessGrants).set({ revokedAt: new Date() }).where(eq(issueAccessGrants.id, grant.id));
|
||||
expect(await deliver(run.id, issueId)).toEqual([]);
|
||||
expect(await deliver(run.id)).toEqual([]);
|
||||
expect(await deliver(maintenance.id)).toHaveLength(1);
|
||||
expect(await deliver(foreign.id)).toEqual([]);
|
||||
@@ -101,7 +126,7 @@ describe("live-event run read boundaries", () => {
|
||||
await db.delete(issues).where(eq(issues.id, issueId));
|
||||
expect(await deliver(run.id)).toEqual([]);
|
||||
|
||||
expect(reads).toHaveLength(9);
|
||||
expect(reads).toHaveLength(deliveries);
|
||||
for (const query of reads) {
|
||||
expect(query).toMatch(/^select "company_id", "scope_kind", "issue_id" from "heartbeat_runs"/);
|
||||
expect(query).not.toMatch(/context_snapshot|runner_profile_json|stdout_excerpt|result_json/);
|
||||
|
||||
@@ -14,7 +14,10 @@ vi.mock("../realtime/task-toast-eligibility.js", () => ({ taskToastEligible: vi.
|
||||
vi.mock("../services/authorization.js", () => ({
|
||||
authorizationService: () => ({ decide: async ({ resource }: { resource: { issueId: string } }) => ({ allowed: resource.issueId !== "hidden" }) }),
|
||||
}));
|
||||
vi.mock("../services/heartbeat-run-privacy.js", () => ({ canActorReadHeartbeatRun: async () => false }));
|
||||
vi.mock("../services/heartbeat-run-privacy.js", () => ({
|
||||
getHeartbeatRunReadDecisions: async (_db: unknown, _access: unknown, _actor: unknown, runs: { issueId?: string }[]) =>
|
||||
runs.map(run => run.issueId !== "hidden"),
|
||||
}));
|
||||
vi.mock("../services/case-access.js", () => ({
|
||||
canActorReadCase: async (_db: unknown, _actor: unknown, record: { id: string }) => record.id !== "hidden-case",
|
||||
}));
|
||||
|
||||
@@ -37,4 +37,15 @@ describe("native harness backup expiration config parsing", () => {
|
||||
vi.stubEnv("PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DAYS", "0");
|
||||
expect(loadConfig()).toMatchObject({ nativeHarnessBackupExpirationDays: 1 });
|
||||
});
|
||||
|
||||
it("keeps native runner evidence for seven days by default and at least one day", () => {
|
||||
vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", undefined);
|
||||
expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 7 });
|
||||
vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", "14");
|
||||
expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 14 });
|
||||
vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", "0");
|
||||
expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 1 });
|
||||
vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", "many");
|
||||
expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 7 });
|
||||
});
|
||||
});
|
||||
@@ -301,4 +301,378 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
|
||||
expect(await exists(leftover)).toBe(false);
|
||||
expect(await exists(session.current)).toBe(true);
|
||||
});
|
||||
async function seedArchive(root: string, input: { stampedDaysAgo?: number; mtimeDaysAgo?: number; name?: string } = {}) {
|
||||
const stampedAt = now.getTime() - (input.stampedDaysAgo ?? 10) * DAY_MS;
|
||||
const archive = path.join(root, "continuity-breaks", input.name ?? `${stampedAt}-${randomUUID()}`);
|
||||
await fs.mkdir(path.join(archive, "codex-home"), { recursive: true });
|
||||
await fs.writeFile(path.join(archive, "codex-home", "history.jsonl"), "x".repeat(64 * 1024));
|
||||
const mtime = new Date(now.getTime() - (input.mtimeDaysAgo ?? input.stampedDaysAgo ?? 10) * DAY_MS);
|
||||
await fs.utimes(archive, mtime, mtime);
|
||||
return archive;
|
||||
}
|
||||
|
||||
async function seedQuarantine(
|
||||
scope: string,
|
||||
input: { stampedDaysAgo?: number; name?: string } = {},
|
||||
) {
|
||||
if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-durable-sessions-"));
|
||||
const stampedAt = now.getTime() - (input.stampedDaysAgo ?? 10) * DAY_MS;
|
||||
const entry = path.join(
|
||||
stateBase,
|
||||
"quarantine",
|
||||
input.name ?? `${scope}.identity_indeterminate.${stampedAt}.4242.${randomUUID()}`,
|
||||
);
|
||||
await fs.mkdir(path.join(entry, "control-plane"), { recursive: true });
|
||||
await fs.writeFile(path.join(entry, "history.jsonl"), "x".repeat(64 * 1024));
|
||||
return entry;
|
||||
}
|
||||
|
||||
function evidenceService(overrides: Parameters<typeof nativeHarnessBackupExpirationService>[1] = {}) {
|
||||
return nativeHarnessBackupExpirationService(db, {
|
||||
stateBase,
|
||||
now,
|
||||
evidenceRetentionDays: 7,
|
||||
isSessionBusy: () => false,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
async function emptyStateBase() {
|
||||
if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-durable-sessions-"));
|
||||
return createHash("sha256").update(randomUUID()).digest("hex");
|
||||
}
|
||||
|
||||
it("removes continuity archives older than the retention window and keeps recent ones", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const old = await seedArchive(session.root, { stampedDaysAgo: 9 });
|
||||
const recent = await seedArchive(session.root, { stampedDaysAgo: 2 });
|
||||
|
||||
const result = await evidenceService().tick();
|
||||
|
||||
expect(result).toMatchObject({ evidenceCandidates: 1, evidenceRemoved: 1, failed: 0 });
|
||||
expect(result.evidenceReclaimedBytes).toBeGreaterThan(0);
|
||||
expect(result.skipped.recent_evidence).toBe(1);
|
||||
expect(await exists(old)).toBe(false);
|
||||
expect(await exists(recent)).toBe(true);
|
||||
expect(await exists(session.current)).toBe(true);
|
||||
});
|
||||
|
||||
it("removes the continuity-breaks folder once its archives are gone", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
await seedArchive(session.root);
|
||||
|
||||
expect(await evidenceService().tick()).toMatchObject({ evidenceRemoved: 1 });
|
||||
expect(await exists(path.join(session.root, "continuity-breaks"))).toBe(false);
|
||||
});
|
||||
|
||||
it("ages a continuity archive by its name and falls back to mtime for a future stamp", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const stampedRecently = await seedArchive(session.root, { stampedDaysAgo: 1, mtimeDaysAgo: 30 });
|
||||
const futureStamp = await seedArchive(session.root, {
|
||||
name: `${now.getTime() + DAY_MS}-${randomUUID()}`,
|
||||
mtimeDaysAgo: 30,
|
||||
});
|
||||
const futureRecent = await seedArchive(session.root, {
|
||||
name: `${now.getTime() + 2 * DAY_MS}-${randomUUID()}`,
|
||||
mtimeDaysAgo: 1,
|
||||
});
|
||||
|
||||
expect(await evidenceService().tick()).toMatchObject({ evidenceRemoved: 1 });
|
||||
expect(await exists(stampedRecently)).toBe(true);
|
||||
expect(await exists(futureStamp)).toBe(false);
|
||||
expect(await exists(futureRecent)).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores folders that are not continuity archives", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const other = await seedArchive(session.root, { name: "notes", stampedDaysAgo: 30 });
|
||||
|
||||
expect(await evidenceService().tick()).toMatchObject({ evidenceCandidates: 0 });
|
||||
expect(await exists(other)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the evidence of a session that is executing in this process", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const archive = await seedArchive(session.root);
|
||||
const quarantine = await seedQuarantine(session.scope);
|
||||
|
||||
const result = await evidenceService({ isSessionBusy: (scope) => scope === session.scope }).tick();
|
||||
|
||||
expect(result).toMatchObject({ evidenceCandidates: 0, evidenceRemoved: 0 });
|
||||
expect(result.skipped.session_active).toBe(3);
|
||||
expect(await exists(archive)).toBe(true);
|
||||
expect(await exists(quarantine)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps evidence when its session starts while the removal is being prepared", async () => {
|
||||
const scope = await emptyStateBase();
|
||||
const root = path.join(stateBase, scope);
|
||||
const archive = await seedArchive(root);
|
||||
let checks = 0;
|
||||
|
||||
const result = await evidenceService({ isSessionBusy: () => ++checks > 1 }).tick();
|
||||
|
||||
expect(result).toMatchObject({ evidenceCandidates: 1, evidenceRemoved: 0 });
|
||||
expect(result.skipped.session_active).toBe(1);
|
||||
expect(await exists(archive)).toBe(true);
|
||||
});
|
||||
|
||||
it("reports evidence without deleting it in dry run", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const archive = await seedArchive(session.root);
|
||||
const quarantine = await seedQuarantine(session.scope);
|
||||
const leftover = path.join(session.root, "continuity-breaks", `.expired-${randomUUID()}`);
|
||||
await fs.mkdir(leftover);
|
||||
|
||||
const result = await evidenceService({ dryRun: true }).tick();
|
||||
|
||||
expect(result).toMatchObject({ evidenceCandidates: 2, evidenceRemoved: 0, evidenceReclaimedBytes: 0 });
|
||||
expect(result.evidenceCandidateBytes).toBeGreaterThan(0);
|
||||
expect(await exists(archive)).toBe(true);
|
||||
expect(await exists(quarantine)).toBe(true);
|
||||
expect(await exists(leftover)).toBe(true);
|
||||
});
|
||||
|
||||
it("skips symlinked evidence without following it", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const outside = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-evidence-outside-"));
|
||||
try {
|
||||
await fs.writeFile(path.join(outside, "keep.txt"), "keep");
|
||||
const stampedAt = now.getTime() - 10 * DAY_MS;
|
||||
await fs.mkdir(path.join(session.root, "continuity-breaks"), { recursive: true });
|
||||
const archiveLink = path.join(session.root, "continuity-breaks", `${stampedAt}-${randomUUID()}`);
|
||||
await fs.symlink(outside, archiveLink);
|
||||
await fs.mkdir(path.join(stateBase, "quarantine"), { recursive: true });
|
||||
const quarantineLink = path.join(
|
||||
stateBase,
|
||||
"quarantine",
|
||||
`${session.scope}.identity_indeterminate.${stampedAt}.4242.${randomUUID()}`,
|
||||
);
|
||||
await fs.symlink(outside, quarantineLink);
|
||||
|
||||
const result = await evidenceService().tick();
|
||||
|
||||
expect(result).toMatchObject({ evidenceCandidates: 0, evidenceRemoved: 0 });
|
||||
expect(result.skipped.unsafe_path).toBe(2);
|
||||
expect(await exists(archiveLink)).toBe(true);
|
||||
expect(await exists(quarantineLink)).toBe(true);
|
||||
expect(await exists(path.join(outside, "keep.txt"))).toBe(true);
|
||||
} finally {
|
||||
await fs.rm(outside, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("skips symlinked evidence folders without following them", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const outside = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-evidence-outside-"));
|
||||
try {
|
||||
const stampedAt = now.getTime() - 10 * DAY_MS;
|
||||
const inside = path.join(outside, `${stampedAt}-${randomUUID()}`);
|
||||
await fs.mkdir(inside);
|
||||
await fs.utimes(inside, new Date(stampedAt), new Date(stampedAt));
|
||||
await fs.symlink(outside, path.join(session.root, "continuity-breaks"));
|
||||
await fs.symlink(outside, path.join(stateBase, "quarantine"));
|
||||
|
||||
const result = await evidenceService().tick();
|
||||
|
||||
expect(result).toMatchObject({ evidenceCandidates: 0, evidenceRemoved: 0 });
|
||||
expect(result.skipped.unsafe_path).toBe(2);
|
||||
expect(await exists(inside)).toBe(true);
|
||||
} finally {
|
||||
await fs.rm(outside, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("removes quarantine entries older than the retention window and keeps recent ones", async () => {
|
||||
const scope = await emptyStateBase();
|
||||
const old = await seedQuarantine(scope, { stampedDaysAgo: 9 });
|
||||
const recent = await seedQuarantine(scope, { stampedDaysAgo: 2 });
|
||||
|
||||
const result = await evidenceService().tick();
|
||||
|
||||
expect(result).toMatchObject({ evidenceCandidates: 1, evidenceRemoved: 1, failed: 0 });
|
||||
expect(result.skipped.recent_evidence).toBe(1);
|
||||
expect(await exists(old)).toBe(false);
|
||||
expect(await exists(recent)).toBe(true);
|
||||
expect(await fs.readdir(path.join(stateBase, "quarantine"))).toEqual([path.basename(recent)]);
|
||||
});
|
||||
|
||||
it("keeps cleanup source archives in quarantine", async () => {
|
||||
const scope = await emptyStateBase();
|
||||
const entry = await seedQuarantine(scope, { name: `${scope}.identity_indeterminate.cleanup.${randomUUID()}` });
|
||||
const old = new Date(now.getTime() - 30 * DAY_MS);
|
||||
await fs.utimes(entry, old, old);
|
||||
|
||||
const result = await evidenceService().tick();
|
||||
|
||||
expect(result.evidenceCandidates).toBe(0);
|
||||
expect(result.skipped.cleanup_archive).toBe(1);
|
||||
expect(await exists(entry)).toBe(true);
|
||||
});
|
||||
|
||||
it("finishes removing evidence left over from an interrupted removal", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const archive = await seedArchive(session.root, { stampedDaysAgo: 1 });
|
||||
const leftover = path.join(session.root, "continuity-breaks", `.expired-${randomUUID()}`);
|
||||
await fs.mkdir(path.join(leftover, "codex-home"), { recursive: true });
|
||||
const quarantineLeftover = path.join(stateBase, "quarantine", `.expired-${randomUUID()}`);
|
||||
await fs.mkdir(quarantineLeftover, { recursive: true });
|
||||
|
||||
await evidenceService().tick();
|
||||
|
||||
expect(await exists(leftover)).toBe(false);
|
||||
expect(await exists(quarantineLeftover)).toBe(false);
|
||||
expect(await exists(archive)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([false, true])("pages evidence removals across ticks (dry run %s)", async (dryRun) => {
|
||||
const session = await seedSession({ issueStatus: "in_progress" });
|
||||
const archives = [await seedArchive(session.root), await seedArchive(session.root), await seedArchive(session.root)];
|
||||
const quarantine = await seedQuarantine(session.scope);
|
||||
const retention = evidenceService({ dryRun, maxRemovals: 2 });
|
||||
|
||||
const results = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const result = await retention.tick();
|
||||
results.push(result);
|
||||
if (!result.truncated) break;
|
||||
}
|
||||
|
||||
expect(results.map((result) => result.evidenceCandidates)).toEqual([2, 2]);
|
||||
expect(results.map((result) => result.truncated)).toEqual([true, false]);
|
||||
for (const archive of archives) expect(await exists(archive)).toBe(dryRun);
|
||||
expect(await exists(quarantine)).toBe(dryRun);
|
||||
});
|
||||
|
||||
it("does not recount a session's backup when its archives resume on the next tick", async () => {
|
||||
const session = await seedSession();
|
||||
await seedArchive(session.root);
|
||||
await seedArchive(session.root);
|
||||
const retention = evidenceService({ dryRun: true, maxRemovals: 2 });
|
||||
|
||||
const first = await retention.tick();
|
||||
const second = await retention.tick();
|
||||
|
||||
expect(first).toMatchObject({ candidates: 1, evidenceCandidates: 1, truncated: true });
|
||||
expect(second).toMatchObject({ candidates: 0, evidenceCandidates: 1, truncated: false });
|
||||
});
|
||||
it("removes old quarantine entries even when their session's issue is still open", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
const entry = await seedQuarantine(session.scope);
|
||||
await fs.cp(path.join(session.root, "control-plane"), path.join(entry, "control-plane"), { recursive: true });
|
||||
|
||||
expect(await evidenceService().tick()).toMatchObject({ evidenceRemoved: 1 });
|
||||
expect(await exists(entry)).toBe(false);
|
||||
});
|
||||
|
||||
it("stays quiet until retained evidence is due and wakes when new evidence is written", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
const recent = await seedArchive(session.root, { stampedDaysAgo: 2 });
|
||||
const retention = evidenceService();
|
||||
const at = (days: number) => ({ now: new Date(now.getTime() + days * DAY_MS) });
|
||||
|
||||
expect((await retention.tick()).skipped.recent_evidence).toBe(1);
|
||||
expect((await retention.tick(at(4))).skipped.recent_evidence).toBeUndefined();
|
||||
expect(await retention.tick(at(6))).toMatchObject({ evidenceRemoved: 1 });
|
||||
expect(await exists(recent)).toBe(false);
|
||||
|
||||
const written = await seedArchive(session.root, { stampedDaysAgo: 0 });
|
||||
expect(await retention.tick(at(8))).toMatchObject({ evidenceCandidates: 0 });
|
||||
retention.noteEvidence(now.getTime());
|
||||
expect(await retention.tick(at(8))).toMatchObject({ evidenceRemoved: 1 });
|
||||
expect(await exists(written)).toBe(false);
|
||||
});
|
||||
|
||||
it("retries evidence of a busy session on the next tick", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
const archive = await seedArchive(session.root);
|
||||
let busy = true;
|
||||
const retention = evidenceService({ isSessionBusy: () => busy });
|
||||
|
||||
expect(await retention.tick()).toMatchObject({ evidenceRemoved: 0 });
|
||||
busy = false;
|
||||
expect(await retention.tick()).toMatchObject({ evidenceRemoved: 1 });
|
||||
expect(await exists(archive)).toBe(false);
|
||||
});
|
||||
|
||||
it("reports dry-run evidence once until new evidence is written", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
await seedArchive(session.root);
|
||||
const retention = evidenceService({ dryRun: true });
|
||||
|
||||
expect(await retention.tick()).toMatchObject({ evidenceCandidates: 1 });
|
||||
expect(await retention.tick()).toMatchObject({ evidenceCandidates: 0 });
|
||||
retention.noteEvidence(now.getTime());
|
||||
expect(await retention.tick({ now: new Date(now.getTime() + 7 * DAY_MS) })).toMatchObject({ evidenceCandidates: 1 });
|
||||
});
|
||||
|
||||
it("counts inspected archives and leftovers against the entry limit", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
const archives = [
|
||||
await seedArchive(session.root, { stampedDaysAgo: 1 }),
|
||||
await seedArchive(session.root, { stampedDaysAgo: 2 }),
|
||||
await seedArchive(session.root, { stampedDaysAgo: 3 }),
|
||||
];
|
||||
const leftovers = [randomUUID(), randomUUID()].map((id) => path.join(session.root, "continuity-breaks", `.expired-${id}`));
|
||||
for (const leftover of leftovers) await fs.mkdir(leftover);
|
||||
const retention = evidenceService({ maxEntries: 2 });
|
||||
|
||||
const first = await retention.tick();
|
||||
expect(first).toMatchObject({ checked: 2, truncated: true });
|
||||
expect(first.skipped.recent_evidence).toBeUndefined();
|
||||
expect((await fs.readdir(path.join(session.root, "continuity-breaks"))).filter((name) => name.startsWith(".expired-"))).toHaveLength(1);
|
||||
|
||||
const results = [first];
|
||||
for (let i = 0; i < 6 && results.at(-1)!.truncated; i++) results.push(await retention.tick());
|
||||
expect(results.at(-1)!.truncated).toBe(false);
|
||||
expect(results.every((result) => result.checked <= 2)).toBe(true);
|
||||
expect(results.every((result) => (result.skipped.recent_evidence ?? 0) <= 2)).toBe(true);
|
||||
expect(results.reduce((sum, result) => sum + (result.skipped.recent_evidence ?? 0), 0)).toBe(3);
|
||||
for (const leftover of leftovers) expect(await exists(leftover)).toBe(false);
|
||||
for (const archive of archives) expect(await exists(archive)).toBe(true);
|
||||
});
|
||||
|
||||
it("stops removing leftovers once the sweep must stop", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
await fs.mkdir(path.join(session.root, "continuity-breaks"));
|
||||
const leftover = path.join(session.root, "continuity-breaks", `.expired-${randomUUID()}`);
|
||||
await fs.mkdir(leftover);
|
||||
let calls = 0;
|
||||
|
||||
const retention = evidenceService({ shouldContinue: () => ++calls !== 2 });
|
||||
|
||||
await retention.tick();
|
||||
expect(await exists(leftover)).toBe(true);
|
||||
await retention.tick();
|
||||
expect(await exists(leftover)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps a due evidence pass pending when drain stops it before any session", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
const archive = await seedArchive(session.root);
|
||||
let draining = true;
|
||||
const retention = evidenceService({ shouldContinue: () => !draining });
|
||||
|
||||
expect(await retention.tick()).toMatchObject({ checked: 0, truncated: true });
|
||||
draining = false;
|
||||
expect(await retention.tick()).toMatchObject({ evidenceRemoved: 1 });
|
||||
expect(await exists(archive)).toBe(false);
|
||||
});
|
||||
|
||||
it("finishes one-entry batches without recounting a resumed session", async () => {
|
||||
const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 });
|
||||
const archives = [await seedArchive(session.root), await seedArchive(session.root)];
|
||||
const quarantine = await seedQuarantine(session.scope);
|
||||
const retention = evidenceService({ maxEntries: 1 });
|
||||
|
||||
const results = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
results.push(await retention.tick());
|
||||
if (!results.at(-1)!.truncated) break;
|
||||
}
|
||||
|
||||
expect(results.at(-1)!.truncated).toBe(false);
|
||||
for (const archive of archives) expect(await exists(archive)).toBe(false);
|
||||
expect(await exists(quarantine)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { spawn } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
@@ -207,6 +208,40 @@ suite("settled native run process identity", () => {
|
||||
expect((await readRun(runId)).processPid).toBeNull();
|
||||
});
|
||||
|
||||
it("releases the stopped identity without reading saved execution payloads", async () => {
|
||||
const { runId } = await seed();
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const before = await readRun(runId);
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { ...before.contextSnapshot, savedTask: bulk },
|
||||
runnerProfileJson: { ...before.runnerProfileJson, sessionCheckpoint: { providerSessionId: "thread", savedState: bulk },
|
||||
nativeExecutionInput: { task: bulk } }, resultJson: { savedResult: bulk }, stdoutExcerpt: bulk,
|
||||
}).where(eq(heartbeatRuns.id, runId));
|
||||
const saved = await readRun(runId);
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try { expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); }
|
||||
finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
expect(summary.responseValueBytes).toBeLessThan(8192);
|
||||
const after = await readRun(runId);
|
||||
expect(after).toEqual({ ...saved, processPid: null, processGroupId: null, processStartedAt: null,
|
||||
nextEventSeq: saved.nextEventSeq + 1, updatedAt: after.updatedAt });
|
||||
expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(true);
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
});
|
||||
|
||||
it.each([null, false, 0, "", {}, []].map(checkpoint => ({ checkpoint })))(
|
||||
"preserves checkpoint presence semantics when releasing an identity ($checkpoint)", async ({ checkpoint }) => {
|
||||
const { runId, pid } = await seed({ checkpoint: false });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: checkpoint } })
|
||||
.where(eq(heartbeatRuns.id, runId));
|
||||
expect(await releaseSettledNativeRunProcess(db, runId)).toBe(checkpoint === null ? "skipped" : "released");
|
||||
expect((await readRun(runId)).processPid).toBe(checkpoint === null ? pid : null);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the identity while the process is still alive, then releases it after exit", async () => {
|
||||
const child = startDetached();
|
||||
children.push(child);
|
||||
|
||||
@@ -14,7 +14,7 @@ import { realizeExecutionWorkspace } from "../services/workspace-runtime.js";
|
||||
import { executionWorkspaceService } from "../services/execution-workspaces.js";
|
||||
import { prepareProjectRepositoryWorkspaces } from "../services/heartbeat/workspaces.js";
|
||||
import { expireReconstructableGitWorkspace } from "../services/git-workspace-expiration.js";
|
||||
import { currentRestorePointSnapshot } from "../services/git-restore-point.js";
|
||||
import { currentRestorePointSnapshot, restorePointHistoryDigest } from "../services/git-restore-point.js";
|
||||
import { nestedRepositoryBaseCandidates, nestedWorktreeBranchPrefix, readLinkedNestedCheckout, withNestedRepositoryBaseLock } from "../services/nested-repository-worktrees.js";
|
||||
import { listNestedCloneMigrationNames, migrateNestedRepositoryClone, nextNestedCloneMigrationBatch, type NestedCloneMigrationOptions } from "../services/nested-clone-migration.js";
|
||||
import { getStorageService } from "../storage/index.js";
|
||||
@@ -225,6 +225,41 @@ describeEmbeddedPostgres("nested repository clone migration", () => {
|
||||
expect(git(f.checkout, "cat-file", "-t", feature)).toBe("commit");
|
||||
});
|
||||
|
||||
it("expires a converted checkout whose only extra history is remote-tracking and restores that history", async () => {
|
||||
const f = await fixture({ issueStatus: "done" });
|
||||
git(f.nested.author, "checkout", "-q", "-b", "squashed-away");
|
||||
await fs.writeFile(path.join(f.nested.author, "feature.txt"), "feature\n");
|
||||
git(f.nested.author, "add", "."); git(f.nested.author, "commit", "-qm", "Feature");
|
||||
const feature = git(f.nested.author, "rev-parse", "HEAD");
|
||||
git(f.nested.author, "push", "-q", f.nested.remote, "squashed-away");
|
||||
git(f.checkout, "fetch", "-q", "origin");
|
||||
git(f.nested.remote, "branch", "-D", "squashed-away");
|
||||
git(f.nested.remote, "reflog", "expire", "--expire=now", "--all");
|
||||
git(f.nested.remote, "gc", "-q", "--prune=now");
|
||||
expect(await f.migrate()).toMatchObject({ outcome: "migrated" });
|
||||
|
||||
expect(await f.expire()).toMatchObject({ expired: true, restorePoint: true, skipReason: null });
|
||||
for (const line of ownedRefs(f.base, f.prefix).split("\n").filter(Boolean)) git(f.base, "update-ref", "-d", line.split(" ")[0]!);
|
||||
git(f.base, "reflog", "expire", "--expire=now", "--all");
|
||||
git(f.base, "gc", "-q", "--prune=now");
|
||||
expect(() => git(f.base, "cat-file", "-e", `${feature}^{commit}`)).toThrow();
|
||||
expect(await f.reopen()).toMatchObject({ ok: true, reopened: true });
|
||||
expect(git(f.checkout, "cat-file", "-t", feature)).toBe("commit");
|
||||
});
|
||||
|
||||
it("fingerprints a converted checkout's own history rather than the shared clone's remote refs", async () => {
|
||||
const f = await fixture();
|
||||
expect(await f.migrate()).toMatchObject({ outcome: "migrated" });
|
||||
const head = git(f.checkout, "rev-parse", "HEAD");
|
||||
const before = await restorePointHistoryDigest(f.checkout);
|
||||
git(f.base, "update-ref", "refs/remotes/origin/another-task", head);
|
||||
expect(await restorePointHistoryDigest(f.checkout)).toBe(before);
|
||||
git(f.base, "update-ref", `refs/heads/${f.prefix}remotes/origin/extra`, head);
|
||||
expect(await restorePointHistoryDigest(f.checkout)).not.toBe(before);
|
||||
for (let index = 0; index < 64; index++) git(f.base, "update-ref", `refs/heads/${f.prefix}remotes/origin/bulk-${index}`, head);
|
||||
await expect(restorePointHistoryDigest(f.checkout)).rejects.toThrow("managed_checkout_local_tip_budget");
|
||||
});
|
||||
|
||||
it("finishes an interrupted conversion after the workspace is archived and forgets one whose workspace is gone", async () => {
|
||||
const f = await fixture();
|
||||
const { unpushed } = await nestedWork(f.checkout);
|
||||
|
||||
@@ -21,6 +21,7 @@ const apiPrefixes: Record<string, string> = {
|
||||
"activity.ts": "/api",
|
||||
"adapters.ts": "/api",
|
||||
"agents.ts": "/api",
|
||||
"agent-project-access.ts": "/api",
|
||||
"agent-commentary.ts": "/api",
|
||||
"agent-avatars.ts": "/api",
|
||||
"agent-profile-avatar.ts": "/api",
|
||||
@@ -987,6 +988,21 @@ describe("openapi routes", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("documents additive project access as board-only with a strict request", () => {
|
||||
const { spec } = loadSpecRoutes();
|
||||
const path = spec.paths["/api/agents/{id}/project-access"];
|
||||
for (const operation of [path.get, path.post]) {
|
||||
expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" });
|
||||
expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]);
|
||||
expect(operation.responses["200"].content["application/json"].schema.required).toEqual(["projectIds", "canManage"]);
|
||||
}
|
||||
expect(path.post.requestBody.content["application/json"].schema).toMatchObject({
|
||||
additionalProperties: false, required: ["projectIds"],
|
||||
properties: { projectIds: { type: "array", items: { type: "string", format: "uuid" } } },
|
||||
});
|
||||
expect(path.post.responses["409"]).toBeDefined();
|
||||
});
|
||||
|
||||
it("documents the authenticated personal primary-agent contract", () => {
|
||||
const { spec } = loadSpecRoutes();
|
||||
const path = spec.paths["/api/companies/{companyId}/primary-agent/me"];
|
||||
|
||||
@@ -29,6 +29,23 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
}
|
||||
return { companyId, agentId, issueId, runId, messageIds };
|
||||
}
|
||||
it.each(["github_credentials", "request_authentication"] as const)("retains typed trust bindings for %s and keeps other identity reads complete", async (readScope) => {
|
||||
const input = await seed();
|
||||
await initializeRunIdentity(db, { ...input, responsibleUserId: "owner", cause: "queued" });
|
||||
for (const value of [null, true, 42, " padded ", [], {}, { future: { opaque: true } }]) {
|
||||
const contextSnapshot = { issueId: input.issueId, projectId: value, executionPolicy: value, savedInput: { retained: true } };
|
||||
const runnerProfileJson = { nativeExecutionInput: { opaque: value }, savedInput: "retained" };
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot, runnerProfileJson }).where(eq(heartbeatRuns.id, input.runId));
|
||||
const narrow = await captureRunIdentity(db, input, { readScope });
|
||||
expect(narrow.run.contextSnapshot).toEqual({ issueId: input.issueId, taskId: null, projectId: value, executionPolicy: value });
|
||||
expect(narrow.run.runnerProfileJson).toBeNull();
|
||||
const full = await captureRunIdentity(db, input);
|
||||
expect(full.run.contextSnapshot).toEqual(contextSnapshot);
|
||||
expect(full.run.runnerProfileJson).toEqual(runnerProfileJson);
|
||||
expect(narrow.context).toEqual(full.context);
|
||||
}
|
||||
});
|
||||
|
||||
it("retains mixed-author delivery order without rewriting task ownership", async () => {
|
||||
const input = await seed();
|
||||
await initializeRunIdentity(db, { ...input, responsibleUserId: "owner", cause: "queued" });
|
||||
@@ -106,24 +123,25 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
},
|
||||
);
|
||||
|
||||
it("holds acquisition during uncertain steering, preserves snapshots, and never rewinds on replay", async () => {
|
||||
it.each([undefined, "github_credentials", "request_authentication"] as const)("holds acquisition during uncertain steering and never rewinds on replay with %s", async (readScope) => {
|
||||
const options = readScope ? { readScope } : undefined;
|
||||
const input = await seed();
|
||||
await initializeRunIdentity(db, { ...input, messageIds: [input.messageIds[0]], responsibleUserId: "A", cause: "instruction" });
|
||||
const startedUnderA = await captureRunIdentity(db, input);
|
||||
const startedUnderA = await captureRunIdentity(db, input, options);
|
||||
const b = await reserveSteeredIdentity(db, { ...input, messageId: input.messageIds[1] });
|
||||
expect(b).not.toBeNull();
|
||||
await expect(captureRunIdentity(db, input)).rejects.toThrow(/reconciled/);
|
||||
await expect(captureRunIdentity(db, input, options)).rejects.toThrow(/reconciled/);
|
||||
await expect(reserveSteeredIdentity(db, { ...input, messageId: input.messageIds[2] })).rejects.toThrow(/reconciled/);
|
||||
await db.transaction(async tx => {
|
||||
await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)).for("update");
|
||||
await acceptSteeredIdentity(tx, b!);
|
||||
});
|
||||
expect((await captureRunIdentity(db, input)).context?.responsibleUserId).toBe("B");
|
||||
expect((await captureRunIdentity(db, input, options)).context?.responsibleUserId).toBe("B");
|
||||
expect(startedUnderA.context?.responsibleUserId).toBe("A");
|
||||
const a = await reserveSteeredIdentity(db, { ...input, messageId: input.messageIds[2] });
|
||||
await acceptSteeredIdentity(db, a!);
|
||||
await acceptSteeredIdentity(db, b!);
|
||||
expect((await captureRunIdentity(db, input)).context?.responsibleUserId).toBe("A");
|
||||
expect((await captureRunIdentity(db, input, options)).context?.responsibleUserId).toBe("A");
|
||||
});
|
||||
it("keeps rejected steering unchanged and preserves the originating identity on retry", async () => {
|
||||
const input = await seed();
|
||||
@@ -158,16 +176,17 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
expect(captured.run.responsibleUserId).toBeNull();
|
||||
expect(captured.context?.parentContextId).toBe(origin.id);
|
||||
});
|
||||
it("recovers an uncertain acknowledgement from the authenticated native event journal", async () => {
|
||||
it.each([undefined, "github_credentials", "request_authentication"] as const)("recovers an uncertain native acknowledgement with %s", async (readScope) => {
|
||||
const options = readScope ? { readScope } : undefined;
|
||||
const input = await seed();
|
||||
await initializeRunIdentity(db, { ...input, messageIds: [], responsibleUserId: "A", cause: "instruction" });
|
||||
const pending = await reserveSteeredIdentity(db, { ...input, messageId: input.messageIds[1] });
|
||||
await expect(captureRunIdentity(db, input)).rejects.toThrow(/reconciled/);
|
||||
await expect(captureRunIdentity(db, input, options)).rejects.toThrow(/reconciled/);
|
||||
await db.insert(heartbeatRunEvents).values({ companyId: input.companyId, runId: input.runId,
|
||||
agentId: input.agentId, seq: 1, eventType: "item.completed", sourceEventId: randomUUID(),
|
||||
payload: { prpEvent: { turnId: "turn-1", itemId: `turn-1:steer:${pending!.messageId}`, payload: {kind: "steering_acknowledgement"} } },
|
||||
});
|
||||
expect((await captureRunIdentity(db, input)).context?.responsibleUserId).toBe("B");
|
||||
expect((await captureRunIdentity(db, input, options)).context?.responsibleUserId).toBe("B");
|
||||
await db.update(heartbeatRuns).set({status: "succeeded"}).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect((await reserveSteeredIdentity(db, { ...input, messageId: input.messageIds[1] }))?.status).toBe("accepted");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { eq, sql, type SQL } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createDb, plugins, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared";
|
||||
import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js";
|
||||
import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("sandbox provider schema reads", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
const clients: Db[] = [];
|
||||
const key = "fixture-schema-provider";
|
||||
const driver = (driverKey = key, configSchema: unknown = { type: "object" }) => ({
|
||||
driverKey, kind: "sandbox_provider", displayName: "Schema fixture", configSchema,
|
||||
}) as PluginEnvironmentDriverDeclaration;
|
||||
const manifest = (id: string, declarations: PluginEnvironmentDriverDeclaration[]): PaperclipPluginManifestV1 => ({
|
||||
id: `test.schema.${id}`, apiVersion: 1, version: "1.0.0", displayName: "Schema fixture",
|
||||
description: "Fixture", author: "Test", categories: ["automation"],
|
||||
capabilities: ["environment.drivers.register"], entrypoints: { worker: "worker.js" }, environmentDrivers: declarations,
|
||||
});
|
||||
async function seed(declarations = [driver()], installOrder = 0) {
|
||||
const id = randomUUID();
|
||||
await db.insert(plugins).values({
|
||||
id, pluginKey: `test.schema.${id}`, packageName: "@test/schema", version: "1.0.0",
|
||||
installOrder, status: "disabled", manifestJson: manifest(id, declarations),
|
||||
});
|
||||
return id;
|
||||
}
|
||||
function observer() {
|
||||
const observed = createDb(temporary.connectionString);
|
||||
clients.push(observed);
|
||||
return observed;
|
||||
}
|
||||
const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey });
|
||||
async function measure<T>(client: Db, operation: () => Promise<T>) {
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
|
||||
installDatabaseResponseCapture(client, { plugins }, capture);
|
||||
try {
|
||||
const value = await operation();
|
||||
capture.flush();
|
||||
const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
|
||||
return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0),
|
||||
calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) };
|
||||
} finally { capture.flush(); }
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("sandbox-schema-reads-");
|
||||
db = createDb(temporary.connectionString);
|
||||
}, 120_000);
|
||||
beforeEach(async () => { await db.delete(plugins); });
|
||||
afterEach(async () => { await Promise.all(clients.splice(0).map(client => client.$client.end())); });
|
||||
afterAll(async () => { await temporary?.cleanup(); });
|
||||
|
||||
it("keeps runtime secret paths current without retransferring an unchanged schema", async () => {
|
||||
const schema = { type: "object", description: randomBytes(16 * 1024).toString("hex"), properties: {
|
||||
credentials: { type: "object", properties: { token: { type: "string", format: "secret-ref" } } },
|
||||
} };
|
||||
const id = await seed([driver(key, schema)]), observed = observer();
|
||||
const before = await db.select().from(plugins).where(eq(plugins.id, id));
|
||||
const cold = await measure(observed, () => resolveSandboxProviderSecretRefPaths(observed, key));
|
||||
const warm = await measure(observed, () => resolveSandboxProviderSecretRefPaths(observed, key));
|
||||
expect(cold.value).toEqual(new Set(["credentials.token"]));
|
||||
expect(warm.value).toEqual(cold.value);
|
||||
expect(cold.calls).toBe(1);
|
||||
expect(warm.calls).toBe(1);
|
||||
expect(cold.bytes).toBeGreaterThan(32 * 1024);
|
||||
expect(warm.bytes).toBeLessThan(256);
|
||||
const changed = { type: "object", allOf: [{ properties: { apiKey: { type: "string", format: "secret-ref" } } }] };
|
||||
await db.update(plugins).set({ manifestJson: manifest(id, [driver(key, changed)]) }).where(eq(plugins.id, id));
|
||||
expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set(["apiKey"]));
|
||||
const environment = { id: randomUUID(), driver: "sandbox" as const, config: { provider: key, apiKey: randomUUID() } };
|
||||
await expect(resolveEnvironmentDriverConfigForRuntime(observed, null, environment)).rejects.toThrow("companyId");
|
||||
await db.update(plugins).set({ manifestJson: before[0].manifestJson }).where(eq(plugins.id, id));
|
||||
expect(await db.select().from(plugins).where(eq(plugins.id, id))).toEqual(before);
|
||||
});
|
||||
|
||||
it("preserves install order, first matching declarations, and default readiness checks", async () => {
|
||||
const a = await seed([driver(key, { title: "first" }), driver(key, { title: "second" })], 1);
|
||||
const b = await seed([driver(key, { title: "other plugin" })], 2), observed = observer();
|
||||
expect((await lookup(observed))).toEqual({ title: "first" });
|
||||
const full = await resolvePluginSandboxProviderDriverByKey({ db: observed, driverKey: key });
|
||||
expect(full?.plugin.id).toBe(a);
|
||||
expect(full?.plugin.status).toBe("disabled");
|
||||
expect(await resolvePluginSandboxProviderDriverByKey({ db: observed, driverKey: key, requireRunning: true })).toBeNull();
|
||||
await db.update(plugins).set({ installOrder: 0 }).where(eq(plugins.id, b));
|
||||
expect((await lookup(observed))).toEqual({ title: "other plugin" });
|
||||
await db.delete(plugins).where(eq(plugins.id, b));
|
||||
expect((await lookup(observed))).toEqual({ title: "first" });
|
||||
await db.update(plugins).set({ manifestJson: manifest(a, [driver("replacement")]) }).where(eq(plugins.id, a));
|
||||
expect(await lookup(observed)).toBeNull();
|
||||
});
|
||||
|
||||
it.each([null, true, "invalid", [], undefined])("preserves non-object schema behavior: %s", async value => {
|
||||
const first = driver(key, value);
|
||||
if (value === undefined) delete (first as Partial<PluginEnvironmentDriverDeclaration>).configSchema;
|
||||
await seed([first, driver(key, { properties: { secret: { format: "secret-ref" } } })]);
|
||||
const observed = observer();
|
||||
expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set());
|
||||
expect(await resolveSandboxProviderSecretRefPaths(observed, key)).toEqual(new Set());
|
||||
});
|
||||
|
||||
it("returns independent schema objects and keeps separate database clients cold", async () => {
|
||||
await seed([driver(key, { description: "x".repeat(4096), properties: { token: { format: "secret-ref" } } })]);
|
||||
const a = observer(), b = observer();
|
||||
const first = await lookup(a);
|
||||
first!.properties = {};
|
||||
expect(await resolveSandboxProviderSecretRefPaths(a, key)).toEqual(new Set(["token"]));
|
||||
const warm = await measure(a, () => lookup(a)), independent = await measure(b, () => lookup(b));
|
||||
expect(warm.bytes).toBeLessThan(256);
|
||||
expect(independent.bytes).toBeGreaterThan(4096);
|
||||
warm.value!.properties = {};
|
||||
expect(await resolveSandboxProviderSecretRefPaths(a, key)).toEqual(new Set(["token"]));
|
||||
});
|
||||
|
||||
it("bounds retained schemas and does not retain oversized definitions", async () => {
|
||||
const declarations = Array.from({ length: 33 }, (_, index) => driver(`provider-${index}`, { description: "x".repeat(2048) }));
|
||||
declarations.push(driver("oversized", { description: "x".repeat(128 * 1024) }));
|
||||
await seed(declarations);
|
||||
const observed = observer();
|
||||
for (let index = 0; index < 33; index++) await lookup(observed, `provider-${index}`);
|
||||
expect((await measure(observed, () => lookup(observed, "provider-32"))).bytes).toBeLessThan(256);
|
||||
expect((await measure(observed, () => lookup(observed, "provider-0"))).bytes).toBeGreaterThan(2048);
|
||||
await lookup(observed, "oversized");
|
||||
expect((await measure(observed, () => lookup(observed, "oversized"))).bytes).toBeGreaterThan(128 * 1024);
|
||||
});
|
||||
|
||||
it("binds unusual keys and never serves retained schemas after a lookup error", async () => {
|
||||
const unusual = "provider-\"quoted' OR true --";
|
||||
await seed([driver(unusual, { title: "exact provider" })]);
|
||||
const observed = observer();
|
||||
let fail = false;
|
||||
const guarded = { execute: async (query: SQL) => {
|
||||
if (fail) throw new Error("fixture database unavailable");
|
||||
return observed.execute(query);
|
||||
} } as unknown as Db;
|
||||
expect((await lookup(guarded, unusual))).toEqual({ title: "exact provider" });
|
||||
fail = true;
|
||||
await expect(lookup(guarded, unusual)).rejects.toThrow("fixture database unavailable");
|
||||
fail = false;
|
||||
expect((await lookup(guarded, unusual))).toEqual({ title: "exact provider" });
|
||||
expect(await lookup(guarded, "unknown-provider")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps each concurrent lookup tied to the database revision it observed", async () => {
|
||||
const id = await seed([driver(key, { title: "old" })]), observed = observer();
|
||||
let pause = false, release!: () => void, reached!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const paused = new Promise<void>(resolve => { reached = resolve; });
|
||||
const raced = { execute: async (query: SQL) => {
|
||||
const rows = await observed.execute(query);
|
||||
if (pause) { pause = false; reached(); await gate; }
|
||||
return rows;
|
||||
} } as unknown as Db;
|
||||
expect((await lookup(raced))).toEqual({ title: "old" });
|
||||
pause = true;
|
||||
const oldRead = lookup(raced);
|
||||
await paused;
|
||||
try {
|
||||
await db.update(plugins).set({ manifestJson: manifest(id, [driver(key, { title: "new" })]) }).where(eq(plugins.id, id));
|
||||
expect((await lookup(raced))).toEqual({ title: "new" });
|
||||
} finally { release(); }
|
||||
expect((await oldRead)).toEqual({ title: "old" });
|
||||
expect((await lookup(raced))).toEqual({ title: "new" });
|
||||
expect((await db.execute<{ title: string }>(sql`select ${plugins.manifestJson}->'environmentDrivers'->0->'configSchema'->>'title' as title from ${plugins} where ${plugins.id}=${id}`))[0].title).toBe("new");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { activityLog, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { createHeartbeatRunState } from "../services/heartbeat/run-state.js";
|
||||
import { activityService } from "../services/activity.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("task cold-open reads", () => {
|
||||
let fixture: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
const companyId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
beforeAll(async () => {
|
||||
fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-");
|
||||
db = createDb(fixture.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Cold open", issuePrefix: `T${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Writer", role: "engineer", adapterType: "codex_local" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Original instruction", description: "Read this first", identifier: `T${companyId.slice(0, 6)}-1`, issueNumber: 1 });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, runtimeMode: "native", status: "succeeded" });
|
||||
}, 30_000);
|
||||
afterAll(async () => { await fixture?.cleanup(); });
|
||||
function app(allowedCompany = companyId) {
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = { type: "board", userId: "local-board", source: allowedCompany === companyId ? "local_implicit" : "cloud_tenant", companyIds: [allowedCompany], isInstanceAdmin: allowedCompany === companyId };
|
||||
next();
|
||||
});
|
||||
app.use("/api", issueRoutes(db, {} as any));
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
}
|
||||
it("returns the core instruction without related detail and keeps company checks", async () => {
|
||||
const response = await request(app()).get(`/api/issues/${issueId}?view=core`);
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(response.body).toMatchObject({ id: issueId, description: "Read this first" });
|
||||
expect(response.body).not.toHaveProperty("ancestors");
|
||||
expect(response.body).not.toHaveProperty("workProducts");
|
||||
const denied = await request(app(randomUUID())).get(`/api/issues/${issueId}?view=core`);
|
||||
expect([403, 404]).toContain(denied.status);
|
||||
});
|
||||
it("returns a bounded recent exchange plus the latest human instruction", async () => {
|
||||
const humanId = randomUUID();
|
||||
await db.insert(issueComments).values([
|
||||
{ id: humanId, companyId, issueId, body: "Human direction", authorType: "user", authorUserId: "local-board", createdAt: new Date(0) },
|
||||
...Array.from({ length: 20 }, (_, i) => ({ companyId, issueId, authorType: "agent" as const, authorAgentId: agentId, body: `Reply ${i}`, createdAt: new Date(1000 + i) })),
|
||||
]);
|
||||
const response = await request(app()).get(`/api/issues/${issueId}/recent-comments`);
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(response.body.hasMore).toBe(true);
|
||||
expect(response.body.comments.map((c: { body: string }) => c.body)).toEqual(["Human direction", "Reply 19", "Reply 18", "Reply 17", "Reply 16"]);
|
||||
expect(response.body.latestCommentId).toBe(response.body.comments[1].id);
|
||||
expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${issueId}/recent-comments`)).status);
|
||||
});
|
||||
it.each(["user", null] as const)("preserves initial and latest human instructions without a description (authorType=%s)", async (authorType) => {
|
||||
const bareIssueId = randomUUID();
|
||||
await db.insert(issues).values({ id: bareIssueId, companyId, title: "No description" });
|
||||
await db.insert(issueComments).values([
|
||||
{ companyId, issueId: bareIssueId, body: "Human direction", authorType, authorUserId: "local-board", createdAt: new Date(0) },
|
||||
{ companyId, issueId: bareIssueId, body: "Newest direction", authorType, authorUserId: "local-board", createdAt: new Date(50_000) },
|
||||
...Array.from({ length: 6 }, (_, i) => ({ companyId, issueId: bareIssueId, body: `New reply ${i}`, authorType: "agent" as const, authorAgentId: agentId, createdAt: new Date(60_000 + i) })),
|
||||
]);
|
||||
const response = await request(app()).get(`/api/issues/${bareIssueId}/recent-comments`);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.comments.map((c: { body: string }) => c.body)).toEqual([
|
||||
"Human direction", "Newest direction", "New reply 5", "New reply 4", "New reply 3", "New reply 2",
|
||||
]);
|
||||
});
|
||||
it("loads the original input receipts outside the recent activity window and ignores replay receipts", async () => {
|
||||
const commentId = randomUUID();
|
||||
const receiptIssueId = randomUUID();
|
||||
await db.insert(issues).values({ id: receiptIssueId, companyId, title: "Steering receipts" });
|
||||
await db.insert(activityLog).values([
|
||||
{ companyId, actorId: "test", entityType: "issue", entityId: receiptIssueId, action: "issue.queued_comment_steered", createdAt: new Date(1), details: { commentId, targetRunId: runId } },
|
||||
{ companyId, actorId: "test", entityType: "issue", entityId: receiptIssueId, action: "issue.queued_comment_steered", createdAt: new Date(2), details: { commentId, targetRunId: runId, duplicate: true } },
|
||||
...Array.from({ length: 51 }, () => ({ companyId, actorId: "test", entityType: "issue", entityId: receiptIssueId, action: "issue.updated" })),
|
||||
]);
|
||||
const activity = activityService(db);
|
||||
expect((await activity.forIssue(receiptIssueId, 50)).some(row => row.action === "issue.queued_comment_steered")).toBe(false);
|
||||
const facts = await activity.inputFactsForIssue(companyId, receiptIssueId, [commentId]);
|
||||
expect(facts).toHaveLength(1);
|
||||
expect(facts[0].createdAt).toEqual(new Date(1));
|
||||
expect(await activity.inputFactsForIssue(companyId, receiptIssueId, [randomUUID()])).toEqual([]);
|
||||
expect(await activity.inputFactsForIssue(randomUUID(), receiptIssueId, [commentId])).toEqual([]);
|
||||
expect(await activity.inputFactsForIssue(companyId, randomUUID(), [commentId])).toEqual([]);
|
||||
});
|
||||
|
||||
it("pages conversation events without tools, including identity-less deltas and requests", async () => {
|
||||
const event = (seq: number, eventType: string, payload: Record<string, unknown>, itemId?: string) => ({
|
||||
companyId, runId, agentId, seq, eventType, payload: { prpEvent: { schema: "paperclip.prp.event.v1", eventType, itemId, payload } },
|
||||
});
|
||||
await db.insert(heartbeatRunEvents).values([
|
||||
event(1, "item.started", { kind: "toolCall", arguments: "large tool arguments" }, "tool"),
|
||||
event(2, "item.completed", { kind: "toolCall", output: "x".repeat(100_000) }, "tool"),
|
||||
event(3, "item.started", { kind: "agentMessage" }, "message"),
|
||||
event(4, "item.delta", { text: "Current progress" }, "message"),
|
||||
event(5, "runtime_request.opened", { requestId: "request", prompt: "Which option?" }),
|
||||
event(6, "runtime_request.resolved", { requestId: "request" }),
|
||||
event(7, "item.completed", { kind: "assistant_message", text: "Final answer" }),
|
||||
event(8, "turn.completed", {}),
|
||||
]);
|
||||
const service = createHeartbeatRunState(db);
|
||||
const first = await service.listEvents(runId, 0, 2, true);
|
||||
expect(first.map(e => e.seq)).toEqual([3, 4]);
|
||||
const rest = await service.listEvents(runId, 4, 1000, true);
|
||||
expect(rest.map(e => e.seq)).toEqual([5, 6, 7, 8]);
|
||||
expect(JSON.stringify([...first, ...rest])).not.toContain("large tool arguments");
|
||||
expect(await service.listEvents(runId)).toHaveLength(8);
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, issueId, status: "succeeded", createdAt: new Date(Date.now() + 1000) });
|
||||
const activity = activityService(db);
|
||||
await db.insert(activityLog).values(Array.from({ length: 55 }, () => ({
|
||||
companyId, actorId: "test", action: "issue.updated", entityType: "issue", entityId: issueId,
|
||||
})));
|
||||
expect(await activity.forIssue(issueId, 50)).toHaveLength(50);
|
||||
expect(await activity.forIssue(issueId)).toHaveLength(55);
|
||||
expect(await activity.runsForIssue(companyId, issueId, null, 1)).toHaveLength(1);
|
||||
expect(await activity.runsForIssue(companyId, issueId)).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,11 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
agents, companies, createDb, toolApplications, toolCatalogEntries,
|
||||
toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { toolAccessService } from "../services/tool-access.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
@@ -49,6 +50,11 @@ describeDatabase("effective tool assignments", () => {
|
||||
expect(assignments).toEqual({ ...full, allowedTools: full.allowedTools.map(
|
||||
({ id, applicationId, connectionId, toolName }) => ({ id, applicationId, connectionId, toolName }),
|
||||
) });
|
||||
const connections = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]);
|
||||
expect(connections).toEqual({
|
||||
installedConnections: full.installedConnections,
|
||||
allowedConnectionIds: [...new Set(full.allowedTools.filter(tool => tool.connectionId === f.connection.id).map(tool => tool.connectionId))],
|
||||
});
|
||||
return { service, full, assignments };
|
||||
}
|
||||
|
||||
@@ -130,4 +136,161 @@ describeDatabase("effective tool assignments", () => {
|
||||
expect(observed[1].bytes).toBeLessThan(observed[0].bytes / 100);
|
||||
expect(full.allowedTools[0].inputSchema).toEqual(f.read.inputSchema);
|
||||
});
|
||||
it("scopes catalog reads, omits rule payloads, and keeps current full definitions", async () => {
|
||||
const f = await fixture();
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const [rule] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id,
|
||||
selectorType: "application", effect: "include", applicationId: f.application.id, conditions: { savedRuleMetadata: bulk } }).returning();
|
||||
const [unrelated] = await db.insert(toolConnections).values({ companyId: f.company.id, applicationId: f.application.id,
|
||||
name: "Other connection", uid: randomUUID(), transport: "mcp_remote", status: "active", enabled: true }).returning();
|
||||
await db.insert(toolCatalogEntries).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.company.id,
|
||||
applicationId: f.application.id, connectionId: unrelated.id, name: `other_${index}`, toolName: `other_${index}`, versionHash: `v${index}` })));
|
||||
const service = toolAccessService(db);
|
||||
const before = await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id);
|
||||
expect(before.allowedTools).toHaveLength(102);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { toolCatalogEntries, toolProfileEntries }, capture);
|
||||
let scoped: Awaited<ReturnType<typeof service.getConnectionAssignmentsForAgent>>;
|
||||
try { scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); }
|
||||
finally { capture.flush(); }
|
||||
expect(Number(events.at(-1)?.completedQueries)).toBeGreaterThan(0);
|
||||
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(16 * 1024);
|
||||
const catalogQueries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& event.table === "tool_catalog_entries" && event.operation === "select");
|
||||
expect(catalogQueries).toHaveLength(1);
|
||||
expect(catalogQueries[0]).toMatchObject({ calls: 1, rows: 2 });
|
||||
expect(scoped!).toEqual({ allowedConnectionIds: [f.connection.id], installedConnections: before.installedConnections });
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
expect((await db.select().from(toolProfileEntries).where(eq(toolProfileEntries.id, rule.id)))[0]).toEqual(rule);
|
||||
expect(await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id)).toEqual(before);
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [])).allowedConnectionIds).toEqual([]);
|
||||
await db.delete(toolProfileBindings).where(eq(toolProfileBindings.profileId, f.profile.id));
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
});
|
||||
|
||||
it("omits unrelated rules and bindings while retaining current connection decisions", async () => {
|
||||
const f = await fixture();
|
||||
const [otherApplication] = await db.insert(toolApplications).values({ companyId: f.company.id, name: "Other app", type: "mcp_http" }).returning();
|
||||
const [otherConnection] = await db.insert(toolConnections).values({ companyId: f.company.id, applicationId: otherApplication.id,
|
||||
name: "Other connection", uid: randomUUID(), transport: "mcp_remote" }).returning();
|
||||
const [otherTool] = await db.insert(toolCatalogEntries).values({ companyId: f.company.id, applicationId: otherApplication.id,
|
||||
connectionId: otherConnection.id, name: "other", toolName: "other", versionHash: "other", riskLevel: "destructive" }).returning();
|
||||
await db.insert(toolProfileEntries).values(Array.from({ length: 200 }, (_, index) => [
|
||||
{ selectorType: "application" as const, applicationId: otherApplication.id },
|
||||
{ selectorType: "connection" as const, connectionId: otherConnection.id },
|
||||
{ selectorType: "catalog_entry" as const, catalogEntryId: otherTool.id },
|
||||
{ selectorType: "tool_name" as const, toolName: `unrelated_${index}` },
|
||||
{ selectorType: "risk_level" as const, riskLevel: "destructive" as const },
|
||||
].map(entry => ({ companyId: f.company.id, profileId: f.profile.id, effect: index % 2 ? "include" as const : "exclude" as const, ...entry }))).flat());
|
||||
await db.insert(toolProfileBindings).values(Array.from({ length: 200 }, () => ({ companyId: f.company.id,
|
||||
profileId: f.profile.id, targetType: "agent" as const, targetId: randomUUID() })));
|
||||
const [include, exclude] = await db.insert(toolProfileEntries).values([
|
||||
{ companyId: f.company.id, profileId: f.profile.id, selectorType: "connection", effect: "include", connectionId: f.connection.id },
|
||||
{ companyId: f.company.id, profileId: f.profile.id, selectorType: "catalog_entry", effect: "exclude", catalogEntryId: f.write.id },
|
||||
]).returning();
|
||||
const service = toolAccessService(db), before = await compare(f);
|
||||
const savedRules = await db.select().from(toolProfileEntries);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { toolProfileEntries, toolProfileBindings }, capture);
|
||||
try {
|
||||
const scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]);
|
||||
capture.flush();
|
||||
const rules = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_entries");
|
||||
const bindings = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
|
||||
expect(rules).toHaveLength(1);
|
||||
expect(Number(rules[0].responseValueBytes)).toBeLessThan(4096);
|
||||
expect(rules[0]).toMatchObject({ calls: 1, rows: 2 });
|
||||
expect(bindings).toHaveLength(1);
|
||||
expect(bindings[0]).toMatchObject({ calls: 1, rows: 1 });
|
||||
expect(scoped).toEqual({ allowedConnectionIds: [f.connection.id], installedConnections: before.full.installedConnections });
|
||||
expect(await db.select().from(toolProfileEntries)).toEqual(savedRules);
|
||||
await db.update(toolProfileEntries).set({ effect: "exclude" }).where(eq(toolProfileEntries.id, include.id));
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
await db.delete(toolProfileEntries).where(eq(toolProfileEntries.id, include.id));
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
expect((await db.select().from(toolProfileEntries).where(eq(toolProfileEntries.id, exclude.id)))[0]).toEqual(exclude);
|
||||
} finally { capture.flush(); }
|
||||
});
|
||||
|
||||
it("preserves null application matches and exact special-character tool names", async () => {
|
||||
const f = await fixture(), toolName = "quoted' path\\tool_🙂";
|
||||
await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.write.id));
|
||||
await db.update(toolCatalogEntries).set({ applicationId: null, toolName }).where(eq(toolCatalogEntries.id, f.read.id));
|
||||
await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id));
|
||||
const [rule] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id,
|
||||
selectorType: "application", applicationId: null, effect: "exclude" }).returning();
|
||||
const service = toolAccessService(db);
|
||||
expect((await compare(f)).assignments.allowedTools).toEqual([]);
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
await db.update(toolProfileEntries).set({ selectorType: "tool_name", toolName }).where(eq(toolProfileEntries.id, rule.id));
|
||||
expect((await compare(f)).assignments.allowedTools).toEqual([]);
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
await db.update(toolProfileEntries).set({ toolName: `${toolName}different` }).where(eq(toolProfileEntries.id, rule.id));
|
||||
expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.read.id]);
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([f.connection.id]);
|
||||
});
|
||||
|
||||
it.each(["application", "connection", "catalog_entry", "tool_name", "risk_level"] as const)("denies the connection when a %s exclusion blocks its only active tool", async selectorType => {
|
||||
const f = await fixture();
|
||||
await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.write.id));
|
||||
await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id));
|
||||
const [rule] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id, selectorType, effect: "exclude",
|
||||
...(selectorType === "application" ? { applicationId: f.application.id } : {}),
|
||||
...(selectorType === "connection" ? { connectionId: f.connection.id } : {}),
|
||||
...(selectorType === "catalog_entry" ? { catalogEntryId: f.read.id } : {}),
|
||||
...(selectorType === "tool_name" ? { toolName: f.read.toolName } : {}),
|
||||
...(selectorType === "risk_level" ? { riskLevel: "read" as const } : {}),
|
||||
}).returning();
|
||||
const service = toolAccessService(db);
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
expect((await compare(f)).assignments.allowedTools).toEqual([]);
|
||||
await db.delete(toolProfileEntries).where(eq(toolProfileEntries.id, rule.id));
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([f.connection.id]);
|
||||
});
|
||||
|
||||
it("filters rules against the catalog rows being evaluated when the catalog changes during the read", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id));
|
||||
await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.write.id));
|
||||
await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id,
|
||||
selectorType: "tool_name", toolName: f.read.toolName, effect: "exclude" });
|
||||
let changed = false;
|
||||
const changingDb = new Proxy(db, { get(target, key) {
|
||||
if (key !== "select") return Reflect.get(target, key);
|
||||
return (...fields: Parameters<Db["select"]>) => {
|
||||
const builder = target.select(...fields), from = builder.from.bind(builder);
|
||||
builder.from = ((...args: Parameters<typeof from>) => {
|
||||
const selection = from(...args);
|
||||
if (args[0] === toolCatalogEntries) {
|
||||
const execute = selection.execute.bind(selection);
|
||||
selection.execute = async () => {
|
||||
const rows = await execute();
|
||||
if (!changed) {
|
||||
changed = true;
|
||||
await db.update(toolCatalogEntries).set({ toolName: "changed-after-read" }).where(eq(toolCatalogEntries.id, f.read.id));
|
||||
}
|
||||
return rows;
|
||||
};
|
||||
}
|
||||
return selection;
|
||||
}) as typeof builder.from;
|
||||
return builder;
|
||||
};
|
||||
} });
|
||||
const service = toolAccessService(changingDb);
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
expect(changed).toBe(true);
|
||||
expect((await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id])).allowedConnectionIds).toEqual([f.connection.id]);
|
||||
});
|
||||
|
||||
it("keeps other companies' catalog rows out of scoped assignments", async () => {
|
||||
const f = await fixture();
|
||||
const other = await fixture();
|
||||
await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id));
|
||||
expect((await toolAccessService(db).getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [other.connection.id])).allowedConnectionIds).toEqual([]);
|
||||
await expect(toolAccessService(db).getConnectionAssignmentsForAgent(f.company.id, other.agent.id, [f.connection.id])).rejects.toThrow("same company");
|
||||
});
|
||||
|
||||
});
|
||||
@@ -95,6 +95,7 @@ export interface Config {
|
||||
nativeHarnessBackupExpirationEnabled: boolean;
|
||||
nativeHarnessBackupExpirationDryRun: boolean;
|
||||
nativeHarnessBackupExpirationDays: number;
|
||||
nativeRunnerEvidenceRetentionDays: number;
|
||||
nativeWorkspaceSyncRetentionEnabled: boolean;
|
||||
nativeWorkspaceSyncRetentionDryRun: boolean;
|
||||
nativeWorkspaceSyncRetentionDays: number;
|
||||
@@ -351,6 +352,7 @@ export function loadConfig(): Config {
|
||||
const skillVersionRetentionKeepNewest = nonNegativeIntegerEnv("PAPERCLIP_SKILL_VERSION_RETENTION_KEEP", 0);
|
||||
const skillVersionRetentionGraceMinutes = nonNegativeIntegerEnv("PAPERCLIP_SKILL_VERSION_RETENTION_GRACE_MINUTES", 60);
|
||||
const nativeHarnessBackupExpirationDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DAYS", 7));
|
||||
const nativeRunnerEvidenceRetentionDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", 7));
|
||||
const nativeWorkspaceSyncRetentionDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_NATIVE_WORKSPACE_SYNC_RETENTION_DAYS", 7));
|
||||
const gitRestorePointRetentionDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_GIT_RESTORE_POINT_RETENTION_DAYS", 90));
|
||||
const gitRestorePointTerminalDelayDays = nonNegativeIntegerEnv("PAPERCLIP_GIT_RESTORE_POINT_TERMINAL_DELAY_DAYS", 3);
|
||||
@@ -417,6 +419,7 @@ export function loadConfig(): Config {
|
||||
nativeHarnessBackupExpirationEnabled: process.env.PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_ENABLED !== "false",
|
||||
nativeHarnessBackupExpirationDryRun: process.env.PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DRY_RUN === "true",
|
||||
nativeHarnessBackupExpirationDays,
|
||||
nativeRunnerEvidenceRetentionDays,
|
||||
nativeWorkspaceSyncRetentionEnabled: process.env.PAPERCLIP_NATIVE_WORKSPACE_SYNC_RETENTION_ENABLED !== "false",
|
||||
nativeWorkspaceSyncRetentionDryRun: process.env.PAPERCLIP_NATIVE_WORKSPACE_SYNC_RETENTION_DRY_RUN === "true",
|
||||
nativeWorkspaceSyncRetentionDays,
|
||||
|
||||
+26
-5
@@ -13,6 +13,7 @@ import {
|
||||
NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS,
|
||||
nativeHarnessBackupExpirationService,
|
||||
} from "./services/native-runtime/native-harness-backup-expiration.js";
|
||||
import { onNativeRunnerEvidence } from "./services/native-runtime/native-runner-evidence-signal.js";
|
||||
import { gitRestorePointRetentionService, registerGitRestorePointWork } from "./services/git-restore-point-ledger.js";
|
||||
import {
|
||||
nativeWorkspaceSyncRetentionService,
|
||||
@@ -38,6 +39,7 @@ import { chatCompletionDeliveryService } from "./services/chat-completion-delive
|
||||
// instrumentationReady before opening DB connections or constructing the
|
||||
// HTTP server, so trace coverage does not depend on incidental timing.
|
||||
import { instrumentationReady, shutdownInstrumentation } from "./instrumentation.js";
|
||||
import { createTrackedSingleFlight } from "./lib/single-flight.js";
|
||||
import { sentryReady, shutdownSentry, captureException } from "./sentry.js";
|
||||
import { setSentryRunLinkResolver } from "./sentry-links.js";
|
||||
import { createSentryRunLinkResolver } from "./services/sentry-run-links.js";
|
||||
@@ -135,6 +137,7 @@ import {
|
||||
} from "./services/device-login-reaper.js";
|
||||
import { createProductionSetupTokenReaper } from "./services/setup-token-reaper.js";
|
||||
import { localAiLoginService } from "./services/local-ai-login.js";
|
||||
import { registerGitHubBotCloudSetupPoller } from "./services/chat-github-cloud-ingress.js";
|
||||
import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js";
|
||||
import {
|
||||
parseAdapterRegistryEnv,
|
||||
@@ -1209,6 +1212,16 @@ async function startServerWithDatabaseTeardown(
|
||||
});
|
||||
heartbeatSchedulerInFlight.add(tracked);
|
||||
};
|
||||
const runPeriodicRecovery = createTrackedSingleFlight({
|
||||
trackWork: trackHeartbeatSchedulerWork,
|
||||
onSkipped: (inFlightForMs) => {
|
||||
if (inFlightForMs >= 3 * config.heartbeatSchedulerIntervalMs) {
|
||||
logger.error({ inFlightForMs }, "periodic heartbeat recovery remains in flight; duplicate pass suppressed");
|
||||
} else {
|
||||
logger.warn({ inFlightForMs }, "skipped periodic heartbeat recovery tick; previous chain still in flight");
|
||||
}
|
||||
},
|
||||
});
|
||||
const waitForHeartbeatSchedulerIdle = async () => {
|
||||
while (heartbeatSchedulerInFlight.size > 0) {
|
||||
await Promise.allSettled([...heartbeatSchedulerInFlight]);
|
||||
@@ -1257,8 +1270,10 @@ async function startServerWithDatabaseTeardown(
|
||||
const nativeHarnessBackupExpiration = nativeHarnessBackupExpirationService(db as any, {
|
||||
dryRun: config.nativeHarnessBackupExpirationDryRun,
|
||||
closedForDays: config.nativeHarnessBackupExpirationDays,
|
||||
evidenceRetentionDays: config.nativeRunnerEvidenceRetentionDays,
|
||||
shouldContinue: () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(),
|
||||
});
|
||||
const unsubscribeNativeRunnerEvidence = onNativeRunnerEvidence(nativeHarnessBackupExpiration.noteEvidence);
|
||||
let nativeHarnessBackupExpirationInFlight = false;
|
||||
let nativeHarnessBackupExpirationContinuation: ReturnType<typeof setTimeout> | undefined;
|
||||
const sweepNativeHarnessBackups = () => {
|
||||
@@ -1266,7 +1281,7 @@ async function startServerWithDatabaseTeardown(
|
||||
nativeHarnessBackupExpirationInFlight = true;
|
||||
trackHeartbeatSchedulerWork(nativeHarnessBackupExpiration.tick()
|
||||
.then((result) => {
|
||||
if (result.candidates > 0 || result.failed > 0) {
|
||||
if (result.candidates > 0 || result.evidenceCandidates > 0 || result.failed > 0) {
|
||||
logger.info({ ...result, dryRun: config.nativeHarnessBackupExpirationDryRun }, "native harness backup expiration");
|
||||
}
|
||||
if (result.truncated && !heartbeatSchedulerStopped) {
|
||||
@@ -1408,9 +1423,9 @@ async function startServerWithDatabaseTeardown(
|
||||
?? process.env.PAPERCLIP_TOOL_RUNTIME_TRUSTED_HOST
|
||||
?? null,
|
||||
});
|
||||
const scheduleGitHubConnectionEventPoll = () => {
|
||||
const scheduleGitHubConnectionEventPoll = (forSetup = false) => {
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
trackHeartbeatSchedulerWork(githubConnectionEvents.pollOnce()
|
||||
const work = githubConnectionEvents.pollOnce({ forSetup })
|
||||
.then((result) => {
|
||||
if (result.leased > 0 || result.failed > 0) {
|
||||
logger.info(result, "GitHub connection event poll completed");
|
||||
@@ -1418,8 +1433,13 @@ async function startServerWithDatabaseTeardown(
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "GitHub connection event poll failed");
|
||||
}));
|
||||
});
|
||||
trackHeartbeatSchedulerWork(work);
|
||||
return work;
|
||||
};
|
||||
server.on("close", registerGitHubBotCloudSetupPoller(db, async () => {
|
||||
await scheduleGitHubConnectionEventPoll(true);
|
||||
}));
|
||||
const scheduleGitHubConnectionContinuitySweep = () => {
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
trackHeartbeatSchedulerWork(tools.sweepGitHubConnectionContinuity()
|
||||
@@ -1887,7 +1907,7 @@ async function startServerWithDatabaseTeardown(
|
||||
if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {
|
||||
// Periodically reap orphaned runs (5-min staleness threshold) and make sure
|
||||
// persisted queued work is still being driven forward.
|
||||
trackHeartbeatSchedulerWork(heartbeat
|
||||
runPeriodicRecovery(() => heartbeat
|
||||
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000 })
|
||||
.then(() => heartbeat.promoteDueScheduledRetries())
|
||||
.then(async (promotion) => {
|
||||
@@ -2078,6 +2098,7 @@ async function startServerWithDatabaseTeardown(
|
||||
clearInterval(nativeHarnessBackupExpirationInterval);
|
||||
clearTimeout(nativeHarnessBackupExpirationStartup);
|
||||
clearTimeout(nativeHarnessBackupExpirationContinuation);
|
||||
unsubscribeNativeRunnerEvidence();
|
||||
await skillVersionRetention.dispose();
|
||||
await skillSourceSync.stop();
|
||||
unsubscribeChatCompletions();
|
||||
|
||||
@@ -98,6 +98,8 @@ export interface CreateAgentOptions {
|
||||
claudeLogin?: ClaudeLoginContext;
|
||||
}
|
||||
|
||||
type AgentEligibilityRow = Pick<typeof agents.$inferSelect, "id" | "companyId" | "name" | "status" | "reportsTo">;
|
||||
|
||||
interface AgentShortnameRow {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -340,7 +342,7 @@ export function agentRecordQueries(db: Db) {
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeAgentRows(rows: (typeof agents.$inferSelect)[], allCompanyRows = rows) {
|
||||
function normalizeAgentRows(rows: (typeof agents.$inferSelect)[], allCompanyRows: AgentEligibilityRow[] = rows) {
|
||||
const eligibilityAgents = allCompanyRows.map(toEligibilityAgent);
|
||||
return rows.map((row) => {
|
||||
const base = normalizeAgentBaseRow(row);
|
||||
@@ -357,7 +359,7 @@ export function agentRecordQueries(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeAgentRow(row: typeof agents.$inferSelect, allCompanyRows?: (typeof agents.$inferSelect)[]) {
|
||||
function normalizeAgentRow(row: typeof agents.$inferSelect, allCompanyRows?: AgentEligibilityRow[]) {
|
||||
return normalizeAgentRows([row], allCompanyRows)[0]!;
|
||||
}
|
||||
|
||||
@@ -365,6 +367,11 @@ export function agentRecordQueries(db: Db) {
|
||||
return db.select().from(agents).where(eq(agents.companyId, companyId));
|
||||
}
|
||||
|
||||
async function listCompanyAgentEligibilityRows(companyId: string): Promise<AgentEligibilityRow[]> {
|
||||
return db.select({ id: agents.id, companyId: agents.companyId, name: agents.name,
|
||||
status: agents.status, reportsTo: agents.reportsTo }).from(agents).where(eq(agents.companyId, companyId));
|
||||
}
|
||||
|
||||
async function getMonthlySpendByAgentIds(companyId: string, agentIds: string[]) {
|
||||
if (agentIds.length === 0) return new Map<string, number>();
|
||||
const { start, end } = currentUtcMonthWindow();
|
||||
@@ -405,7 +412,7 @@ export function agentRecordQueries(db: Db) {
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!row) return null;
|
||||
const [companyRows, hydrated] = await Promise.all([
|
||||
listCompanyAgentRows(row.companyId),
|
||||
listCompanyAgentEligibilityRows(row.companyId),
|
||||
hydrateAgentSpend([row]).then((rows) => rows[0]!),
|
||||
]);
|
||||
return normalizeAgentRow(hydrated, companyRows);
|
||||
@@ -472,6 +479,6 @@ export function agentRecordQueries(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
return { normalizeAgentRows, normalizeAgentRow, listCompanyAgentRows, hydrateAgentSpend,
|
||||
return { normalizeAgentRows, normalizeAgentRow, listCompanyAgentRows, listCompanyAgentEligibilityRows, hydrateAgentSpend,
|
||||
getById, ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed };
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createSingleFlight, createTrackedSingleFlight } from "./single-flight.js";
|
||||
import { beginIdleTrackedWork, idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void;
|
||||
let reject!: (err: unknown) => void;
|
||||
const promise = new Promise<void>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
describe("createSingleFlight", () => {
|
||||
it("runs work when idle and reports busy until it settles", async () => {
|
||||
const gate = createSingleFlight();
|
||||
const gap = deferred();
|
||||
|
||||
const first = gate.run(() => gap.promise);
|
||||
expect(first.started).toBe(true);
|
||||
expect(gate.busy).toBe(true);
|
||||
|
||||
gap.resolve();
|
||||
await (first as { settled: Promise<void> }).settled;
|
||||
expect(gate.busy).toBe(false);
|
||||
});
|
||||
|
||||
it("skips a tick while the previous invocation is still in flight", async () => {
|
||||
const clock = { nowMs: 1_000 };
|
||||
const gate = createSingleFlight(() => clock.nowMs);
|
||||
const gap = deferred();
|
||||
const work = vi.fn(() => gap.promise);
|
||||
|
||||
const first = gate.run(work);
|
||||
clock.nowMs = 31_000;
|
||||
const second = gate.run(work);
|
||||
|
||||
expect(second).toEqual({ started: false, inFlightForMs: 30_000 });
|
||||
expect(work).toHaveBeenCalledTimes(1);
|
||||
|
||||
gap.resolve();
|
||||
await (first as { settled: Promise<void> }).settled;
|
||||
});
|
||||
|
||||
it("admits new work after the previous invocation resolves", async () => {
|
||||
const gate = createSingleFlight();
|
||||
const gap = deferred();
|
||||
|
||||
const first = gate.run(() => gap.promise);
|
||||
gap.resolve();
|
||||
await (first as { settled: Promise<void> }).settled;
|
||||
|
||||
const second = gate.run(async () => {});
|
||||
expect(second.started).toBe(true);
|
||||
await (second as { settled: Promise<void> }).settled;
|
||||
});
|
||||
|
||||
it("releases the gate when work rejects and never rejects the settled promise", async () => {
|
||||
const gate = createSingleFlight();
|
||||
const gap = deferred();
|
||||
|
||||
const first = gate.run(() => gap.promise);
|
||||
gap.reject(new Error("chain failed"));
|
||||
await expect((first as { settled: Promise<void> }).settled).resolves.toBeUndefined();
|
||||
expect(gate.busy).toBe(false);
|
||||
|
||||
const second = gate.run(async () => {});
|
||||
expect(second.started).toBe(true);
|
||||
await (second as { settled: Promise<void> }).settled;
|
||||
});
|
||||
|
||||
it("propagates a synchronous throw and leaves the gate free", async () => {
|
||||
const gate = createSingleFlight();
|
||||
|
||||
expect(() =>
|
||||
gate.run(() => {
|
||||
throw new Error("sync failure");
|
||||
}),
|
||||
).toThrow("sync failure");
|
||||
expect(gate.busy).toBe(false);
|
||||
|
||||
const second = gate.run(async () => {});
|
||||
expect(second.started).toBe(true);
|
||||
await (second as { settled: Promise<void> }).settled;
|
||||
});
|
||||
|
||||
it("keeps the owned pass active after a caller stops waiting", async () => {
|
||||
const gate = createSingleFlight(), work = deferred(), observer = deferred();
|
||||
const first = gate.run(() => work.promise);
|
||||
if (!first.started) throw new Error("initial pass was not admitted");
|
||||
const stopped = Promise.race([first.settled, observer.promise]);
|
||||
observer.reject(new Error("observer stopped"));
|
||||
await expect(stopped).rejects.toThrow("observer stopped");
|
||||
const duplicate = vi.fn(async () => undefined);
|
||||
expect(gate.run(duplicate).started).toBe(false);
|
||||
expect(duplicate).not.toHaveBeenCalled();
|
||||
expect(gate.busy).toBe(true);
|
||||
work.resolve();
|
||||
await first.settled;
|
||||
expect(gate.busy).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps cleanup joined after the main operation completes", async () => {
|
||||
const gate = createSingleFlight(), operation = deferred(), cleanup = deferred();
|
||||
const first = gate.run(() => operation.promise.finally(() => cleanup.promise));
|
||||
if (!first.started) throw new Error("initial pass was not admitted");
|
||||
operation.resolve();
|
||||
await Promise.resolve();
|
||||
expect(gate.busy).toBe(true);
|
||||
const duplicate = vi.fn(async () => undefined);
|
||||
expect(gate.run(duplicate).started).toBe(false);
|
||||
expect(duplicate).not.toHaveBeenCalled();
|
||||
cleanup.resolve();
|
||||
await first.settled;
|
||||
expect(gate.busy).toBe(false);
|
||||
const next = gate.run(duplicate);
|
||||
expect(next.started).toBe(true);
|
||||
if (next.started) await next.settled;
|
||||
expect(duplicate).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("tracks one periodic pass through scheduler shutdown and idle sleep", async () => {
|
||||
vi.useFakeTimers();
|
||||
const before = idleWorkSnapshot().active;
|
||||
const operation = deferred(), cleanup = deferred();
|
||||
const tracked: Promise<void>[] = [];
|
||||
const trackWork = vi.fn((work: Promise<void>) => {
|
||||
const finish = beginIdleTrackedWork();
|
||||
tracked.push(work.finally(finish));
|
||||
});
|
||||
const onSkipped = vi.fn();
|
||||
const tick = createTrackedSingleFlight({ trackWork, onSkipped });
|
||||
const heartbeat = vi.fn(() => operation.promise.finally(() => cleanup.promise));
|
||||
const timer = setInterval(() => tick(heartbeat), 30_000);
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(heartbeat).toHaveBeenCalledOnce();
|
||||
expect(trackWork).toHaveBeenCalledOnce();
|
||||
expect(onSkipped).toHaveBeenCalledExactlyOnceWith(30_000);
|
||||
clearInterval(timer);
|
||||
let shutdownComplete = false;
|
||||
const shutdown = Promise.all(tracked).then(() => { shutdownComplete = true; });
|
||||
await Promise.resolve();
|
||||
expect(shutdownComplete).toBe(false);
|
||||
expect(idleWorkSnapshot().active).toBe(before + 1);
|
||||
operation.resolve();
|
||||
await Promise.resolve();
|
||||
expect(shutdownComplete).toBe(false);
|
||||
expect(idleWorkSnapshot().active).toBe(before + 1);
|
||||
cleanup.resolve();
|
||||
await shutdown;
|
||||
expect(shutdownComplete).toBe(true);
|
||||
expect(idleWorkSnapshot().active).toBe(before);
|
||||
} finally {
|
||||
clearInterval(timer);
|
||||
operation.resolve();
|
||||
cleanup.resolve();
|
||||
await Promise.all(tracked);
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
export interface SingleFlightSkip {
|
||||
started: false;
|
||||
inFlightForMs: number;
|
||||
}
|
||||
|
||||
export interface SingleFlightStart {
|
||||
started: true;
|
||||
settled: Promise<void>;
|
||||
}
|
||||
|
||||
export type SingleFlightResult = SingleFlightSkip | SingleFlightStart;
|
||||
|
||||
export function createSingleFlight(now: () => number = Date.now) {
|
||||
let inFlight: Promise<void> | null = null;
|
||||
let startedAtMs = 0;
|
||||
|
||||
return {
|
||||
get busy(): boolean {
|
||||
return inFlight !== null;
|
||||
},
|
||||
run(work: () => Promise<unknown>): SingleFlightResult {
|
||||
if (inFlight) {
|
||||
return { started: false, inFlightForMs: now() - startedAtMs };
|
||||
}
|
||||
startedAtMs = now();
|
||||
const settled = Promise.resolve(work())
|
||||
.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
)
|
||||
.finally(() => {
|
||||
inFlight = null;
|
||||
});
|
||||
inFlight = settled;
|
||||
return { started: true, settled };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createTrackedSingleFlight(options: {
|
||||
trackWork: (work: Promise<void>) => void;
|
||||
onSkipped: (inFlightForMs: number) => void;
|
||||
now?: () => number;
|
||||
}) {
|
||||
const gate = createSingleFlight(options.now);
|
||||
return (work: () => Promise<unknown>) => {
|
||||
const tick = gate.run(work);
|
||||
if (tick.started) options.trackWork(tick.settled);
|
||||
else options.onSkipped(tick.inFlightForMs);
|
||||
};
|
||||
}
|
||||
@@ -429,7 +429,8 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa
|
||||
return;
|
||||
}
|
||||
if (identityRun?.activeIdentityContextId && identityRun.status === "running") {
|
||||
const captured = await captureRunIdentity(db, { companyId: claims.company_id, agentId: claims.sub, runId: claims.run_id });
|
||||
const captured = await captureRunIdentity(db, { companyId: claims.company_id, agentId: claims.sub, runId: claims.run_id },
|
||||
{ readScope: "request_authentication" });
|
||||
identityRun.activeIdentityContextId = captured.context?.id ?? null;
|
||||
identityRun.responsibleUserId = captured.context?.responsibleUserId ?? null;
|
||||
}
|
||||
|
||||
@@ -206,7 +206,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
return { recorded, rows };
|
||||
}
|
||||
|
||||
it("keeps in-flight work and every recovery receipt while omitting output from release reads", async () => {
|
||||
it.each(["legacy", "native"])("keeps %s in-flight work and every recovery receipt while omitting output from release reads", async runtimeMode => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
@@ -218,7 +218,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId, companyId, issueId },
|
||||
executionCancellation: { state: "acknowledged" }, executionRecovery: { kind: "interrupted", providerStopped: true },
|
||||
configurationIncomplete: { reason: "workspace_unavailable" }, unknownFutureReceipt: { retained: true } };
|
||||
await db.update(heartbeatRuns).set({ resultJson: { ...evidence, stdout: payload.repeat(3), stderr: payload } }).where(eq(heartbeatRuns.id, runId));
|
||||
await db.update(heartbeatRuns).set({ runtimeMode,
|
||||
...(runtimeMode === "native" ? { runnerProfileJson: { savedInput: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
|
||||
contextSnapshot: { issueId, paperclipWake: { savedTask: payload }, paperclipTaskMarkdown: payload } } : {}),
|
||||
resultJson: { ...evidence, stdout: payload.repeat(3), stderr: payload } }).where(eq(heartbeatRuns.id, runId));
|
||||
await db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, issueId));
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
@@ -230,7 +233,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
);
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toEqual([{ ...run, resultJson: evidence }]);
|
||||
expect(rows[0]).toEqual([{ ...run, resultJson: evidence,
|
||||
...(runtimeMode === "native" ? { runnerProfileJson: null, stdoutExcerpt: null, stderrExcerpt: null,
|
||||
contextSnapshot: { issueId } } : {}) }]);
|
||||
expect(result.run.contextSnapshot).toEqual({ issueId });
|
||||
expect(Buffer.byteLength(JSON.stringify(rows))).toBeLessThan(8 * 1024);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]).toEqual(run);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toEqual(issue);
|
||||
|
||||
@@ -5,7 +5,8 @@ import { isCompletedOnboardingHandoffWake } from "../../../services/chat-complet
|
||||
import { instanceSettingsService } from "../../../services/instance-settings.js";
|
||||
import { currentConversationCommentCondition } from "../../../services/agent-conversations.js";
|
||||
import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js";
|
||||
import { and, asc, eq, getTableColumns, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
|
||||
import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
|
||||
import { readReleaseRun, type ReleaseRun } from "./release-run-read.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { extractIssueReferenceIdentifiers, isUuidLike } from "@paperclipai/shared";
|
||||
import {
|
||||
@@ -74,14 +75,6 @@ const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_r
|
||||
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueRow = typeof issues.$inferSelect;
|
||||
|
||||
const releaseRunFields = {
|
||||
...getTableColumns(heartbeatRuns),
|
||||
resultJson: sql<Record<string, unknown> | null>`case
|
||||
when jsonb_typeof(${heartbeatRuns.resultJson}) = 'object'
|
||||
then ${heartbeatRuns.resultJson} - 'stdout' - 'stderr'
|
||||
else ${heartbeatRuns.resultJson} end`,
|
||||
};
|
||||
|
||||
function normalizeAgentNameKey(value: string | null | undefined): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
@@ -94,7 +87,7 @@ function toRequestedByActorType(value: string | null): "user" | "agent" | "syste
|
||||
return value === "user" || value === "agent" || value === "system" ? value : null;
|
||||
}
|
||||
|
||||
function toRunSnapshot(row: HeartbeatRunRow): RunSnapshot {
|
||||
function toRunSnapshot(row: ReleaseRun): RunSnapshot {
|
||||
const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete);
|
||||
return {
|
||||
id: row.id,
|
||||
@@ -206,7 +199,7 @@ function buildHost(_tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueHost {
|
||||
};
|
||||
}
|
||||
|
||||
function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, run: HeartbeatRunRow): WakeQueueTransaction {
|
||||
function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, run: ReleaseRun): WakeQueueTransaction {
|
||||
const treeControlSvc = issueTreeControlService(tx);
|
||||
const issuesSvc = issueService(tx);
|
||||
const interruptQueueId = run.runtimeMode !== "native" && run.status === "cancelled"
|
||||
@@ -750,7 +743,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
|
||||
};
|
||||
}
|
||||
|
||||
async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: HeartbeatRunRow, issue: IssueRow, now: Date): Promise<boolean> {
|
||||
async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: ReleaseRun, issue: IssueRow, now: Date): Promise<boolean> {
|
||||
const applies =
|
||||
run.runtimeMode === "native" &&
|
||||
["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) &&
|
||||
@@ -1058,11 +1051,8 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
async withIssueExecutionLock(input, fn): Promise<ReleaseTransactionResult & { run: RunSnapshot }> {
|
||||
return db.transaction(async (rawTx) => {
|
||||
const tx = rawTx as unknown as Db;
|
||||
const run = await tx
|
||||
.select(releaseRunFields)
|
||||
.from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const evidence = await readReleaseRun(tx, input);
|
||||
const run = evidence.run;
|
||||
if (!run) {
|
||||
throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`);
|
||||
}
|
||||
@@ -1164,7 +1154,8 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
issueStatus: issueRow?.status ?? "",
|
||||
hasAssigneeUser: Boolean(issueRow?.assigneeUserId),
|
||||
assigneeAgentMatchesRunAgent: issueRow?.assigneeAgentId === run.agentId,
|
||||
legacyExecutionNeedsReconciliation: await legacyExecutionNeedsReconciliationWithEvidence(tx as unknown as Db, run),
|
||||
legacyExecutionNeedsReconciliation: evidence.kind === "complete" &&
|
||||
await legacyExecutionNeedsReconciliationWithEvidence(tx as unknown as Db, evidence.run),
|
||||
// An operator stop never promotes old queued work by itself. The
|
||||
// next explicit wake adopts those messages atomically when it
|
||||
// queues a run.
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { readReleaseRun } from "./release-run-read.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("wake release run reads", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let companyId: string;
|
||||
let agentId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("wake-release-read-");
|
||||
db = createDb(database.connectionString);
|
||||
const [company] = await db.insert(companies).values({ name: "Wake release fixture", issuePrefix: "WRR" }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Release agent", adapterType: "process" }).returning();
|
||||
companyId = company.id;
|
||||
agentId = agent.id;
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
it("omits native bulk at the driver boundary and preserves the complete legacy fallback", async () => {
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const context = { issueId: randomUUID(), opaqueFutureContext: { savedInput: "keep this input" }, explicitUserContinuation: true };
|
||||
const evidence = { nativeCancellation: { schema: "paperclip.native-cancellation.v1", acknowledged: true }, unknownFutureReceipt: [true, null, 42] };
|
||||
const [before] = await db.insert(heartbeatRuns).values({
|
||||
companyId, agentId, runtimeMode: "native", status: "running", contextSnapshot: context,
|
||||
runnerProfileJson: { nativeExecutionInput: { savedInput: bulk } }, stdoutExcerpt: bulk, stderrExcerpt: bulk,
|
||||
resultJson: { ...evidence, stdout: bulk, stderr: bulk },
|
||||
}).returning();
|
||||
const scope = { companyId, runId: before.id };
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const native = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, scope));
|
||||
expect(native.kind).toBe("native");
|
||||
expect(native.run?.contextSnapshot).toEqual(context);
|
||||
expect(native.run?.resultJson).toEqual(evidence);
|
||||
expect(native.run).not.toHaveProperty("runnerProfileJson");
|
||||
expect(native.run).not.toHaveProperty("stdoutExcerpt");
|
||||
expect(native.run).not.toHaveProperty("stderrExcerpt");
|
||||
} finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(1);
|
||||
expect(summary.responseValueBytes).toBeLessThan(8192);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, before.id)))[0]).toEqual(before);
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "legacy" }).where(eq(heartbeatRuns.id, before.id));
|
||||
const legacy = await readReleaseRun(db, scope);
|
||||
expect(legacy.kind).toBe("complete");
|
||||
expect(legacy.run).toEqual({ ...before, runtimeMode: "legacy", resultJson: evidence });
|
||||
expect((await readReleaseRun(db, { ...scope, companyId: randomUUID() })).run).toBeUndefined();
|
||||
expect((await readReleaseRun(db, { companyId, runId: randomUUID() })).run).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([null, ["stdout", { stderr: "evidence" }], "scalar", 42].map(result => ({ result })))(
|
||||
"preserves non-object native result evidence $result", async ({ result }) => {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, runtimeMode: "native",
|
||||
resultJson: result as Record<string, unknown> | null }).returning();
|
||||
expect((await readReleaseRun(db, { companyId, runId: run.id })).run?.resultJson).toEqual(result);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps launch text in storage while releasing native runs with their control context", async () => {
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const context = {
|
||||
issueId: randomUUID(), taskId: randomUUID(), taskKey: "saved-task",
|
||||
retryReason: "execution_review_participant_recovery", wakeReason: "issue_commented",
|
||||
explicitUserContinuation: true,
|
||||
interactionId: randomUUID(), interactionKind: "review", interactionStatus: "resolved",
|
||||
wakeCommentIds: [randomUUID()], queuedCommentSourceWakeIds: [randomUUID()],
|
||||
chatConversationId: randomUUID(), executionPolicy: { requireApproval: true },
|
||||
opaqueFutureContext: { retained: [null, false, "input"] },
|
||||
};
|
||||
const launch = {
|
||||
executionContinuation: { instruction: bulk, state: "saved" },
|
||||
paperclipWake: { savedTask: bulk }, paperclipWakeComment: bulk, paperclipTaskMarkdown: bulk,
|
||||
paperclipTaskMarkdownCompact: bulk, paperclipTaskMarkdownAssignment: bulk,
|
||||
paperclipTaskMarkdownAssignmentCompact: bulk, paperclipWorkspace: { files: bulk },
|
||||
paperclipEnvironment: { input: bulk }, paperclipContinuationSummary: bulk,
|
||||
};
|
||||
const evidence = { nativeCancellation: { schema: "paperclip.native-cancellation.v1", dispatched: true },
|
||||
executionCancellation: { state: "acknowledged" }, workspaceRestoreRecovery: { leaseIds: [randomUUID()] },
|
||||
configurationIncomplete: { reason: "secret_binding_missing" }, unknownFutureReceipt: { retained: true } };
|
||||
const [before] = await db.insert(heartbeatRuns).values({ companyId, agentId, runtimeMode: "native",
|
||||
contextSnapshot: { ...context, ...launch }, resultJson: evidence }).returning();
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let selected: Awaited<ReturnType<typeof readReleaseRun>>;
|
||||
try { selected = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, { companyId, runId: before.id })); }
|
||||
finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBe(1);
|
||||
expect(summary.responseValueBytes).toBeLessThan(8192);
|
||||
expect(selected!.run?.contextSnapshot).toEqual(context);
|
||||
expect(selected!.run?.resultJson).toEqual(evidence);
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, before.id)))[0]).toEqual(before);
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "legacy" }).where(eq(heartbeatRuns.id, before.id));
|
||||
expect((await readReleaseRun(db, { companyId, runId: before.id })).run?.contextSnapshot).toEqual(before.contextSnapshot);
|
||||
});
|
||||
|
||||
it.each([null, ["paperclipWake", { savedInput: "keep" }], "scalar", 42].map(context => ({ context })))(
|
||||
"preserves non-object native context $context", async ({ context }) => {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, runtimeMode: "native",
|
||||
contextSnapshot: context as Record<string, unknown> | null }).returning();
|
||||
expect((await readReleaseRun(db, { companyId, runId: run.id })).run?.contextSnapshot).toEqual(context);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, eq, getTableColumns, sql } from "drizzle-orm";
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
export type ReleaseRun = Omit<Run, "runnerProfileJson" | "stdoutExcerpt" | "stderrExcerpt">;
|
||||
|
||||
export async function readReleaseRun(db: Db, scope: { companyId: string; runId: string }) {
|
||||
const [selected] = await db.select({
|
||||
...getTableColumns(heartbeatRuns),
|
||||
contextSnapshot: sql<Run["contextSnapshot"]>`case when ${heartbeatRuns.runtimeMode} = 'native'
|
||||
and jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object'
|
||||
then ${heartbeatRuns.contextSnapshot} - array['executionContinuation', 'paperclipWake', 'paperclipWakeComment',
|
||||
'paperclipTaskMarkdown', 'paperclipTaskMarkdownCompact', 'paperclipTaskMarkdownAssignment',
|
||||
'paperclipTaskMarkdownAssignmentCompact', 'paperclipWorkspace', 'paperclipEnvironment',
|
||||
'paperclipContinuationSummary']
|
||||
else ${heartbeatRuns.contextSnapshot} end`,
|
||||
runnerProfileJson: sql<Run["runnerProfileJson"]>`case when ${heartbeatRuns.runtimeMode} = 'native'
|
||||
then null else ${heartbeatRuns.runnerProfileJson} end`,
|
||||
stdoutExcerpt: sql<Run["stdoutExcerpt"]>`case when ${heartbeatRuns.runtimeMode} = 'native'
|
||||
then null else ${heartbeatRuns.stdoutExcerpt} end`,
|
||||
stderrExcerpt: sql<Run["stderrExcerpt"]>`case when ${heartbeatRuns.runtimeMode} = 'native'
|
||||
then null else ${heartbeatRuns.stderrExcerpt} end`,
|
||||
resultJson: sql<Run["resultJson"]>`case when jsonb_typeof(${heartbeatRuns.resultJson}) = 'object'
|
||||
then ${heartbeatRuns.resultJson} - 'stdout' - 'stderr' else ${heartbeatRuns.resultJson} end`,
|
||||
}).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, scope.runId), eq(heartbeatRuns.companyId, scope.companyId)));
|
||||
if (!selected || selected.runtimeMode !== "native") return { kind: "complete" as const, run: selected };
|
||||
const { runnerProfileJson: _profile, stdoutExcerpt: _stdout, stderrExcerpt: _stderr, ...run } = selected;
|
||||
return { kind: "native" as const, run };
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import { createRequire } from "node:module";
|
||||
import type { Duplex } from "node:stream";
|
||||
import { and, eq, isNull } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agentApiKeys, companyMemberships, instanceUserRoles, heartbeatRuns, issues, projects } from "@paperclipai/db";
|
||||
import { agentApiKeys, companyMemberships, instanceUserRoles, heartbeatRuns, projects } from "@paperclipai/db";
|
||||
import type { DeploymentMode, LiveEvent } from "@paperclipai/shared";
|
||||
import type { BetterAuthSessionResult } from "../auth/better-auth.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
@@ -45,7 +45,7 @@ const { WebSocket, WebSocketServer } = require("ws") as {
|
||||
};
|
||||
|
||||
import { authorizationService, type AuthorizationActor } from "../services/authorization.js";
|
||||
import { canActorReadHeartbeatRun } from "../services/heartbeat-run-privacy.js";
|
||||
import { getHeartbeatRunReadDecisions } from "../services/heartbeat-run-privacy.js";
|
||||
import { canActorReadCase } from "../services/case-access.js";
|
||||
|
||||
interface UpgradeContext {
|
||||
@@ -301,12 +301,22 @@ export function setupLiveEventsWebSocketServer(
|
||||
const payload = event.payload;
|
||||
const issueId = typeof payload.issueId === "string" ? payload.issueId
|
||||
: payload.entityType === "issue" && typeof payload.entityId === "string" ? payload.entityId : null;
|
||||
const runId = typeof payload.runId === "string" ? payload.runId
|
||||
: payload.entityType === "heartbeat_run" && typeof payload.entityId === "string" ? payload.entityId : null;
|
||||
const readBindings: Parameters<typeof getHeartbeatRunReadDecisions>[3][number][] = [];
|
||||
if (issueId) {
|
||||
const [issue] = await db.select({ id: issues.id }).from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, event.companyId))).limit(1);
|
||||
if (!issue || !(await access.decide({ actor: context!.actor, action: "issue:read",
|
||||
resource: { type: "issue", companyId: event.companyId, issueId } })).allowed) return null;
|
||||
readBindings.push({ companyId: event.companyId, scopeKind: "issue", issueId });
|
||||
}
|
||||
if (runId) {
|
||||
const [run] = await db.select({
|
||||
companyId: heartbeatRuns.companyId,
|
||||
scopeKind: heartbeatRuns.scopeKind,
|
||||
issueId: heartbeatRuns.issueId,
|
||||
}).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, event.companyId)));
|
||||
if (!run) return null;
|
||||
readBindings.push(run);
|
||||
}
|
||||
if ((await getHeartbeatRunReadDecisions(db, access, context!.actor, readBindings)).some(allowed => !allowed)) return null;
|
||||
if (["company_skill", "company_skill_source"].includes(String(payload.entityType)) && typeof payload.entityId === "string") {
|
||||
try {
|
||||
const skills = skillAccessService(db, context!.actor);
|
||||
@@ -320,16 +330,6 @@ export function setupLiveEventsWebSocketServer(
|
||||
if (!project || !(await access.decide({ actor: context!.actor, action: "project:read",
|
||||
resource: { type: "project", companyId: event.companyId, projectId: project.id } })).allowed) return null;
|
||||
}
|
||||
const runId = typeof payload.runId === "string" ? payload.runId
|
||||
: payload.entityType === "heartbeat_run" && typeof payload.entityId === "string" ? payload.entityId : null;
|
||||
if (runId) {
|
||||
const [run] = await db.select({
|
||||
companyId: heartbeatRuns.companyId,
|
||||
scopeKind: heartbeatRuns.scopeKind,
|
||||
issueId: heartbeatRuns.issueId,
|
||||
}).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, event.companyId)));
|
||||
if (!run || !(await canActorReadHeartbeatRun(db, access, context!.actor, run))) return null;
|
||||
}
|
||||
if (event.type.startsWith("heartbeat.run.")) return runId ? event : null;
|
||||
if (event.type === "agent.session.goal.changed") return issueId ? event : null;
|
||||
// Company-wide invalidations carry no task-derived content. Authorized
|
||||
|
||||
@@ -355,7 +355,15 @@ export function activityRoutes(db: Db) {
|
||||
const issue = await getAccessibleResource(req, res, resolveIssueByRef(rawId), "Issue not found");
|
||||
if (!issue) return;
|
||||
if (!(await assertIssueReadAllowed(req, res, issue))) return;
|
||||
const result = await svc.forIssue(issue.id);
|
||||
if (req.query.view === "comment-inputs") {
|
||||
const parsed = z.array(z.string().uuid()).max(100).safeParse(
|
||||
typeof req.query.commentIds === "string" && req.query.commentIds ? req.query.commentIds.split(",") : [],
|
||||
);
|
||||
if (!parsed.success) throw badRequest("commentIds must contain at most 100 UUIDs");
|
||||
res.json(await svc.inputFactsForIssue(issue.companyId, issue.id, parsed.data));
|
||||
return;
|
||||
}
|
||||
const result = await (req.query.view === "recent" ? svc.forIssue(issue.id, 50) : svc.forIssue(issue.id));
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
@@ -364,8 +372,10 @@ export function activityRoutes(db: Db) {
|
||||
const issue = await getAccessibleResource(req, res, resolveIssueByRef(rawId), "Issue not found");
|
||||
if (!issue) return;
|
||||
if (!(await assertIssueReadAllowed(req, res, issue))) return;
|
||||
const result = await svc.runsForIssue(issue.companyId, issue.id,
|
||||
req.actor.type === "board" ? getActorInfo(req).actorId : null);
|
||||
const viewerUserId = req.actor.type === "board" ? getActorInfo(req).actorId : null;
|
||||
const result = await (req.query.view === "recent"
|
||||
? svc.runsForIssue(issue.companyId, issue.id, viewerUserId, 2)
|
||||
: svc.runsForIssue(issue.companyId, issue.id, viewerUserId));
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import type { Router } from "express";
|
||||
import { agents, type Db } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { addAgentProjectAccessSchema } from "@paperclipai/shared";
|
||||
import { notFound } from "../errors.js";
|
||||
import { validate } from "../middleware/validate.js";
|
||||
import { addAgentProjectAccess, readAgentProjectAccess } from "../services/agent-project-access.js";
|
||||
import { assertBoard, assertCompanyAccess, hasCompanyAccess } from "./authz.js";
|
||||
|
||||
export function registerAgentProjectAccessRoutes(router: Router, db: Db) {
|
||||
router.get("/agents/:id/project-access", async (req, res) => {
|
||||
assertBoard(req);
|
||||
const [agent] = await db.select({ id: agents.id, companyId: agents.companyId }).from(agents).where(eq(agents.id, req.params.id as string));
|
||||
if (!agent || !hasCompanyAccess(req, agent.companyId)) throw notFound("Agent not found");
|
||||
assertCompanyAccess(req, agent.companyId);
|
||||
res.json(await readAgentProjectAccess(db, req.actor, agent.companyId, agent.id));
|
||||
});
|
||||
router.post("/agents/:id/project-access", validate(addAgentProjectAccessSchema), async (req, res) => {
|
||||
assertBoard(req);
|
||||
const [agent] = await db.select({ id: agents.id, companyId: agents.companyId }).from(agents).where(eq(agents.id, req.params.id as string));
|
||||
if (!agent || !hasCompanyAccess(req, agent.companyId)) throw notFound("Agent not found");
|
||||
assertCompanyAccess(req, agent.companyId);
|
||||
res.json(await addAgentProjectAccess(db, req.actor, agent.companyId, agent.id, req.body.projectIds));
|
||||
});
|
||||
}
|
||||
+25
-10
@@ -98,6 +98,7 @@ import {
|
||||
} from "@paperclipai/adapter-utils/server-utils";
|
||||
import { trackAgentCreated } from "@paperclipai/shared/telemetry";
|
||||
import { validate } from "../middleware/validate.js";
|
||||
import { registerAgentProjectAccessRoutes } from "./agent-project-access.js";
|
||||
import { inheritNativeRunnerAdapterConfig } from "../services/native-runtime/native-agent-runtime-inheritance.js";
|
||||
import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js";
|
||||
import { agentInstructionWorkingCopyService } from "../services/agent-instruction-working-copies.js";
|
||||
@@ -4918,6 +4919,8 @@ export function agentRoutes(
|
||||
res.status(201).json({ ...redactAgentRowForResponse(agent), ...(configurationWarnings.length ? { configurationWarnings } : {}) });
|
||||
});
|
||||
|
||||
registerAgentProjectAccessRoutes(router, db);
|
||||
|
||||
router.patch("/agents/:id/permissions", validate(updateAgentPermissionsSchema), async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const existing = await getAccessibleResource(req, res, svc.getById(id), "Agent not found");
|
||||
@@ -7555,7 +7558,11 @@ export function agentRoutes(
|
||||
|
||||
const afterSeq = Number(req.query.afterSeq ?? 0);
|
||||
const limit = Number(req.query.limit ?? 200);
|
||||
const events = await heartbeat.listEvents(runId, Number.isFinite(afterSeq) ? afterSeq : 0, Number.isFinite(limit) ? limit : 200);
|
||||
const cursor = Number.isFinite(afterSeq) ? afterSeq : 0;
|
||||
const pageLimit = Number.isFinite(limit) ? limit : 200;
|
||||
const events = await (req.query.view === "conversation"
|
||||
? heartbeat.listEvents(runId, cursor, pageLimit, true)
|
||||
: heartbeat.listEvents(runId, cursor, pageLimit));
|
||||
const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
|
||||
const redactedEvents = events.map((event) =>
|
||||
redactCurrentUserValue({
|
||||
@@ -7575,24 +7582,32 @@ export function agentRoutes(
|
||||
|
||||
const offset = Number(req.query.offset ?? 0);
|
||||
const limitBytes = readRunLogLimitBytes(req.query.limitBytes);
|
||||
const result = await heartbeat.readLog(run, {
|
||||
offset: Number.isFinite(offset) ? offset : 0,
|
||||
limitBytes,
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const cancelRead = () => controller.abort();
|
||||
res.once("close", cancelRead);
|
||||
try {
|
||||
const result = await heartbeat.readLog(run, {
|
||||
offset: Number.isFinite(offset) ? offset : 0,
|
||||
limitBytes,
|
||||
...(req.query.alignStartToRecord === "true" ? { alignStartToRecord: true } : {}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
res.set("Cache-Control", "no-cache, no-store");
|
||||
res.json(await runRedactions.redactForRun(run.companyId, run.id, result));
|
||||
res.set("Cache-Control", "no-cache, no-store");
|
||||
res.json(await runRedactions.redactForRun(run.companyId, run.id, result));
|
||||
} finally {
|
||||
res.off("close", cancelRead);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/heartbeat-runs/:runId/workspace-operations", async (req, res) => {
|
||||
const runId = readHeartbeatRunId(req);
|
||||
const run = await getAccessibleResource(req, res, heartbeat.getRun(runId), "Heartbeat run not found");
|
||||
const run = await getAccessibleResource(req, res, heartbeat.getRunWorkspaceAccess(runId), "Heartbeat run not found");
|
||||
if (!run) return;
|
||||
if (!(await assertRunTelemetryReadAllowed(req, res, run.companyId))) return;
|
||||
if (!(await assertRunReadAllowed(req, res, run))) return;
|
||||
|
||||
const context = asRecord(run.contextSnapshot);
|
||||
const executionWorkspaceId = asNonEmptyString(context?.executionWorkspaceId);
|
||||
const executionWorkspaceId = asNonEmptyString(run.executionWorkspaceId);
|
||||
const operations = await workspaceOperations.listForRun(runId, executionWorkspaceId);
|
||||
const visibleOperations = [];
|
||||
for (const operation of operations) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
} from "@paperclipai/db";
|
||||
import { and, eq, inArray, isNotNull } from "drizzle-orm";
|
||||
import { getExecutionBlocker } from "../services/execution-blocker.js";
|
||||
import { ISSUE_TREE_RESUME_EXECUTABLE_STATUSES } from "../services/issue-tree-control.js";
|
||||
import { conflict } from "../errors.js";
|
||||
import {
|
||||
createIssueTreeHoldSchema,
|
||||
@@ -26,7 +27,6 @@ import { assertBoard, getAccessibleResource, getActorInfo } from "./authz.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
|
||||
const TREE_RUN_CANCELLATION_RESPONSE_WAIT_MS = 1_000;
|
||||
const RESUME_EXECUTABLE_STATUSES = ["todo", "in_progress", "in_review"];
|
||||
|
||||
function errorToMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
@@ -436,15 +436,13 @@ export function issueTreeControlRoutes(
|
||||
const activeHold = await treeControlSvc.getHold(root.companyId, holdId);
|
||||
const issueIds =
|
||||
activeHold?.mode === "pause" && activeHold.rootIssueId === root.id
|
||||
? (activeHold.members ?? [])
|
||||
.filter((member) => !member.skipped)
|
||||
.map((member) => member.issueId)
|
||||
? (activeHold.members ?? []).map((member) => member.issueId)
|
||||
: [];
|
||||
if (issueIds.length > 0) {
|
||||
const candidates = await db.select({ id: issueRows.id, identifier: issueRows.identifier })
|
||||
.from(issueRows).where(and(
|
||||
eq(issueRows.companyId, root.companyId), inArray(issueRows.id, issueIds),
|
||||
inArray(issueRows.status, RESUME_EXECUTABLE_STATUSES), isNotNull(issueRows.assigneeAgentId),
|
||||
inArray(issueRows.status, ISSUE_TREE_RESUME_EXECUTABLE_STATUSES), isNotNull(issueRows.assigneeAgentId),
|
||||
));
|
||||
for (const task of candidates) {
|
||||
const blocked = await getExecutionBlocker(db, root.companyId, task.id);
|
||||
@@ -489,16 +487,18 @@ export function issueTreeControlRoutes(
|
||||
const wakeFailures: Array<{ issueId: string; message: string }> = [];
|
||||
if (hold.mode === "pause" && req.body.metadata?.wakeAgents === true) {
|
||||
for (const member of hold.members ?? []) {
|
||||
if (member.skipped) continue;
|
||||
try {
|
||||
const issue = await issuesSvc.getById(member.issueId);
|
||||
if (
|
||||
!issue ||
|
||||
issue.companyId !== root.companyId ||
|
||||
!issue.assigneeAgentId ||
|
||||
!RESUME_EXECUTABLE_STATUSES.includes(issue.status)
|
||||
!ISSUE_TREE_RESUME_EXECUTABLE_STATUSES.includes(issue.status)
|
||||
)
|
||||
continue;
|
||||
// The snapshot may predate reopening a task or releasing another
|
||||
// pause. Current task state and remaining holds govern execution.
|
||||
if (await treeControlSvc.getActivePauseHoldGate(root.companyId, issue.id)) continue;
|
||||
await heartbeat.wakeup(issue.assigneeAgentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
|
||||
+55
-11
@@ -7247,18 +7247,20 @@ export function issueRoutes(
|
||||
id: string;
|
||||
assigneeAgentId: string | null;
|
||||
executionRunId?: string | null;
|
||||
}) {
|
||||
}, options?: { queueStateOnly: true }) {
|
||||
let runToInterrupt = issue.executionRunId
|
||||
? await heartbeat.getRun(issue.executionRunId)
|
||||
? options?.queueStateOnly
|
||||
? await heartbeat.getRunIssueControl(issue.executionRunId)
|
||||
: await heartbeat.getRun(issue.executionRunId)
|
||||
: null;
|
||||
|
||||
if (
|
||||
(!runToInterrupt || runToInterrupt.status !== "running") &&
|
||||
issue.assigneeAgentId
|
||||
) {
|
||||
const activeRun = await heartbeat.getActiveRunForAgent(
|
||||
issue.assigneeAgentId,
|
||||
);
|
||||
const activeRun = options?.queueStateOnly
|
||||
? await heartbeat.getActiveRunIssueControlForAgent(issue.assigneeAgentId)
|
||||
: await heartbeat.getActiveRunForAgent(issue.assigneeAgentId);
|
||||
const activeIssueId =
|
||||
activeRun &&
|
||||
activeRun.contextSnapshot &&
|
||||
@@ -7285,9 +7287,9 @@ export function issueRoutes(
|
||||
type IssueQueueWake = typeof agentWakeupRequests.$inferSelect;
|
||||
type IssueQueueRun = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueQueueState = {
|
||||
wake: IssueQueueWake;
|
||||
wake: Pick<IssueQueueWake, "id" | "companyId" | "payload" | "status" | "runId">;
|
||||
state: "deferred" | "queued";
|
||||
queueRun: IssueQueueRun | null;
|
||||
queueRun: Pick<IssueQueueRun, "runtimeMode"> | null;
|
||||
};
|
||||
|
||||
async function findQueuedCommentWake(
|
||||
@@ -7296,12 +7298,29 @@ export function issueRoutes(
|
||||
): Promise<IssueQueueState | null> {
|
||||
if (!issue.assigneeAgentId) return null;
|
||||
const rows = await executor
|
||||
.select()
|
||||
.select({
|
||||
id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
|
||||
status: agentWakeupRequests.status, runId: agentWakeupRequests.runId,
|
||||
payload: sql<Record<string, unknown>>`jsonb_build_object(
|
||||
'issueId', ${agentWakeupRequests.payload}->'issueId',
|
||||
'mutation', ${agentWakeupRequests.payload}->'mutation',
|
||||
'interactionId', ${agentWakeupRequests.payload}->'interactionId',
|
||||
'interactionStatus', ${agentWakeupRequests.payload}->'interactionStatus',
|
||||
'externalChatContinuation', ${agentWakeupRequests.payload}->'externalChatContinuation',
|
||||
'executionWait', jsonb_build_object(
|
||||
'reason', ${agentWakeupRequests.payload}->'executionWait'->'reason',
|
||||
'message', ${agentWakeupRequests.payload}->'executionWait'->'message'),
|
||||
'_paperclipWakeContext', jsonb_build_object(
|
||||
'wakeCommentIds', ${agentWakeupRequests.payload}->'_paperclipWakeContext'->'wakeCommentIds',
|
||||
'forceFreshSession', ${agentWakeupRequests.payload}->'_paperclipWakeContext'->'forceFreshSession')
|
||||
)`,
|
||||
})
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, issue.companyId),
|
||||
eq(agentWakeupRequests.agentId, issue.assigneeAgentId),
|
||||
sql`${agentWakeupRequests.payload}->'issueId' = to_jsonb(${issue.id}::text)`,
|
||||
inArray(agentWakeupRequests.status, [
|
||||
"deferred_issue_execution",
|
||||
"queued",
|
||||
@@ -7321,7 +7340,7 @@ export function issueRoutes(
|
||||
}
|
||||
if (!wake.runId) continue;
|
||||
const queueRun = await executor
|
||||
.select()
|
||||
.select({ runtimeMode: heartbeatRuns.runtimeMode })
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
@@ -7342,7 +7361,7 @@ export function issueRoutes(
|
||||
async function queueCommentsForWake(
|
||||
executor: IssueQueueDb,
|
||||
issueId: string,
|
||||
wake: IssueQueueWake | null,
|
||||
wake: Pick<IssueQueueWake, "companyId" | "payload"> | null,
|
||||
) {
|
||||
if (wake && queuedInteractionId(wake.payload)) {
|
||||
const response = await readQueuedInteractionResponse(executor as Db, wake.companyId, issueId, wake.payload);
|
||||
@@ -9298,6 +9317,13 @@ export function issueRoutes(
|
||||
);
|
||||
if (!issue) return;
|
||||
if (!(await timing.time("authorization", () => assertIssueReadAllowed(req, res, issue, { allowBreakGlass: true })))) return;
|
||||
// The cold-open read owns only the task row. Related objects and recovery
|
||||
// projections retain their existing authorized detail path.
|
||||
if (req.query.view === "core") {
|
||||
res.setHeader("Server-Timing", timing.header());
|
||||
res.json(issue);
|
||||
return;
|
||||
}
|
||||
const inboxArchiveFieldsPromise =
|
||||
req.actor.type === "board" && req.actor.userId
|
||||
? timing.time("inbox", () => svc.getActiveInboxArchiveFields(issue, req.actor.userId!))
|
||||
@@ -16255,6 +16281,24 @@ export function issueRoutes(
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
router.get("/issues/:id/recent-comments", async (req, res) => {
|
||||
const issue = await getAccessibleResource(req, res, getIssueById(req, req.params.id as string), "Issue not found");
|
||||
if (!issue || !(await assertIssueReadAllowed(req, res, issue))) return;
|
||||
const [recent, human, firstInstruction] = await Promise.all([
|
||||
svc.listComments(issue.id, { order: "desc", limit: 5 }),
|
||||
svc.listComments(issue.id, { order: "desc", limit: 1, humanOnly: true }),
|
||||
issue.description?.trim()
|
||||
? Promise.resolve([])
|
||||
: svc.listComments(issue.id, { order: "asc", limit: 1, humanOnly: true }),
|
||||
]);
|
||||
const comments = [...new Map([...firstInstruction, ...human, ...recent.slice(0, 4)].map((comment) => [comment.id, comment])).values()];
|
||||
res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, {
|
||||
comments,
|
||||
latestCommentId: recent[0]?.id ?? null,
|
||||
hasMore: recent.length > 4,
|
||||
}));
|
||||
});
|
||||
|
||||
router.get("/issues/:id/comments", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await getAccessibleResource(
|
||||
@@ -16343,7 +16387,7 @@ export function issueRoutes(
|
||||
const queue = await buildQueuedCommentQueue({
|
||||
executor: db,
|
||||
issue,
|
||||
activeRun: await resolveActiveIssueRun(issue),
|
||||
activeRun: await resolveActiveIssueRun(issue, { queueStateOnly: true }),
|
||||
actor: getActorInfo(req),
|
||||
});
|
||||
res.json(
|
||||
|
||||
@@ -48,6 +48,7 @@ import {
|
||||
updateAgentSchema,
|
||||
updatePrimaryAgentSchema,
|
||||
updateAgentPermissionsSchema,
|
||||
addAgentProjectAccessSchema,
|
||||
updateAgentInstructionsPathSchema,
|
||||
updateAgentInstructionsBundleSchema,
|
||||
upsertAgentInstructionsFileSchema,
|
||||
@@ -1390,6 +1391,8 @@ const browserUseOperations = [
|
||||
] as const;
|
||||
|
||||
const BOARD_ONLY_OPERATIONS = new Set([
|
||||
"GET /api/agents/{id}/project-access",
|
||||
"POST /api/agents/{id}/project-access",
|
||||
"POST /api/issues/{id}/interactions/{interactionId}/dismiss",
|
||||
"POST /api/issues/{id}/retry",
|
||||
"GET /api/companies/{companyId}/prompt-injection-prevention",
|
||||
@@ -3790,6 +3793,26 @@ registry.registerPath({
|
||||
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized },
|
||||
});
|
||||
|
||||
const agentProjectAccessResponse = z.object({ projectIds: z.array(z.string().uuid()), canManage: z.boolean() });
|
||||
registry.registerPath({
|
||||
method: "get",
|
||||
path: "/api/agents/{id}/project-access",
|
||||
tags: ["agents"],
|
||||
summary: "Read an agent's allowed projects and the current operator's management capability",
|
||||
request: { params: z.object({ id: z.string() }) },
|
||||
responses: { 200: r.ok(agentProjectAccessResponse), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
|
||||
});
|
||||
registry.registerPath({
|
||||
method: "post",
|
||||
path: "/api/agents/{id}/project-access",
|
||||
tags: ["agents"],
|
||||
summary: "Add allowed projects without replacing an agent's other permissions",
|
||||
description: "Requires board access and agents:create in the agent's company. Accepts 1 to 100 project IDs. Selected projects must be readable and in that company. Locks the low-trust agent, preserves existing containment, audits new additions, and rejects pending-approval agents or changed trust settings. Repeated additions are idempotent. Saving does not start or retry a task.",
|
||||
request: { params: z.object({ id: z.string() }), body: jsonBody(addAgentProjectAccessSchema) },
|
||||
responses: { 200: r.ok(agentProjectAccessResponse), 400: r.badRequest, 401: r.unauthorized,
|
||||
403: r.forbidden, 404: r.notFound, 409: r.conflict },
|
||||
});
|
||||
|
||||
registry.registerPath({
|
||||
method: "patch",
|
||||
path: "/api/agents/{id}/instructions-path",
|
||||
@@ -4294,7 +4317,7 @@ registry.registerPath({
|
||||
path: "/api/issues/{id}",
|
||||
tags: ["issues"],
|
||||
summary: "Get an issue",
|
||||
request: { params: z.object({ id: z.string() }) },
|
||||
request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("core").optional().describe("Core task fields without related enrichment.") }) },
|
||||
responses: { 200: r.ok(), 401: r.unauthorized, 404: r.notFound },
|
||||
});
|
||||
|
||||
@@ -4601,6 +4624,16 @@ registry.registerPath({
|
||||
responses: { 200: r.ok(), 401: r.unauthorized },
|
||||
});
|
||||
|
||||
registry.registerPath({
|
||||
method: "get",
|
||||
path: "/api/issues/{id}/recent-comments",
|
||||
tags: ["issues"],
|
||||
summary: "Read a bounded recent task exchange",
|
||||
description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.",
|
||||
request: { params: z.object({ id: z.string() }) },
|
||||
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
|
||||
});
|
||||
|
||||
registry.registerPath({
|
||||
method: "get",
|
||||
path: "/api/issues/{id}/comments",
|
||||
@@ -6057,8 +6090,11 @@ registry.registerPath({
|
||||
path: "/api/issues/{id}/activity",
|
||||
tags: ["activity"],
|
||||
summary: "List activity for an issue",
|
||||
request: { params: z.object({ id: z.string() }) },
|
||||
responses: { 200: r.ok(), 401: r.unauthorized },
|
||||
request: { params: z.object({ id: z.string() }), query: z.object({
|
||||
view: z.enum(["recent", "comment-inputs"]).optional().describe("Recent activity (at most 50 rows), or original comment/steering facts for selected comments."),
|
||||
commentIds: z.string().optional().describe("For comment-inputs: up to 100 comma-separated comment UUIDs. Returns at most two original facts per comment."),
|
||||
}) },
|
||||
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized },
|
||||
});
|
||||
|
||||
registry.registerPath({
|
||||
@@ -6066,7 +6102,7 @@ registry.registerPath({
|
||||
path: "/api/issues/{id}/runs",
|
||||
tags: ["activity"],
|
||||
summary: "List runs for an issue",
|
||||
request: { params: z.object({ id: z.string() }) },
|
||||
request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries.") }) },
|
||||
responses: { 200: r.ok(), 401: r.unauthorized },
|
||||
});
|
||||
|
||||
@@ -7798,7 +7834,11 @@ registry.registerPath({
|
||||
path: "/api/heartbeat-runs/{runId}/events",
|
||||
tags: ["runs"],
|
||||
summary: "Get events for a heartbeat run",
|
||||
request: { params: z.object({ runId: heartbeatRunIdParamSchema }) },
|
||||
request: { params: z.object({ runId: heartbeatRunIdParamSchema }), query: z.object({
|
||||
view: z.literal("conversation").optional().describe("Assistant text and request lifecycle events, excluding historical tool activity."),
|
||||
afterSeq: z.coerce.number().int().min(0).optional(),
|
||||
limit: z.coerce.number().int().min(1).max(1000).optional(),
|
||||
}) },
|
||||
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized },
|
||||
});
|
||||
|
||||
@@ -7807,7 +7847,14 @@ registry.registerPath({
|
||||
path: "/api/heartbeat-runs/{runId}/log",
|
||||
tags: ["runs"],
|
||||
summary: "Get log for a heartbeat run",
|
||||
request: { params: z.object({ runId: heartbeatRunIdParamSchema }) },
|
||||
request: {
|
||||
params: z.object({ runId: heartbeatRunIdParamSchema }),
|
||||
query: z.object({
|
||||
offset: z.coerce.number().int().min(0).optional(),
|
||||
limitBytes: z.coerce.number().int().min(1).max(1024 * 1024).optional(),
|
||||
alignStartToRecord: z.enum(["true", "false"]).optional().describe("Extend the tail backward to include its first complete log record. The response can exceed limitBytes by that record's prefix."),
|
||||
}),
|
||||
},
|
||||
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized },
|
||||
});
|
||||
|
||||
|
||||
@@ -1162,6 +1162,7 @@ export function accessService(db: Db) {
|
||||
return {
|
||||
isInstanceAdmin,
|
||||
decide,
|
||||
decideReads: authorization.decideReads,
|
||||
projectDiscoveryCandidateIds: authorization.projectDiscoveryCandidateIds,
|
||||
canUser,
|
||||
hasPermission,
|
||||
|
||||
@@ -37,7 +37,8 @@ export function isAcknowledgedNativeStop(run: Parameters<typeof hasAcknowledgedN
|
||||
}
|
||||
|
||||
/** A Stop acknowledgement alone does not prove provider cleanup completed. */
|
||||
export async function acknowledgedNativeStopExecutionHasStopped(db: Db, run: typeof heartbeatRuns.$inferSelect) {
|
||||
export async function acknowledgedNativeStopExecutionHasStopped(db: Db, run: Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "nativeIssueId" | "status" | "resultJson" | "processPid" | "processGroupId">) {
|
||||
if (!isAcknowledgedNativeStop(run)) return false;
|
||||
const leases = await db.select().from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
||||
|
||||
@@ -396,8 +396,24 @@ export function activityService(db: Db) {
|
||||
.then((rows) => rows.map((r) => r.activityLog));
|
||||
},
|
||||
|
||||
forIssue: (issueId: string) =>
|
||||
db
|
||||
inputFactsForIssue: async (companyId: string, issueId: string, commentIds: string[]) => {
|
||||
if (!commentIds.length) return [];
|
||||
const commentId = sql<string>`${activityLog.details}->>'commentId'`;
|
||||
return db.selectDistinctOn([commentId, activityLog.action])
|
||||
.from(activityLog)
|
||||
.where(and(
|
||||
eq(activityLog.companyId, companyId),
|
||||
eq(activityLog.entityType, "issue"),
|
||||
eq(activityLog.entityId, issueId),
|
||||
inArray(commentId, commentIds),
|
||||
inArray(activityLog.action, ["issue.queued_comment_steered", "issue.comment_added"]),
|
||||
sql`${activityLog.details}->>'duplicate' is distinct from 'true'`,
|
||||
))
|
||||
.orderBy(commentId, activityLog.action, asc(activityLog.createdAt), asc(activityLog.id));
|
||||
},
|
||||
|
||||
forIssue: (issueId: string, limit?: number) => {
|
||||
const query = db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(
|
||||
@@ -407,11 +423,13 @@ export function activityService(db: Db) {
|
||||
sql`${activityLog.companyId} = (select company_id from issues where id = ${issueId})`),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(activityLog.createdAt)),
|
||||
.orderBy(desc(activityLog.createdAt), desc(activityLog.id));
|
||||
return limit === undefined ? query : query.limit(limit);
|
||||
},
|
||||
|
||||
runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null) => {
|
||||
runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number) => {
|
||||
scheduleRunLivenessBackfill(companyId, issueId);
|
||||
const runs = await db
|
||||
const runsQuery = db
|
||||
.select({
|
||||
runId: heartbeatRuns.id,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
@@ -473,7 +491,8 @@ export function activityService(db: Db) {
|
||||
),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(heartbeatRuns.createdAt));
|
||||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id));
|
||||
const runs = await (limit === undefined ? runsQuery : runsQuery.limit(limit));
|
||||
|
||||
if (runs.length === 0) return runs;
|
||||
const runIds = runs.map((run) => run.runId);
|
||||
|
||||
@@ -282,7 +282,7 @@ export async function prepareConversationTurn(
|
||||
/** Finalizers only park a turn with a durable response, interaction, or processed /new. */
|
||||
export async function settleConversationTurn(
|
||||
db: Db,
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "status" | "contextSnapshot">,
|
||||
) {
|
||||
if (run.status !== "succeeded") return false;
|
||||
const context = run.contextSnapshot ?? {};
|
||||
@@ -291,7 +291,12 @@ export async function settleConversationTurn(
|
||||
let publication: ActivityPublication | null = null;
|
||||
const settled = await db.transaction(async (tx) => {
|
||||
const [issue] = await tx
|
||||
.select()
|
||||
.select({
|
||||
id: issues.id, companyId: issues.companyId, status: issues.status,
|
||||
conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId,
|
||||
conversationState: issues.conversationState, conversationSessionGeneration: issues.conversationSessionGeneration,
|
||||
executionRunId: issues.executionRunId,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
|
||||
.for("update");
|
||||
@@ -344,7 +349,7 @@ export async function settleConversationTurn(
|
||||
const [wake] =
|
||||
typeof wakeId === "string"
|
||||
? await tx
|
||||
.select()
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.id, wakeId))
|
||||
: [];
|
||||
@@ -407,6 +412,26 @@ export async function settleConversationTurn(
|
||||
return settled;
|
||||
}
|
||||
|
||||
export async function settleConversationTurnForRun(
|
||||
db: Db,
|
||||
companyId: string,
|
||||
runId: string,
|
||||
) {
|
||||
const [run] = await db.select({
|
||||
id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, status: heartbeatRuns.status,
|
||||
contextSnapshot: sql<Record<string, unknown> | null>`case when jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object'
|
||||
then jsonb_build_object(
|
||||
'issueId', ${heartbeatRuns.contextSnapshot}->'issueId',
|
||||
'conversationReset', ${heartbeatRuns.contextSnapshot}->'conversationReset',
|
||||
'conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration',
|
||||
'conversationReplyBoundaryCommentId', ${heartbeatRuns.contextSnapshot}->'conversationReplyBoundaryCommentId',
|
||||
'wakeCommentId', ${heartbeatRuns.contextSnapshot}->'wakeCommentId',
|
||||
'commentId', ${heartbeatRuns.contextSnapshot}->'commentId'
|
||||
) else null end`,
|
||||
}).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId)));
|
||||
return run ? settleConversationTurn(db, run) : false;
|
||||
}
|
||||
|
||||
/** Fresh provider context uses only messages in this session, up to this turn. */
|
||||
export async function conversationReplay(
|
||||
db: Db,
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { agents, projects, type Db } from "@paperclipai/db";
|
||||
import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared";
|
||||
import { conflict, forbidden, notFound } from "../errors.js";
|
||||
import { normalizeAgentPermissions } from "../lib/agent-permissions.js";
|
||||
import { accessService } from "./access.js";
|
||||
import type { AuthorizationActor } from "./authorization.js";
|
||||
import { logActivity, publishActivity, type ActivityPublication } from "./activity-log.js";
|
||||
|
||||
export async function readAgentProjectAccess(db: Db, actor: AuthorizationActor, companyId: string, agentId: string): Promise<AgentProjectAccess> {
|
||||
const [agent] = await db.select({ permissions: agents.permissions, status: agents.status }).from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.id, agentId)));
|
||||
if (!agent) throw notFound("Agent not found");
|
||||
const decision = await accessService(db).decide({ actor, action: "agents:create", resource: { type: "company", companyId } });
|
||||
const permissions = normalizeAgentPermissions(agent.permissions);
|
||||
const policy = trustAuthorizationPolicySchema.safeParse(permissions.authorizationPolicy ?? {});
|
||||
const boundary = policy.success ? policy.data.trustBoundary : undefined;
|
||||
return { projectIds: policy.success ? policy.data.trustBoundary?.projectIds ?? [] : [],
|
||||
canManage: actor.type === "board" && decision.allowed && agent.status !== "pending_approval"
|
||||
&& policy.success && permissions.trustPreset === "low_trust_review"
|
||||
&& (!boundary || (boundary.mode === "low_trust_review" && (!boundary.companyId || boundary.companyId === companyId))) };
|
||||
}
|
||||
|
||||
/** An additive operation under the agent row lock, never a stale policy replacement. */
|
||||
export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, companyId: string, agentId: string, projectIds: string[]) {
|
||||
if (actor.type !== "board") throw forbidden("Board access required");
|
||||
const publications: ActivityPublication[] = [];
|
||||
const result = await db.transaction(async tx => {
|
||||
const [agent] = await tx.select().from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))).for("update");
|
||||
if (!agent) throw notFound("Agent not found");
|
||||
const transactionDb = tx as unknown as Db;
|
||||
const access = accessService(transactionDb);
|
||||
const decision = await access.decide({ actor, action: "agents:create", resource: { type: "company", companyId } });
|
||||
if (!decision.allowed) throw forbidden(decision.explanation);
|
||||
if (agent.status === "pending_approval") throw conflict("Pending approval agent permissions cannot be changed before board approval");
|
||||
const permissions = normalizeAgentPermissions(agent.permissions);
|
||||
const parsedPolicy = trustAuthorizationPolicySchema.safeParse(permissions.authorizationPolicy ?? {});
|
||||
if (!parsedPolicy.success) throw conflict("The agent’s trust settings need review before adding project access.");
|
||||
const policy = parsedPolicy.data;
|
||||
const boundary = policy?.trustBoundary;
|
||||
if (permissions.trustPreset !== "low_trust_review" || (boundary && (boundary.mode !== "low_trust_review"
|
||||
|| (boundary.companyId && boundary.companyId !== companyId)))) {
|
||||
throw conflict("The agent’s trust settings changed. Refresh before adding project access.");
|
||||
}
|
||||
const requested = [...new Set(projectIds)];
|
||||
const rows = await tx.select({ id: projects.id }).from(projects)
|
||||
.where(and(eq(projects.companyId, companyId), inArray(projects.id, requested)));
|
||||
if (rows.length !== requested.length) throw notFound("A selected project is unavailable");
|
||||
for (const project of rows) {
|
||||
const projectDecision = await access.decide({ actor, action: "project:read", resource: { type: "project", companyId, projectId: project.id } });
|
||||
if (!projectDecision.allowed) throw forbidden("A selected project is unavailable");
|
||||
}
|
||||
const previous = boundary?.projectIds ?? [];
|
||||
const added = requested.filter(id => !previous.includes(id));
|
||||
const next = [...previous, ...added];
|
||||
if (added.length) {
|
||||
await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: {
|
||||
...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next },
|
||||
} }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
|
||||
await logActivity(transactionDb, { companyId, actorType: "user", actorId: actor.userId ?? "local-board",
|
||||
action: "agent.project_access_added", entityType: "agent", entityId: agentId,
|
||||
details: { addedProjectIds: added, trustPreset: "low_trust_review" } }, publications);
|
||||
}
|
||||
return { projectIds: next, canManage: true } satisfies AgentProjectAccess;
|
||||
});
|
||||
for (const publication of publications) publishActivity(publication);
|
||||
return result;
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { type BudgetServiceHooks } from "./budgets.js";
|
||||
|
||||
import { and, desc, eq, inArray, ne } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agents, agentConfigRevisions, agentApiKeys, heartbeatRuns } from "@paperclipai/db";
|
||||
import { isUuidLike, normalizeAgentApiKeyScope, normalizeAgentUrlKey, type AgentApiKeyScope } from "@paperclipai/shared";
|
||||
@@ -31,14 +31,10 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
createAgentLifecycle(db, budgetHooks).updateConfiguration(...args);
|
||||
return {
|
||||
list: async (companyId: string, options?: { includeTerminated?: boolean }) => {
|
||||
const conditions = [eq(agents.companyId, companyId)];
|
||||
if (!options?.includeTerminated) {
|
||||
conditions.push(ne(agents.status, "terminated"));
|
||||
}
|
||||
const [rows, allCompanyRows] = await Promise.all([
|
||||
db.select().from(agents).where(and(...conditions)),
|
||||
listCompanyAgentRows(companyId),
|
||||
]);
|
||||
const allCompanyRows = await listCompanyAgentRows(companyId);
|
||||
const rows = options?.includeTerminated
|
||||
? allCompanyRows
|
||||
: allCompanyRows.filter((row) => row.status !== "terminated");
|
||||
const hydrated = await hydrateAgentSpend(rows);
|
||||
return normalizeAgentRows(hydrated, allCompanyRows);
|
||||
},
|
||||
|
||||
@@ -1099,6 +1099,65 @@ function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
|
||||
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
|
||||
}
|
||||
|
||||
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
|
||||
|
||||
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
|
||||
|
||||
async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
const withNewerRun = new Set<string>();
|
||||
const keyed = failedRuns.filter(run => run.issueId !== null);
|
||||
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
|
||||
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
|
||||
}
|
||||
const unkeyed = failedRuns.filter(run => run.issueId === null);
|
||||
if (unkeyed.length === 0) return withNewerRun;
|
||||
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
|
||||
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
|
||||
unkeyed[0]!.createdAt);
|
||||
const latestByAgent = new Map<string, Date>();
|
||||
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const rows = await db.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
|
||||
gt(heartbeatRuns.createdAt, oldestCreatedAt),
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
|
||||
)).groupBy(heartbeatRuns.agentId);
|
||||
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
|
||||
}
|
||||
for (const run of unkeyed) {
|
||||
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
|
||||
}
|
||||
return withNewerRun;
|
||||
}
|
||||
|
||||
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
if (failedRuns.length === 0) return [];
|
||||
const failed = sql.join(
|
||||
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
|
||||
sql`, `,
|
||||
);
|
||||
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
|
||||
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
|
||||
const newerRunExists = (match: SQL) => sql`exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${companyId}
|
||||
and ${heartbeatRuns.agentId} = failed.agent_id
|
||||
and ${heartbeatRuns.createdAt} >= failed.newer_from
|
||||
and ${match}
|
||||
)`;
|
||||
const rows = await db.execute(sql`
|
||||
select failed.id
|
||||
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
|
||||
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
|
||||
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
|
||||
`) as unknown as Array<{ id: string }>;
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) {
|
||||
const openDecisionLimit = Math.min(
|
||||
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
|
||||
@@ -1717,12 +1776,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
|
||||
const failedRows = await listAttentionExhaustedRuns(db, companyId);
|
||||
const failedIssueIds = failedRows.map((row) => readRunIssueId(row.contextSnapshot));
|
||||
const failedAgentIds = [...new Set(failedRows.map((row) => row.agentId))];
|
||||
const oldestFailedRunCreatedAt = failedRows.reduce<Date | null>((oldest, row) => {
|
||||
if (!oldest || row.createdAt < oldest) return row.createdAt;
|
||||
return oldest;
|
||||
}, null);
|
||||
const [failedIssueMap, failedImageMap, newerRuns] = await Promise.all([
|
||||
const [failedIssueMap, failedImageMap, failedRunsWithNewerRun] = await Promise.all([
|
||||
issueSummaryMap(
|
||||
db,
|
||||
companyId,
|
||||
@@ -1730,38 +1784,16 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
readCondition,
|
||||
),
|
||||
issueImageMap(db, companyId, failedIssueIds, readCondition),
|
||||
oldestFailedRunCreatedAt && failedAgentIds.length > 0
|
||||
? db
|
||||
.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: heartbeatRuns.createdAt,
|
||||
// Project just the ids readRunIssueId needs; pulling the whole
|
||||
// context_snapshot detoasts megabytes per feed build.
|
||||
runIssueId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'issueId'`,
|
||||
runTaskId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'taskId'`,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, failedAgentIds),
|
||||
gt(heartbeatRuns.createdAt, oldestFailedRunCreatedAt),
|
||||
))
|
||||
: Promise.resolve([]),
|
||||
listFailedRunsWithNewerRun(db, companyId, failedRows.map((row, index) => ({
|
||||
id: row.id,
|
||||
agentId: row.agentId,
|
||||
issueId: failedIssueIds[index],
|
||||
createdAt: row.createdAt,
|
||||
}))),
|
||||
]);
|
||||
const latestRunCreatedAtByKey = new Map<string, Date>();
|
||||
for (const newerRun of newerRuns) {
|
||||
const newerRunIssueId = readRunIssueId({ issueId: newerRun.runIssueId, taskId: newerRun.runTaskId });
|
||||
const newerRunKey = `${newerRun.agentId}:${newerRunIssueId ?? ""}`;
|
||||
const latestCreatedAt = latestRunCreatedAtByKey.get(newerRunKey);
|
||||
if (!latestCreatedAt || newerRun.createdAt > latestCreatedAt) {
|
||||
latestRunCreatedAtByKey.set(newerRunKey, newerRun.createdAt);
|
||||
}
|
||||
}
|
||||
for (const run of failedRows) {
|
||||
const issueId = readRunIssueId(run.contextSnapshot);
|
||||
const runKey = `${run.agentId}:${issueId ?? ""}`;
|
||||
const hasNewerRun = (latestRunCreatedAtByKey.get(runKey)?.getTime() ?? 0) > run.createdAt.getTime();
|
||||
if (hasNewerRun) continue;
|
||||
if (failedRunsWithNewerRun.has(run.id)) continue;
|
||||
|
||||
const issue = issueId ? failedIssueMap.get(issueId) ?? null : null;
|
||||
if (issueId && !issue) continue;
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { companies, companySkills, companySkillVersions, createDb } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
import { verifyExistingBundledSkillRelease } from "./bundled-skill-release-validation.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const hashInventory = (files: Array<{ path: string; content: string }>) => createHash("sha256")
|
||||
.update(JSON.stringify(files.map(file => [file.path, file.content]))).digest("hex");
|
||||
|
||||
(support.supported ? describe : describe.skip)("bundled release validation with current database fingerprints", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("bundled-release-validation-");
|
||||
db = createDb(temporary.connectionString);
|
||||
}, 30_000);
|
||||
afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); });
|
||||
|
||||
async function seed(content = "saved release content") {
|
||||
const companyId = randomUUID(), skillId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Release validation", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(companySkills).values({ id: skillId, companyId, key: "fixture/release", slug: "release",
|
||||
name: "Release", markdown: "# Release", sourceType: "local_path" });
|
||||
const files = [{ path: "SKILL.md", kind: "skill" as const, content }];
|
||||
const [version] = await db.insert(companySkillVersions).values({ companyId, companySkillId: skillId,
|
||||
revisionNumber: 1, releaseId: "release-1", fileInventory: files }).returning();
|
||||
const expectedInventoryHash = hashInventory(files);
|
||||
const validateInventory = vi.fn((row: typeof companySkillVersions.$inferSelect) => {
|
||||
if (hashInventory(row.fileInventory) !== expectedInventoryHash) throw new Error("snapshot mismatch");
|
||||
});
|
||||
return { version: version!, input: { db, companyId, skillId, releaseId: "release-1", expectedInventoryHash, validateInventory } };
|
||||
}
|
||||
|
||||
it("reuses proof only within current company, skill and release bindings", async () => {
|
||||
const f = await seed();
|
||||
expect(await verifyExistingBundledSkillRelease(f.input)).toBe(true);
|
||||
expect(await verifyExistingBundledSkillRelease(f.input)).toBe(true);
|
||||
expect(f.input.validateInventory).toHaveBeenCalledTimes(1);
|
||||
expect(await verifyExistingBundledSkillRelease({ ...f.input, companyId: randomUUID() })).toBe(false);
|
||||
expect(await verifyExistingBundledSkillRelease({ ...f.input, skillId: randomUUID() })).toBe(false);
|
||||
expect(await verifyExistingBundledSkillRelease({ ...f.input, releaseId: "other-release" })).toBe(false);
|
||||
expect(f.input.validateInventory).toHaveBeenCalledTimes(1);
|
||||
expect(await verifyExistingBundledSkillRelease(f.input)).toBe(true);
|
||||
});
|
||||
|
||||
it("revalidates when the expected on-disk inventory changes without poisoning valid proof", async () => {
|
||||
const f = await seed();
|
||||
await verifyExistingBundledSkillRelease(f.input);
|
||||
const changedHash = hashInventory([{ path: "SKILL.md", content: "new on-disk content" }]);
|
||||
const validateChanged = vi.fn((row: typeof companySkillVersions.$inferSelect) => {
|
||||
if (hashInventory(row.fileInventory) !== changedHash) throw new Error("snapshot mismatch");
|
||||
});
|
||||
await expect(verifyExistingBundledSkillRelease({ ...f.input, expectedInventoryHash: changedHash,
|
||||
validateInventory: validateChanged })).rejects.toThrow("snapshot mismatch");
|
||||
expect(validateChanged).toHaveBeenCalledOnce();
|
||||
expect(await verifyExistingBundledSkillRelease(f.input)).toBe(true);
|
||||
expect(f.input.validateInventory).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("rechecks current content after a change overlaps an earlier validation", async () => {
|
||||
const f = await seed();
|
||||
let entered!: () => void, release!: () => void;
|
||||
const validating = new Promise<void>(resolve => { entered = resolve; });
|
||||
const held = new Promise<void>(resolve => { release = resolve; });
|
||||
const first = verifyExistingBundledSkillRelease({ ...f.input, validateInventory: async row => {
|
||||
f.input.validateInventory(row);
|
||||
entered();
|
||||
await held;
|
||||
} });
|
||||
await validating;
|
||||
try {
|
||||
await db.update(companySkillVersions).set({ fileInventory: [{ path: "SKILL.md", kind: "skill", content: "changed" }] })
|
||||
.where(and(eq(companySkillVersions.companyId, f.input.companyId), eq(companySkillVersions.id, f.version.id)));
|
||||
} finally { release(); }
|
||||
expect(await first).toBe(true);
|
||||
await expect(verifyExistingBundledSkillRelease(f.input)).rejects.toThrow("snapshot mismatch");
|
||||
expect(f.input.validateInventory).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not accept a deleted row from earlier proof", async () => {
|
||||
const f = await seed();
|
||||
await verifyExistingBundledSkillRelease(f.input);
|
||||
await db.delete(companySkillVersions).where(eq(companySkillVersions.id, f.version.id));
|
||||
expect(await verifyExistingBundledSkillRelease(f.input)).toBe(false);
|
||||
expect(f.input.validateInventory).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("bounds retained proof and safely revalidates an evicted release", async () => {
|
||||
const f = await seed();
|
||||
await verifyExistingBundledSkillRelease(f.input);
|
||||
for (let revisionNumber = 2; revisionNumber <= 66; revisionNumber++) {
|
||||
const releaseId = `release-${revisionNumber}`;
|
||||
await db.insert(companySkillVersions).values({ companyId: f.input.companyId, companySkillId: f.input.skillId,
|
||||
revisionNumber, releaseId, fileInventory: f.version.fileInventory });
|
||||
expect(await verifyExistingBundledSkillRelease({ ...f.input, releaseId })).toBe(true);
|
||||
}
|
||||
const before = f.input.validateInventory.mock.calls.length;
|
||||
expect(await verifyExistingBundledSkillRelease(f.input)).toBe(true);
|
||||
expect(f.input.validateInventory).toHaveBeenCalledTimes(before + 1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { companySkillVersions, type Db } from "@paperclipai/db";
|
||||
import { and, eq, getTableColumns, sql } from "drizzle-orm";
|
||||
|
||||
type Version = typeof companySkillVersions.$inferSelect;
|
||||
type Verification = { inventorySha256: string; expectedInventoryHash: string };
|
||||
const verifiedInventories = new WeakMap<Db, Map<string, Verification>>();
|
||||
const inventorySha256 = sql<string>`encode(sha256(convert_to(${companySkillVersions.fileInventory}::text, 'UTF8')), 'hex')`;
|
||||
|
||||
export async function verifyExistingBundledSkillRelease(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
skillId: string;
|
||||
releaseId: string;
|
||||
expectedInventoryHash: string;
|
||||
validateInventory: (version: Version) => void | Promise<void>;
|
||||
}): Promise<boolean> {
|
||||
const scope = and(eq(companySkillVersions.companyId, input.companyId),
|
||||
eq(companySkillVersions.companySkillId, input.skillId), eq(companySkillVersions.releaseId, input.releaseId));
|
||||
const [current] = await input.db.select({ id: companySkillVersions.id, inventorySha256 })
|
||||
.from(companySkillVersions).where(scope).limit(1);
|
||||
if (!current) return false;
|
||||
let verified = verifiedInventories.get(input.db);
|
||||
if (!verified) {
|
||||
verified = new Map();
|
||||
verifiedInventories.set(input.db, verified);
|
||||
}
|
||||
const previous = verified.get(current.id);
|
||||
if (previous?.inventorySha256 === current.inventorySha256
|
||||
&& previous.expectedInventoryHash === input.expectedInventoryHash) {
|
||||
verified.delete(current.id);
|
||||
verified.set(current.id, previous);
|
||||
return true;
|
||||
}
|
||||
const [version] = await input.db.select({ ...getTableColumns(companySkillVersions), inventorySha256 })
|
||||
.from(companySkillVersions).where(scope).limit(1);
|
||||
if (!version) return false;
|
||||
await input.validateInventory(version);
|
||||
verified.delete(version.id);
|
||||
verified.set(version.id, { inventorySha256: version.inventorySha256, expectedInventoryHash: input.expectedInventoryHash });
|
||||
while (verified.size > 64) {
|
||||
const oldest = verified.keys().next().value;
|
||||
if (oldest !== undefined) verified.delete(oldest);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { readChatAuthorizationRun } from "./authorization-run-read.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("chat authorization run reads", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let companyId: string;
|
||||
let agentId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("chat-authorization-read-");
|
||||
db = createDb(database.connectionString);
|
||||
const [company] = await db.insert(companies).values({ name: "Chat authorization fixture", issuePrefix: "CAR" }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Chat agent", adapterType: "process" }).returning();
|
||||
companyId = company.id;
|
||||
agentId = agent.id;
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
async function seed(contextSnapshot: unknown = {}) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({
|
||||
companyId, agentId, contextSnapshot: contextSnapshot as Record<string, unknown> | null,
|
||||
}).returning();
|
||||
return { companyId, agentId, failedRunId: run.id };
|
||||
}
|
||||
|
||||
it("reads committed authority below the byte budget and keeps complete retry evidence", async () => {
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const context = {
|
||||
source: "chat:slack", issueId: randomUUID(), interactionId: null, taskKey: "CAR-1",
|
||||
wakeCommentIds: [randomUUID()], wakeCommentId: null,
|
||||
chatFailedRunRetry: { version: 1, actionId: randomUUID(), failedRunId: randomUUID() },
|
||||
retryOfRunId: randomUUID(),
|
||||
};
|
||||
const scope = await seed({ ...context, savedInput: bulk });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { savedInput: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk })
|
||||
.where(eq(heartbeatRuns.id, scope.failedRunId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, scope.failedRunId));
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const selected = await db.transaction(tx => readChatAuthorizationRun(tx, scope, true));
|
||||
expect(selected.kind).toBe("committed");
|
||||
expect(selected.run?.contextSnapshot).toEqual(context);
|
||||
expect(selected.run).not.toHaveProperty("runnerProfileJson");
|
||||
expect(selected.run).not.toHaveProperty("stdoutExcerpt");
|
||||
} finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
expect(summary.completedQueries).toBeGreaterThan(0);
|
||||
expect(summary.responseValueBytes).toBeLessThan(4096);
|
||||
const retry = await db.transaction(tx => readChatAuthorizationRun(tx, scope, false));
|
||||
expect(retry.kind).toBe("retry");
|
||||
expect(retry.run).toEqual(before);
|
||||
const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, scope.failedRunId));
|
||||
expect(after).toEqual(before);
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
});
|
||||
|
||||
it.each([{}, { wakeCommentId: null }, { wakeCommentId: false, wakeCommentIds: "invalid", interactionId: [] }])(
|
||||
"preserves key presence and JSON types in %j", async context => {
|
||||
const scope = await seed(context);
|
||||
expect((await readChatAuthorizationRun(db, scope, true)).run?.contextSnapshot).toEqual(context);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([null, [], "invalid", 42])("handles non-object context %j", async context => {
|
||||
const scope = await seed(context);
|
||||
expect((await readChatAuthorizationRun(db, scope, true)).run?.contextSnapshot).toEqual(context === null ? null : {});
|
||||
expect((await readChatAuthorizationRun(db, scope, false)).run?.contextSnapshot).toEqual(context);
|
||||
});
|
||||
|
||||
it("reads current authority and fails closed for another company or agent", async () => {
|
||||
const scope = await seed({ source: "chat:slack", taskKey: "CAR-1" });
|
||||
expect((await readChatAuthorizationRun(db, scope, true)).run?.contextSnapshot).toEqual({ source: "chat:slack", taskKey: "CAR-1" });
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { source: "chat:discord", taskKey: "CAR-2" } })
|
||||
.where(eq(heartbeatRuns.id, scope.failedRunId));
|
||||
expect((await readChatAuthorizationRun(db, scope, true)).run?.contextSnapshot).toEqual({ source: "chat:discord", taskKey: "CAR-2" });
|
||||
for (const committed of [true, false]) {
|
||||
expect((await readChatAuthorizationRun(db, { ...scope, companyId: randomUUID() }, committed)).run).toBeUndefined();
|
||||
expect((await readChatAuthorizationRun(db, { ...scope, agentId: randomUUID() }, committed)).run).toBeUndefined();
|
||||
expect((await readChatAuthorizationRun(db, { ...scope, failedRunId: randomUUID() }, committed)).run).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves nonblocking authorization row locks", async () => {
|
||||
const scope = await seed();
|
||||
let acquired!: () => void, release!: () => void;
|
||||
const ready = new Promise<void>(resolve => { acquired = resolve; });
|
||||
const released = new Promise<void>(resolve => { release = resolve; });
|
||||
const holder = db.transaction(async tx => {
|
||||
await tx.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, scope.failedRunId));
|
||||
acquired();
|
||||
await released;
|
||||
});
|
||||
try {
|
||||
await ready;
|
||||
for (const committed of [true, false]) {
|
||||
await expect(db.transaction(tx => readChatAuthorizationRun(tx, scope, committed))).rejects.toMatchObject({ cause: { code: "55P03" } });
|
||||
}
|
||||
} finally { release(); await holder; }
|
||||
expect((await readChatAuthorizationRun(db, scope, true)).run?.status).toBe("running");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
|
||||
type Database = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
type Scope = { companyId: string; agentId: string; failedRunId: string };
|
||||
|
||||
const contextKeys = [
|
||||
"source", "issueId", "interactionId", "taskKey", "wakeCommentIds",
|
||||
"wakeCommentId", "chatFailedRunRetry", "retryOfRunId",
|
||||
] as const;
|
||||
|
||||
const committedContext = sql<Record<string, unknown> | null>`
|
||||
case when ${heartbeatRuns.contextSnapshot} is null then null else (
|
||||
select coalesce(jsonb_object_agg(field.key, field.value), '{}'::jsonb)
|
||||
from jsonb_each(case when jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object'
|
||||
then ${heartbeatRuns.contextSnapshot} else '{}'::jsonb end) as field(key, value)
|
||||
where field.key in (${sql.join(contextKeys.map(key => sql`${key}`), sql`, `)})
|
||||
) end
|
||||
`;
|
||||
|
||||
export async function readChatAuthorizationRun(db: Database, scope: Scope, committed: boolean) {
|
||||
const where = and(
|
||||
eq(heartbeatRuns.id, scope.failedRunId),
|
||||
eq(heartbeatRuns.companyId, scope.companyId),
|
||||
eq(heartbeatRuns.agentId, scope.agentId),
|
||||
);
|
||||
if (!committed) {
|
||||
const [run] = await db.select().from(heartbeatRuns).where(where).for("share", { noWait: true });
|
||||
return { kind: "retry" as const, run };
|
||||
}
|
||||
const [run] = await db.select({
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
status: heartbeatRuns.status,
|
||||
finishedAt: heartbeatRuns.finishedAt,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
completionContractId: heartbeatRuns.completionContractId,
|
||||
retryOfRunId: heartbeatRuns.retryOfRunId,
|
||||
contextSnapshot: committedContext,
|
||||
}).from(heartbeatRuns).where(where).for("share", { noWait: true });
|
||||
return { kind: "committed" as const, run };
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { continuingTeamsFilePublication, type ContinuingTeamsFileTransfer } from
|
||||
import { readTaskChatRetryIntent, taskChatRetryCanAwaitRecovery, taskChatRetryExecutionStopped } from "./task-retry-authority.js";
|
||||
import { notifyChatActionWork, notifyChatPublicationWork } from "../chat-work-notifications.js";
|
||||
import { FAILED_RUN_RETRY_STALE_MS, dueChatAction } from "../chat-action-work.js";
|
||||
import { readChatAuthorizationRun } from "./authorization-run-read.js";
|
||||
|
||||
import {
|
||||
createHash,
|
||||
@@ -688,17 +689,8 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
),
|
||||
)
|
||||
.for("share", { noWait: true });
|
||||
const [failedRun] = await tx
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.id, input.failedRunId),
|
||||
eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId),
|
||||
),
|
||||
)
|
||||
.for("share", { noWait: true });
|
||||
const evidence = await readChatAuthorizationRun(tx, input, Boolean(input.committedResponse));
|
||||
const failedRun = evidence.run;
|
||||
if (
|
||||
!issue ||
|
||||
issue.assigneeAgentId !== input.agentId ||
|
||||
@@ -719,7 +711,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
"process_exit",
|
||||
"timeout",
|
||||
].includes(failedRun.errorCode ?? "") ||
|
||||
(await isPreProviderReviewedChatFailure(tx, failedRun))
|
||||
(evidence.kind === "retry" && await isPreProviderReviewedChatFailure(tx, evidence.run))
|
||||
)) ||
|
||||
(!input.publication &&
|
||||
["backlog", "done", "cancelled"].includes(issue.status))
|
||||
@@ -729,8 +721,8 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
companyId: input.companyId, issueId: issue.id, agentId: input.agentId,
|
||||
sourceRunId: failedRun.id, intentId: input.taskRetryIntentId,
|
||||
})) throw failedChatRetryDenied();
|
||||
if (failedRun.runtimeMode === "native" && !input.committedResponse && !input.taskRetryIntentId)
|
||||
await failedNativeRetryCoordinator(tx, failedRun, issue.id);
|
||||
if (failedRun.runtimeMode === "native" && evidence.kind === "retry" && !input.taskRetryIntentId)
|
||||
await failedNativeRetryCoordinator(tx, evidence.run, issue.id);
|
||||
const context = failedRun.contextSnapshot ?? {};
|
||||
const provider = CHAT_PROVIDERS.find(
|
||||
(value) => context.source === `chat:${value}`,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { chatConversations, chatDeliveries, chatEndpointResources, chatEndpoints, chatGitHubConfigurations } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
|
||||
const authorizationEvent = sql<Record<string, unknown>>`case
|
||||
when jsonb_typeof(${chatDeliveries.normalizedEvent}) = 'object' then (
|
||||
select coalesce(jsonb_object_agg(field.key, case
|
||||
when jsonb_typeof(field.value->'context') = 'object' then
|
||||
jsonb_build_object('context', (field.value->'context') - 'body' - 'title')
|
||||
else field.value end), '{}'::jsonb)
|
||||
from jsonb_each(${chatDeliveries.normalizedEvent}) as field(key, value)
|
||||
where field.key in ('githubAutomatic', 'githubIssue')
|
||||
) else ${chatDeliveries.normalizedEvent} end`;
|
||||
|
||||
export const githubResponseAuthorityFields = {
|
||||
endpoint: {
|
||||
id: chatEndpoints.id,
|
||||
companyId: chatEndpoints.companyId,
|
||||
assignedAgentId: chatEndpoints.assignedAgentId,
|
||||
botExternalId: chatEndpoints.botExternalId,
|
||||
botUsername: chatEndpoints.botUsername,
|
||||
provider: chatEndpoints.provider,
|
||||
status: chatEndpoints.status,
|
||||
allowUnlinkedPeople: chatEndpoints.allowUnlinkedPeople,
|
||||
setup: sql<unknown>`case when jsonb_typeof(${chatEndpoints.setup}) = 'object'
|
||||
then jsonb_build_object('runtimeGeneration', ${chatEndpoints.setup}->'runtimeGeneration')
|
||||
else ${chatEndpoints.setup} end`,
|
||||
},
|
||||
conversation: {
|
||||
id: chatConversations.id,
|
||||
issueId: chatConversations.issueId,
|
||||
externalThreadId: chatConversations.externalThreadId,
|
||||
},
|
||||
delivery: {
|
||||
id: chatDeliveries.id,
|
||||
principalId: chatDeliveries.principalId,
|
||||
state: chatDeliveries.state,
|
||||
createdAt: chatDeliveries.createdAt,
|
||||
normalizedEvent: authorizationEvent,
|
||||
},
|
||||
resource: {
|
||||
enabled: chatEndpointResources.enabled,
|
||||
availability: chatEndpointResources.availability,
|
||||
providerResourceId: chatEndpointResources.providerResourceId,
|
||||
metadata: sql<typeof chatEndpointResources.$inferSelect["metadata"]>`case
|
||||
when jsonb_typeof(${chatEndpointResources.metadata}) = 'object'
|
||||
then jsonb_build_object('providerRepositoryId', ${chatEndpointResources.metadata}->'providerRepositoryId')
|
||||
else ${chatEndpointResources.metadata} end`,
|
||||
},
|
||||
toolsEnabled: sql<unknown>`${chatGitHubConfigurations.configuration}->'toolsEnabled'`,
|
||||
};
|
||||
@@ -4,9 +4,11 @@ import { chatActions, chatConversations, chatDeliveries, chatEndpoints, type Db
|
||||
import { forbidden, HttpError } from "../../../errors.js";
|
||||
|
||||
export type GitHubResponseSource = {
|
||||
endpoint: typeof chatEndpoints.$inferSelect;
|
||||
conversation: typeof chatConversations.$inferSelect;
|
||||
delivery: typeof chatDeliveries.$inferSelect;
|
||||
endpoint: Pick<typeof chatEndpoints.$inferSelect,
|
||||
"id" | "companyId" | "assignedAgentId" | "botExternalId" | "botUsername"
|
||||
> & { setup: unknown };
|
||||
conversation: Pick<typeof chatConversations.$inferSelect, "id" | "issueId" | "externalThreadId">;
|
||||
delivery: Pick<typeof chatDeliveries.$inferSelect, "id" | "principalId" | "createdAt">;
|
||||
number: number;
|
||||
replyId: string | null;
|
||||
repositoryId?: string;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
chatEndpoints,
|
||||
chatExternalPrincipals,
|
||||
@@ -13,12 +13,15 @@ type Database = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
/** Current authority shared by task admission and every later publication. */
|
||||
export async function githubChatPrincipalAccess(
|
||||
db: Database,
|
||||
endpoint: typeof chatEndpoints.$inferSelect,
|
||||
endpoint: Pick<typeof chatEndpoints.$inferSelect, "id" | "companyId" | "provider" | "allowUnlinkedPeople">,
|
||||
principalId: string,
|
||||
) {
|
||||
if (endpoint.provider !== "github") return null;
|
||||
const [saved] = await db
|
||||
.select()
|
||||
.select({ configuration: sql<Pick<typeof chatGitHubConfigurations.$inferSelect["configuration"], "people" | "memberAccess">>`jsonb_build_object(
|
||||
'people', ${chatGitHubConfigurations.configuration}->'people',
|
||||
'memberAccess', ${chatGitHubConfigurations.configuration}->'memberAccess'
|
||||
)` })
|
||||
.from(chatGitHubConfigurations)
|
||||
.where(
|
||||
and(
|
||||
@@ -29,7 +32,7 @@ export async function githubChatPrincipalAccess(
|
||||
.for("share");
|
||||
if (!saved) return null; // Preserve the existing connector until explicitly upgraded.
|
||||
const [principal] = await db
|
||||
.select()
|
||||
.select({ externalId: chatExternalPrincipals.externalId })
|
||||
.from(chatExternalPrincipals)
|
||||
.where(
|
||||
and(
|
||||
@@ -50,7 +53,7 @@ export async function githubChatPrincipalAccess(
|
||||
(p) => p.githubUserId === principal.externalId,
|
||||
);
|
||||
const [link] = await db
|
||||
.select()
|
||||
.select({ status: chatIdentityLinks.status, paperclipUserId: chatIdentityLinks.paperclipUserId })
|
||||
.from(chatIdentityLinks)
|
||||
.where(
|
||||
and(
|
||||
@@ -62,7 +65,7 @@ export async function githubChatPrincipalAccess(
|
||||
.for("share");
|
||||
const activeMember = async (userId: string) => {
|
||||
const [membership] = await db
|
||||
.select()
|
||||
.select({ membershipRole: companyMemberships.membershipRole })
|
||||
.from(companyMemberships)
|
||||
.where(
|
||||
and(
|
||||
|
||||
@@ -427,19 +427,24 @@ export function githubReviewCheckService(db: Db, fetchImpl = fetch) {
|
||||
and(
|
||||
inArray(chatGitHubReviews.state, ["queued", "running"]),
|
||||
isNull(chatGitHubReviews.assessment),
|
||||
sql`exists (
|
||||
select 1 from ${chatActions} action
|
||||
join ${chatDeliveries} delivery
|
||||
on delivery.id::text = ${chatGitHubReviews.deliveryId}
|
||||
sql`${chatGitHubReviews.deliveryId} ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'`,
|
||||
sql`(
|
||||
select action.id from ${chatActions} action
|
||||
where action.company_id = ${chatGitHubReviews.companyId}
|
||||
and action.endpoint_id = ${chatGitHubReviews.endpointId}
|
||||
and delivery.endpoint_id = action.endpoint_id
|
||||
and action.kind = 'github_review_check'
|
||||
and action.status = 'cancelled'
|
||||
and action.result->>'code' = 'stale_head'
|
||||
and action.payload->'event'->>'deliveryId' =
|
||||
delivery.normalized_event->'githubAutomatic'->'context'->>'deliveryId'
|
||||
)`,
|
||||
and action.payload->'event'->>'deliveryId' = (
|
||||
select delivery.normalized_event->'githubAutomatic'->'context'->>'deliveryId'
|
||||
from ${chatDeliveries} delivery
|
||||
where delivery.id = case
|
||||
when ${chatGitHubReviews.deliveryId} ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
||||
then ${chatGitHubReviews.deliveryId}::uuid end
|
||||
and delivery.endpoint_id = ${chatGitHubReviews.endpointId}
|
||||
)
|
||||
limit 1
|
||||
) is not null`,
|
||||
),
|
||||
);
|
||||
const rows = await db
|
||||
|
||||
@@ -7,6 +7,18 @@ type Event = SealedConnectorEvents["events"][number];
|
||||
type Endpoint = typeof chatEndpoints.$inferSelect;
|
||||
type Handler = (endpoint: Endpoint, request: Request) => Promise<void>;
|
||||
const handlers = new WeakMap<Db, Handler>();
|
||||
const setupPollers = new WeakMap<Db, () => Promise<void>>();
|
||||
|
||||
/** The active wizard reuses the server-owned, serialized event drain. */
|
||||
export function registerGitHubBotCloudSetupPoller(db: Db, poll: () => Promise<void>) {
|
||||
setupPollers.set(db, poll);
|
||||
return () => { if (setupPollers.get(db) === poll) setupPollers.delete(db); };
|
||||
}
|
||||
|
||||
export async function pollGitHubBotCloudSetupEvents(db: Db): Promise<void> {
|
||||
await setupPollers.get(db)?.();
|
||||
}
|
||||
|
||||
export function registerGitHubBotCloudIngress(db: Db, handler: Handler) {
|
||||
handlers.set(db, handler);
|
||||
return () => {
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
import {
|
||||
effectiveGitHubReviewPolicy,
|
||||
githubReviewSchedulingDecision,
|
||||
type GitHubAutomaticPolicyContext,
|
||||
} from "./chat-github-review-policy.js";
|
||||
|
||||
const id = z.union([
|
||||
@@ -226,8 +227,8 @@ export function githubAutomaticReviewEvent(
|
||||
|
||||
export async function githubAutomaticAdmission(
|
||||
db: Db | Parameters<Parameters<Db["transaction"]>[0]>[0],
|
||||
endpoint: typeof chatEndpoints.$inferSelect,
|
||||
context: GitHubAutomaticEventContext,
|
||||
endpoint: Pick<typeof chatEndpoints.$inferSelect, "id" | "companyId">,
|
||||
context: GitHubAutomaticPolicyContext,
|
||||
) {
|
||||
const [saved] = await db
|
||||
.select()
|
||||
|
||||
@@ -11,7 +11,8 @@ import { githubAutomaticAdmission } from "./chat-github-events.js";
|
||||
import { githubBotRepositoryToken, githubBotRequest } from "./chat-github-client.js";
|
||||
import { withGitHubPublicationLease } from "./chat-github-publication-lease.js";
|
||||
import { projectSafeChatPublicationText } from "./chat-publication-projection.js";
|
||||
import type { GitHubAutomaticEventContext } from "@paperclipai/shared";
|
||||
import type { GitHubAutomaticPolicyContext } from "./chat-github-review-policy.js";
|
||||
import { githubResponseAuthorityFields } from "./chat-channels/github/response-authority-read.js";
|
||||
import { githubRunReplyState } from "./chat-run-publications.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import { boundRunIssueId } from "./run-task-binding.js";
|
||||
@@ -159,8 +160,7 @@ export async function writeGitHubResponseComment(
|
||||
|
||||
export function githubResponseCommentService(db: Db, fetchImpl = fetch) {
|
||||
async function context(deliveryId: string) {
|
||||
const [row] = await db.select({ endpoint: chatEndpoints, conversation: chatConversations,
|
||||
delivery: chatDeliveries, resource: chatEndpointResources, config: chatGitHubConfigurations,
|
||||
const [row] = await db.select({ ...githubResponseAuthorityFields,
|
||||
assignee: issues.assigneeAgentId, issueStatus: issues.status, sourceMessageId: chatMessageLinks.commentId }).from(chatMessageLinks)
|
||||
.innerJoin(chatDeliveries, and(eq(chatDeliveries.id, chatMessageLinks.deliveryId), eq(chatDeliveries.companyId, chatMessageLinks.companyId)))
|
||||
.innerJoin(chatConversations, and(eq(chatConversations.id, chatMessageLinks.conversationId), eq(chatConversations.companyId, chatMessageLinks.companyId)))
|
||||
@@ -170,13 +170,13 @@ export function githubResponseCommentService(db: Db, fetchImpl = fetch) {
|
||||
.innerJoin(issues, and(eq(issues.id, chatConversations.issueId), eq(issues.companyId, chatEndpoints.companyId)))
|
||||
.where(and(eq(chatMessageLinks.deliveryId, deliveryId), eq(chatMessageLinks.direction, "inbound"), eq(chatEndpoints.provider, "github")));
|
||||
if (!row || !row.delivery.principalId || !["processing", "processed"].includes(row.delivery.state) ||
|
||||
!["active", "verifying"].includes(row.endpoint.status) || !row.config.configuration.toolsEnabled ||
|
||||
!["active", "verifying"].includes(row.endpoint.status) || !row.toolsEnabled ||
|
||||
!row.resource.enabled || row.resource.availability !== "available" || row.assignee !== row.endpoint.assignedAgentId)
|
||||
throw forbidden("GitHub request is no longer authorized");
|
||||
const access = await githubChatPrincipalAccess(db, row.endpoint, row.delivery.principalId);
|
||||
if (!access?.allowed) throw forbidden("The initiating GitHub person is no longer authorized");
|
||||
const automatic = (row.delivery.normalizedEvent.githubAutomatic ?? row.delivery.normalizedEvent.githubIssue) as
|
||||
{ context: GitHubAutomaticEventContext } | undefined;
|
||||
{ context: GitHubAutomaticPolicyContext } | undefined;
|
||||
if (automatic && !(await githubAutomaticAdmission(db, row.endpoint, automatic.context))?.allowed)
|
||||
throw forbidden("Automatic GitHub task authority changed");
|
||||
const match = /^github:([^:]+):(?:issue:)?([1-9][0-9]*)(?::rc:([1-9][0-9]*))?$/.exec(row.conversation.externalThreadId);
|
||||
|
||||
@@ -8,9 +8,14 @@ import {
|
||||
type GitHubReviewConclusion,
|
||||
type GitHubReviewEventContext,
|
||||
type GitHubAutomaticEventContext,
|
||||
type GitHubIssueEventContext,
|
||||
type GitHubReviewPolicy,
|
||||
} from "@paperclipai/shared";
|
||||
|
||||
export type GitHubAutomaticPolicyContext =
|
||||
| Pick<GitHubReviewEventContext, "event" | "repositoryId" | "repository" | "author" | "sender" | "draft" | "baseBranch" | "labels">
|
||||
| Pick<GitHubIssueEventContext, "event" | "repositoryId" | "repository" | "author" | "sender" | "labels">;
|
||||
|
||||
export function effectiveGitHubReviewPolicy(
|
||||
configuration: GitHubChatConfiguration,
|
||||
repositoryId: string,
|
||||
@@ -85,7 +90,7 @@ export function githubReviewLineIsInPatch(
|
||||
|
||||
export function githubReviewSchedulingDecision(input: {
|
||||
configuration: GitHubChatConfiguration;
|
||||
context: GitHubAutomaticEventContext;
|
||||
context: GitHubAutomaticPolicyContext;
|
||||
repositoryEnabled: boolean;
|
||||
/** Resolved against current company membership and the user's confirmed link. */
|
||||
linkedMemberUserId: string | null;
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { createDb, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { githubReviewCheckService } from "./chat-github-checks.js";
|
||||
|
||||
type QueryParameters = NonNullable<Parameters<Db["$client"]["unsafe"]>[1]>;
|
||||
type CapturedQuery = { text: string; parameters: QueryParameters };
|
||||
type Plan = { "Node Type"?: string; "Relation Name"?: string; "Index Name"?: string; "Actual Loops"?: number; Plans?: Plan[] };
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("GitHub stale review repair PostgreSQL behavior", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let queries: CapturedQuery[];
|
||||
const noProviderCall = vi.fn(async () => { throw new Error("Repair must not contact a provider"); });
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("github-review-repair-");
|
||||
const setup = createDb(temporary.connectionString, { maxConnections: 1 });
|
||||
try {
|
||||
await setup.execute(sql`create schema github_review_repair_fixture`);
|
||||
await setup.execute(sql`set search_path=github_review_repair_fixture`);
|
||||
await setup.execute(sql`create table chat_github_reviews (id uuid primary key, company_id uuid not null,
|
||||
endpoint_id uuid not null, delivery_id text not null, state text not null, assessment jsonb,
|
||||
updated_at timestamptz not null default now())`);
|
||||
await setup.execute(sql`create table chat_deliveries (id uuid primary key, endpoint_id uuid not null, normalized_event jsonb)`);
|
||||
await setup.execute(sql`create table chat_actions (id uuid primary key, company_id uuid not null,
|
||||
endpoint_id uuid not null, provider_action_id text, kind text not null, status text not null,
|
||||
payload jsonb not null, result jsonb, updated_at timestamptz not null default now())`);
|
||||
await setup.execute(sql`create unique index chat_actions_provider_action_uq
|
||||
on chat_actions(company_id, endpoint_id, provider_action_id)`);
|
||||
} finally {
|
||||
await setup.$client.end();
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
queries = [];
|
||||
db = createDb(temporary.connectionString, { maxConnections: 1 });
|
||||
db.$client.options.debug = (_connection: number, text: string, parameters: QueryParameters) => {
|
||||
if (/^update "chat_github_reviews"/i.test(text)) queries.push({ text, parameters: [...parameters] });
|
||||
};
|
||||
await db.execute(sql`set search_path=github_review_repair_fixture`);
|
||||
await db.execute(sql`truncate chat_github_reviews, chat_deliveries, chat_actions`);
|
||||
noProviderCall.mockClear();
|
||||
});
|
||||
afterEach(async () => { await db?.$client.end(); });
|
||||
afterAll(async () => { await temporary?.cleanup(); });
|
||||
|
||||
async function seed(overrides: Record<string, unknown> = {}) {
|
||||
const companyId = randomUUID(), endpointId = randomUUID();
|
||||
const deliveryId = "a" + randomUUID().slice(1), reviewId = randomUUID(), actionId = randomUUID();
|
||||
const providerDeliveryId = "provider-" + randomUUID();
|
||||
const values = { companyId, endpointId, deliveryId, reviewId, actionId, providerDeliveryId,
|
||||
actionCompanyId: companyId, actionEndpointId: endpointId, deliveryEndpointId: endpointId,
|
||||
actionDeliveryId: providerDeliveryId, reviewDeliveryId: deliveryId, state: "queued", assessment: null,
|
||||
kind: "github_review_check", status: "cancelled", code: "stale_head", ...overrides };
|
||||
await db.execute(sql`insert into chat_deliveries(id, endpoint_id, normalized_event)
|
||||
values (${deliveryId}::uuid, ${values.deliveryEndpointId}::uuid,
|
||||
${JSON.stringify({ githubAutomatic: { context: { deliveryId: providerDeliveryId } } })}::jsonb)`);
|
||||
await db.execute(sql`insert into chat_actions(id, company_id, endpoint_id, kind, status, payload, result)
|
||||
values (${actionId}::uuid, ${values.actionCompanyId}::uuid, ${values.actionEndpointId}::uuid,
|
||||
${values.kind}, ${values.status}, ${JSON.stringify({ event: { deliveryId: values.actionDeliveryId } })}::jsonb,
|
||||
${JSON.stringify({ code: values.code })}::jsonb)`);
|
||||
await db.execute(sql`insert into chat_github_reviews(id, company_id, endpoint_id, delivery_id, state, assessment, updated_at)
|
||||
values (${reviewId}::uuid, ${companyId}::uuid, ${endpointId}::uuid, ${values.reviewDeliveryId},
|
||||
${values.state}, ${values.assessment === null ? null : JSON.stringify(values.assessment)}::jsonb, '2000-01-01T00:00:00Z')`);
|
||||
return values;
|
||||
}
|
||||
|
||||
async function retainedRows() {
|
||||
return {
|
||||
reviews: await db.$client.unsafe("select * from chat_github_reviews order by id"),
|
||||
deliveries: await db.$client.unsafe("select * from chat_deliveries order by id"),
|
||||
actions: await db.$client.unsafe("select * from chat_actions order by id"),
|
||||
};
|
||||
}
|
||||
|
||||
it.each(["queued", "running"])("supersedes only the exact %s unassessed review", async state => {
|
||||
const fixture = await seed({ state });
|
||||
const before = await retainedRows();
|
||||
const service = githubReviewCheckService(db, noProviderCall);
|
||||
await service.processPending();
|
||||
const after = await retainedRows();
|
||||
expect(after.reviews).toHaveLength(1);
|
||||
expect(after.reviews[0]).toMatchObject({ id: fixture.reviewId, state: "superseded", assessment: null });
|
||||
expect(after.deliveries).toEqual(before.deliveries);
|
||||
expect(after.actions).toEqual(before.actions);
|
||||
await service.processPending();
|
||||
expect(await retainedRows()).toEqual(after);
|
||||
expect(noProviderCall).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["another action company", { actionCompanyId: randomUUID() }],
|
||||
["another action endpoint", { actionEndpointId: randomUUID() }],
|
||||
["another delivery endpoint", { deliveryEndpointId: randomUUID() }],
|
||||
["another provider delivery", { actionDeliveryId: "another-delivery" }],
|
||||
["another cancellation reason", { code: "another_reason" }],
|
||||
["another action kind", { kind: "another_kind" }],
|
||||
["a processed action", { status: "processed" }],
|
||||
["an assessed review", { assessment: { retained: true } }],
|
||||
["a completed review", { state: "completed" }],
|
||||
])("preserves rows for %s", async (_name, overrides) => {
|
||||
await seed(overrides as Record<string, unknown>);
|
||||
const before = await retainedRows();
|
||||
await githubReviewCheckService(db, noProviderCall).processPending();
|
||||
expect(await retainedRows()).toEqual(before);
|
||||
expect(noProviderCall).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["malformed", "uppercase", "whitespace", "braces", "unhyphenated"])("preserves %s delivery references without cast errors", async kind => {
|
||||
const fixture = await seed();
|
||||
const reference = kind === "malformed" ? "not-a-uuid" : kind === "uppercase" ? fixture.deliveryId.toUpperCase()
|
||||
: kind === "whitespace" ? ` ${fixture.deliveryId} ` : kind === "braces" ? `{${fixture.deliveryId}}`
|
||||
: fixture.deliveryId.replaceAll("-", "");
|
||||
await db.execute(sql`update chat_github_reviews set delivery_id=${reference} where id=${fixture.reviewId}::uuid`);
|
||||
const before = await retainedRows();
|
||||
await githubReviewCheckService(db, noProviderCall).processPending();
|
||||
expect(await retainedRows()).toEqual(before);
|
||||
expect(noProviderCall).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["missing delivery", "missing provider identity"])("preserves a review with %s", async kind => {
|
||||
const fixture = await seed();
|
||||
if (kind === "missing delivery") {
|
||||
await db.execute(sql`delete from chat_deliveries where id=${fixture.deliveryId}::uuid`);
|
||||
} else {
|
||||
await db.execute(sql`update chat_deliveries set normalized_event='{}'::jsonb where id=${fixture.deliveryId}::uuid`);
|
||||
}
|
||||
const before = await retainedRows();
|
||||
await githubReviewCheckService(db, noProviderCall).processPending();
|
||||
expect(await retainedRows()).toEqual(before);
|
||||
expect(noProviderCall).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("repairs a late review from the retained cancellation without replaying provider work", async () => {
|
||||
const fixture = await seed();
|
||||
await db.execute(sql`delete from chat_github_reviews where id=${fixture.reviewId}::uuid`);
|
||||
const service = githubReviewCheckService(db, noProviderCall);
|
||||
await service.processPending();
|
||||
await db.execute(sql`insert into chat_github_reviews(id, company_id, endpoint_id, delivery_id, state)
|
||||
values (${fixture.reviewId}::uuid, ${fixture.companyId}::uuid, ${fixture.endpointId}::uuid, ${fixture.deliveryId}, 'queued')`);
|
||||
await service.processPending();
|
||||
const rows = await db.$client.unsafe("select state from chat_github_reviews");
|
||||
expect(rows).toEqual([{ state: "superseded" }]);
|
||||
expect(noProviderCall).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resolves each candidate delivery once through its primary key with retained history", async () => {
|
||||
const companyId = randomUUID(), endpointId = randomUUID();
|
||||
const candidates = 200;
|
||||
await db.execute(sql`insert into chat_deliveries(id, endpoint_id, normalized_event)
|
||||
select gen_random_uuid(), ${endpointId}::uuid,
|
||||
jsonb_build_object('githubAutomatic', jsonb_build_object('context', jsonb_build_object('deliveryId', 'delivery-' || n)))
|
||||
from generate_series(1,8000) n`);
|
||||
await db.execute(sql`insert into chat_github_reviews(id, company_id, endpoint_id, delivery_id, state)
|
||||
select gen_random_uuid(), ${companyId}::uuid, ${endpointId}::uuid, id::text, 'queued'
|
||||
from chat_deliveries order by id limit ${candidates}`);
|
||||
await db.execute(sql`insert into chat_actions(id, company_id, endpoint_id, provider_action_id, kind, status, payload, result)
|
||||
select gen_random_uuid(), ${companyId}::uuid, ${endpointId}::uuid, 'action-' || n,
|
||||
case when n<=100 then 'github_review_check' else 'another_kind' end, 'cancelled',
|
||||
jsonb_build_object('event', jsonb_build_object('deliveryId', 'unmatched-' || n)),
|
||||
jsonb_build_object('code', 'stale_head') from generate_series(1,1100) n`);
|
||||
await db.execute(sql`analyze chat_deliveries`);
|
||||
await db.execute(sql`analyze chat_actions`);
|
||||
await db.execute(sql`analyze chat_github_reviews`);
|
||||
await db.execute(sql`set statement_timeout='5s'`);
|
||||
await githubReviewCheckService(db, noProviderCall).processPending();
|
||||
expect(queries).toHaveLength(1);
|
||||
const query = queries[0];
|
||||
const explained = await db.$client.unsafe(`explain (analyze,buffers,format json) ${query.text}`, query.parameters);
|
||||
const root = (explained[0]["QUERY PLAN"] as Array<{ Plan: Plan }>)[0].Plan;
|
||||
function flatten(plan: Plan): Plan[] { return [plan, ...(plan.Plans ?? []).flatMap(flatten)]; }
|
||||
const deliveries = flatten(root).filter(node => node["Relation Name"] === "chat_deliveries");
|
||||
expect(deliveries).toHaveLength(1);
|
||||
expect(deliveries[0]["Index Name"]).toBe("chat_deliveries_pkey");
|
||||
expect(deliveries[0]["Actual Loops"]).toBeLessThanOrEqual(candidates);
|
||||
expect((await db.$client.unsafe("select count(*)::int as count from chat_github_reviews where state='queued'"))[0].count).toBe(candidates);
|
||||
expect(noProviderCall).not.toHaveBeenCalled();
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
type PaperclipCloudConnector,
|
||||
} from "./paperclip-cloud-connector.js";
|
||||
import { paperclipCloudConnectorEnrollmentStatus } from "./paperclip-cloud-connector-enrollment.js";
|
||||
import { pollGitHubBotCloudSetupEvents } from "./chat-github-cloud-ingress.js";
|
||||
|
||||
/** Loopback aliases are equivalent locally, but Cloud requires the approved exact origin. */
|
||||
export function githubWizardBrowserOrigin(
|
||||
@@ -920,6 +921,10 @@ export function githubChatWizardService(
|
||||
// Keep the same vaulted App in the wizard instead of throwing JSON from
|
||||
// the browser callback or relaxing the configure action's signature gate.
|
||||
bot = await endpoint(id, userId);
|
||||
if (!bot.setup.webhookVerifiedAt && bot.setup.github?.cloudRegistrationId) {
|
||||
await pollGitHubBotCloudSetupEvents(db);
|
||||
bot = await endpoint(id, userId);
|
||||
}
|
||||
if (!bot.setup.webhookVerifiedAt)
|
||||
return {
|
||||
endpointId: id,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { ensureSkillCacheDirectory, touchSkillVersionCache, withSkillVersionCacheLock } from "./skill-version-cache.js";
|
||||
import { verifyExistingBundledSkillRelease } from "./bundled-skill-release-validation.js";
|
||||
import { retainProjectSkillAccess, skillAccessService } from "./skill-access.js";
|
||||
import type { AuthorizationActor } from "./authorization.js";
|
||||
import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js";
|
||||
@@ -3094,30 +3095,22 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
|
||||
const fileInventory = serializeVersionFileInventory(
|
||||
await collectVersionFileInventoryFromDirectory(release.releaseDir),
|
||||
);
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(companySkillVersions)
|
||||
.where(and(
|
||||
eq(companySkillVersions.companyId, companyId),
|
||||
eq(companySkillVersions.companySkillId, paperclipSkill.id),
|
||||
eq(companySkillVersions.releaseId, release.id),
|
||||
))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (existing) {
|
||||
const existingInventory = toCompanySkillVersion(existing).fileInventory;
|
||||
const existingHash = buildInventoryContentHash(existingInventory.map((entry) => ({
|
||||
path: entry.path,
|
||||
sha256: sha256Buffer(entry.content),
|
||||
})));
|
||||
const registryHash = buildInventoryContentHash(fileInventory.map((entry) => ({
|
||||
path: entry.path,
|
||||
sha256: sha256Buffer(entry.content),
|
||||
})));
|
||||
if (existingHash !== registryHash) {
|
||||
throw new Error(`Bundled skill release ${release.id} does not match its seeded snapshot.`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const registryHash = buildInventoryContentHash(fileInventory.map((entry) => ({
|
||||
path: entry.path,
|
||||
sha256: sha256Buffer(entry.content),
|
||||
})));
|
||||
if (await verifyExistingBundledSkillRelease({ db, companyId, skillId: paperclipSkill.id, releaseId: release.id,
|
||||
expectedInventoryHash: registryHash, validateInventory: existing => {
|
||||
const existingInventory = toCompanySkillVersion(existing).fileInventory;
|
||||
const existingHash = buildInventoryContentHash(existingInventory.map((entry) => ({
|
||||
path: entry.path,
|
||||
sha256: sha256Buffer(entry.content),
|
||||
})));
|
||||
if (existingHash !== registryHash) {
|
||||
throw new Error(`Bundled skill release ${release.id} does not match its seeded snapshot.`);
|
||||
}
|
||||
},
|
||||
})) continue;
|
||||
const releasedAt = new Date(release.releasedAt);
|
||||
if (Number.isNaN(releasedAt.getTime())) {
|
||||
throw new Error(`Invalid bundled skill release date: ${release.releasedAt}`);
|
||||
|
||||
@@ -291,9 +291,9 @@ export function connectionIntentService(db: Db) {
|
||||
&& connection.status !== "archived" && connection.connectionPurpose !== "ai"
|
||||
);
|
||||
if (matching.length === 0) return null;
|
||||
const effective = await access.getEffectiveToolAssignmentsForAgent(input.companyId, input.agentId);
|
||||
const effective = await access.getConnectionAssignmentsForAgent(input.companyId, input.agentId, matching.map(connection => connection.id));
|
||||
const installedIds = new Set(effective.installedConnections.map((connection) => connection.id));
|
||||
const permittedIds = new Set(effective.allowedTools.map((tool) => tool.connectionId));
|
||||
const permittedIds = new Set(effective.allowedConnectionIds);
|
||||
const usable = (connection: ToolConnection | undefined) => connection
|
||||
&& installedIds.has(connection.id) && permittedIds.has(connection.id)
|
||||
&& connection.status === "active" && connection.enabled
|
||||
|
||||
@@ -17,7 +17,7 @@ import { unprocessable } from "../errors.js";
|
||||
import { parseObject } from "../adapters/utils.js";
|
||||
import { secretService } from "./secrets.js";
|
||||
import {
|
||||
resolvePluginSandboxProviderDriverByKey,
|
||||
resolvePluginSandboxProviderConfigSchema,
|
||||
validatePluginEnvironmentDriverConfig,
|
||||
validatePluginSandboxProviderConfig,
|
||||
} from "./plugin-environment-driver.js";
|
||||
@@ -179,14 +179,10 @@ async function getSandboxProviderConfigSchema(
|
||||
db: Db,
|
||||
provider: string,
|
||||
): Promise<Record<string, unknown> | null> {
|
||||
const resolved = await resolvePluginSandboxProviderDriverByKey({
|
||||
return resolvePluginSandboxProviderConfigSchema({
|
||||
db,
|
||||
driverKey: provider,
|
||||
});
|
||||
const schema = resolved?.driver.configSchema;
|
||||
return schema && typeof schema === "object" && !Array.isArray(schema)
|
||||
? schema as Record<string, unknown>
|
||||
: null;
|
||||
}
|
||||
|
||||
export async function resolveSandboxProviderSecretRefPaths(
|
||||
|
||||
@@ -100,7 +100,9 @@ describe.skipIf(!support.supported)("execution blocker PostgreSQL read projectio
|
||||
});
|
||||
const input = { schema: "paperclip.native-execution-input.v6",
|
||||
binding: { runId: f.run.id, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId },
|
||||
provider: { kind: "grok_bot" }, workspace: { access: "none", ...(hasCwd ? { cwd: null } : {}) } };
|
||||
provider: { kind: "grok_bot" }, workspace: { access: "none", ...(hasCwd ? { cwd: null } : {}) },
|
||||
task: { description: randomBytes(128 * 1024).toString("base64") },
|
||||
runtimeContext: { instructions: randomBytes(128 * 1024).toString("base64") } };
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: f.issueId,
|
||||
runnerProfileJson: { ...f.run.runnerProfileJson, nativeExecutionInput: input } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await compare(f)).toMatchObject({ workspaceRepairRequired: !hasCwd, canRetry: false });
|
||||
@@ -108,6 +110,39 @@ describe.skipIf(!support.supported)("execution blocker PostgreSQL read projectio
|
||||
expect(stored.runnerProfileJson?.nativeExecutionInput).toEqual(input);
|
||||
});
|
||||
|
||||
it.each(["run", "company", "agent", "issue", "provider", "schema", "access", "workspace", "binding", "missing_binding",
|
||||
"null_input", "array_input", "scalar_input", "number_input"])(
|
||||
"does not relax native workspace recovery for invalid input (%s)", async kind => {
|
||||
const f = await fixture({ runtimeMode: "native", driverKind: "grok_bot_mcp" }, {
|
||||
workspaceRestoreFailure: "restore_unsafe_archive",
|
||||
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [] },
|
||||
});
|
||||
const input: Record<string, unknown> = {
|
||||
schema: "paperclip.native-execution-input.v6",
|
||||
binding: { runId: f.run.id, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId },
|
||||
provider: { kind: "grok_bot" }, workspace: { access: "none", cwd: null },
|
||||
task: { description: randomBytes(128 * 1024).toString("base64") },
|
||||
};
|
||||
const binding = input.binding as Record<string, unknown>;
|
||||
if (kind === "run") binding.runId = randomUUID();
|
||||
if (kind === "company") binding.companyId = randomUUID();
|
||||
if (kind === "agent") binding.agentId = randomUUID();
|
||||
if (kind === "issue") binding.issueId = randomUUID();
|
||||
if (kind === "provider") input.provider = { kind: "codex" };
|
||||
if (kind === "schema") input.schema = "unsupported";
|
||||
if (kind === "access") input.workspace = { cwd: null };
|
||||
if (kind === "workspace") input.workspace = "none";
|
||||
if (kind === "binding") input.binding = [];
|
||||
if (kind === "missing_binding") delete input.binding;
|
||||
const selectedInput = kind === "null_input" ? null : kind === "array_input" ? []
|
||||
: kind === "scalar_input" ? "input" : kind === "number_input" ? 42 : input;
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: f.issueId,
|
||||
runnerProfileJson: { ...f.run.runnerProfileJson, nativeExecutionInput: selectedInput } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await compare(f)).toMatchObject({ workspaceRepairRequired: true, canRetry: false });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].runnerProfileJson?.nativeExecutionInput).toEqual(selectedInput);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([null, [], "scalar", 42, { issueId: 42, taskId: false }].map(context => ({ context })))("preserves decisions for malformed context $context", async ({ context }) => {
|
||||
const f = await fixture({ contextSnapshot: context as Record<string, unknown> | null });
|
||||
expect(await compare(f)).toMatchObject({ canRetry: true });
|
||||
|
||||
@@ -28,6 +28,11 @@ const blockerProfileColumn = sql<Record<string, unknown> | null>`case
|
||||
when ${heartbeatRuns.runnerProfileJson} is null then null else (
|
||||
select coalesce(jsonb_object_agg(field.key, case field.key
|
||||
when 'adapterDispatch' then jsonb_build_object('adapterType', field.value->'adapterType')
|
||||
when 'nativeExecutionInput' then case when jsonb_typeof(field.value) = 'object' then (
|
||||
select coalesce(jsonb_object_agg(input_field.key, input_field.value), '{}'::jsonb)
|
||||
from jsonb_each(field.value) as input_field(key, value)
|
||||
where input_field.key in ('schema', 'binding', 'provider', 'workspace')
|
||||
) else field.value end
|
||||
else field.value end), '{}'::jsonb)
|
||||
from jsonb_each(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object'
|
||||
then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) as field(key, value)
|
||||
|
||||
@@ -98,7 +98,7 @@ import {
|
||||
type PullRequestMergeDetailsResolver,
|
||||
} from "./github-pull-request-merge.js";
|
||||
import { visibleIssueCondition } from "./issue-visibility.js";
|
||||
import { createGitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { createIssueGitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js";
|
||||
import { workspaceGitOperationScheduler } from "./workspace-git-operation-scheduler.js";
|
||||
import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.js";
|
||||
@@ -4113,7 +4113,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
try { recipe = readGitCacheRestoration(snapshot); }
|
||||
catch { return { ok: false, code: "rebuild_failed", message: new GitCacheRestorationFailure().message }; }
|
||||
const freshStart = recipe !== null && isClosedExecutionWorkspaceStatus(snapshot.status) && gitRestorePointExpired(recipe);
|
||||
const gitAuth = createGitRemoteAuthProvider(db, snapshot.companyId, { issueId: issue.id });
|
||||
const gitAuth = createIssueGitRemoteAuthProvider(db, snapshot.companyId, issue.id);
|
||||
const [
|
||||
{ ensurePersistedExecutionWorkspaceAvailable },
|
||||
{ workspaceOperationService },
|
||||
@@ -4168,9 +4168,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
});
|
||||
};
|
||||
},
|
||||
resolveGitAuth: createGitRemoteAuthProvider(db, snapshot.companyId, {
|
||||
issueId: issue.id,
|
||||
}),
|
||||
resolveGitAuth: createIssueGitRemoteAuthProvider(db, snapshot.companyId, issue.id),
|
||||
});
|
||||
resolvedBaseCwd = managedWorkspace.cwd;
|
||||
} catch {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
companySecrets,
|
||||
heartbeatRuns,
|
||||
issues,
|
||||
companyMemberships,
|
||||
connectionGrantDelegations,
|
||||
connectionGrants,
|
||||
@@ -9,7 +10,7 @@ import {
|
||||
userSecretDefinitions,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { and, eq, inArray, or } from "drizzle-orm";
|
||||
import { and, eq, inArray, or, sql } from "drizzle-orm";
|
||||
import { isGitHubDotCom } from "./github-fetch.js";
|
||||
import { secretService } from "./secrets.js";
|
||||
import { toolAccessService } from "./tool-access.js";
|
||||
@@ -207,6 +208,19 @@ type GitCredentialSecretsDeps = {
|
||||
* provider instance so one run performs at most one secret resolution (and writes at most
|
||||
* one audit event) no matter how many git operations it authenticates.
|
||||
*/
|
||||
export function createIssueGitRemoteAuthProvider(db: Db, companyId: string, issueId: string,
|
||||
create: typeof createGitRemoteAuthProvider = createGitRemoteAuthProvider): GitRemoteAuthProvider {
|
||||
let provider: Promise<GitRemoteAuthProvider> | null = null;
|
||||
return async (remoteUrl: string) => {
|
||||
provider ??= db.select({ responsibleUserId: issues.responsibleUserId, assigneeAgentId: issues.assigneeAgentId }).from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)))
|
||||
.then(([issue]) => create(db, companyId, {
|
||||
issueId, responsibleUserId: issue?.responsibleUserId ?? null, agentId: issue?.assigneeAgentId ?? null,
|
||||
}));
|
||||
return (await provider)(remoteUrl);
|
||||
};
|
||||
}
|
||||
|
||||
export function createGitRemoteAuthProvider(
|
||||
db: Db,
|
||||
companyId: string,
|
||||
@@ -310,6 +324,10 @@ export async function resolveManagedGitHubIdentitySelection(
|
||||
}> {
|
||||
const connections = await db.select().from(toolConnections).where(and(
|
||||
eq(toolConnections.companyId, companyId),
|
||||
or(
|
||||
sql`${toolConnections.config}->'sourceTemplateKey' = '"github"'::jsonb`,
|
||||
sql`${toolConnections.transportConfig}->'sourceTemplateKey' = '"github"'::jsonb`,
|
||||
),
|
||||
));
|
||||
const githubConnections = connections.filter((connection) => {
|
||||
const config = connection.config && typeof connection.config === "object" ? connection.config as Record<string, unknown> : {};
|
||||
|
||||
@@ -595,9 +595,21 @@ export async function restorePointHistoryDigest(input: string, signal?: AbortSig
|
||||
const cwd = await fs.realpath(input);
|
||||
const reflog = await gitText(cwd, ["reflog", "show", "--format=%H", "HEAD"], { signal }).catch(() => "");
|
||||
const original = await gitText(cwd, ["rev-parse", "--verify", "-q", "ORIG_HEAD^{commit}"], { signal }).catch(() => "");
|
||||
const remotes = await gitText(cwd, ["for-each-ref", "--format=%(objectname) %(refname)", "refs/remotes"], { signal }).catch(() => "");
|
||||
const remoteReflogs = (await gitText(cwd, ["reflog", "show", "--all", "--format=%H %gD"], { signal }).catch(() => ""))
|
||||
.split("\n").filter((line) => / refs\/remotes\//.test(line));
|
||||
const linked = (await fs.lstat(path.join(cwd, ".git")).catch(() => null))?.isFile() ?? false;
|
||||
const nested = linked ? await import("./nested-repository-worktrees.js") : null;
|
||||
const scope = nested ? `refs/heads/${await nested.nestedWorktreeBranchPrefix(cwd)}` : "refs/remotes";
|
||||
const remotes = await gitText(cwd, ["for-each-ref", ...nested ? [`--count=${nested.OWNED_NESTED_REF_LIMIT + 1}`] : [],
|
||||
"--format=%(objectname) %(refname)", scope], { signal }).catch(() => "");
|
||||
const remoteReflogs: string[] = [];
|
||||
if (nested) {
|
||||
if (remotes.split("\n").filter(Boolean).length > nested.OWNED_NESTED_REF_LIMIT) throw new Error("managed_checkout_local_tip_budget");
|
||||
for (const ref of remotes.split("\n").map((line) => line.split(" ")[1]).filter(Boolean)) {
|
||||
remoteReflogs.push(...(await gitText(cwd, ["reflog", "show", "--format=%H", ref!], { signal }).catch(() => "")).split("\n").filter((line) => SHA.test(line)).map((sha) => `${sha} ${ref}`));
|
||||
}
|
||||
} else {
|
||||
remoteReflogs.push(...(await gitText(cwd, ["reflog", "show", "--all", "--format=%H %gD"], { signal }).catch(() => ""))
|
||||
.split("\n").filter((line) => / refs\/remotes\//.test(line)));
|
||||
}
|
||||
return createHash("sha256").update(JSON.stringify([reflog.split("\n").filter((line) => SHA.test(line)), SHA.test(original) ? original : null,
|
||||
remotes.split("\n").filter(Boolean), remoteReflogs])).digest("hex");
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { acquireGitExpirationLocks } from "./git-workspace-expiration-locks.js";
|
||||
import { and, eq, inArray, ne, sql } from "drizzle-orm";
|
||||
import { executionWorkspaces, issues, heartbeatRuns, nativeRunFinalizations, environmentLeases,
|
||||
workspaceRuntimeServices, issueRecoveryActions, projectWorkspaces, withDedicatedDbConnection, type Db } from "@paperclipai/db";
|
||||
import { createGitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { createIssueGitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { TASK_DIRECTORY_RETENTION_KEY } from "./isolated-task-directory-snapshot.js";
|
||||
import { currentRestorePointSnapshot, RestorePointSkip, type RestorePointLimits } from "./git-restore-point.js";
|
||||
@@ -37,7 +37,7 @@ export function withAnonymousGitFallback(provider: GitRemoteAuthProvider): GitRe
|
||||
}
|
||||
|
||||
function expirationGitAuth(db: Db, workspace: Workspace) {
|
||||
return withAnonymousGitFallback(createGitRemoteAuthProvider(db, workspace.companyId, { issueId: workspace.sourceIssueId! }));
|
||||
return withAnonymousGitFallback(createIssueGitRemoteAuthProvider(db, workspace.companyId, workspace.sourceIssueId!));
|
||||
}
|
||||
|
||||
const EXPIRATION_KEY = "gitCacheExpiration";
|
||||
|
||||
@@ -18,7 +18,7 @@ import { ensureGitRestorePointCommit, validateGitRestorePointRef, type GitRestor
|
||||
import { getStorageService } from "../storage/index.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync";
|
||||
import { forgetExpiredNestedRepositoryWorktree, nestedRepositoryBaseCandidates, readLinkedNestedCheckout, recordedNestedRepositoryBase, type LinkedNestedCheckout } from "./nested-repository-worktrees.js";
|
||||
import { OWNED_NESTED_REF_LIMIT, forgetExpiredNestedRepositoryWorktree, nestedRepositoryBaseCandidates, readLinkedNestedCheckout, recordedNestedRepositoryBase, type LinkedNestedCheckout } from "./nested-repository-worktrees.js";
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
const shaPattern = /^[a-f0-9]{40}$/;
|
||||
@@ -249,7 +249,6 @@ async function assertCommitMessageSaved(cwd: string, gitDir: string, signal?: Ab
|
||||
if (!saved.includes(message.trim())) throw new Error("uncommitted_git_message");
|
||||
}
|
||||
|
||||
const OWNED_NESTED_REF_LIMIT = 64;
|
||||
|
||||
async function ownedNestedRefs(linked: LinkedNestedCheckout, signal?: AbortSignal) {
|
||||
const refs = (await gitCacheCommand(linked.checkout, ["for-each-ref", `--count=${OWNED_NESTED_REF_LIMIT + 1}`,
|
||||
@@ -380,9 +379,13 @@ async function provenNestedWorktreeHistory(linked: LinkedNestedCheckout, repoUrl
|
||||
const covered = new Set([head,
|
||||
...shas(await gitCacheCommand(checkout, ["reflog", "show", "--format=%H", "HEAD"], signal).catch(() => "")),
|
||||
...shas(await gitCacheCommand(checkout, ["rev-parse", "--verify", "-q", "ORIG_HEAD^{commit}"], signal).catch(() => ""))]);
|
||||
const owned = await ownedNestedRefs(linked, signal);
|
||||
const allOwned = await ownedNestedRefs(linked, signal);
|
||||
const remotePrefix = `refs/heads/${linked.branchPrefix}remotes/`;
|
||||
const owned = allOwned.filter(([, ref]) => !ref.startsWith(remotePrefix));
|
||||
const remoteTracked = [...new Set([...allOwned.filter(([, ref]) => ref.startsWith(remotePrefix)).map(([sha]) => sha),
|
||||
...await ownedNestedReflogs(linked, allOwned.filter(([, ref]) => ref.startsWith(remotePrefix)), signal)])];
|
||||
const recorded = new Set([...owned.map(([sha]) => sha), ...await ownedNestedReflogs(linked, owned, signal)]);
|
||||
if (recorded.size + covered.size > MANAGED_CHECKOUT_LOCAL_TIP_LIMIT) throw new Error("managed_checkout_local_tip_budget");
|
||||
if (recorded.size + covered.size + remoteTracked.length > MANAGED_CHECKOUT_LOCAL_TIP_LIMIT) throw new Error("managed_checkout_local_tip_budget");
|
||||
const current = (await gitCacheCommand(checkout, ["symbolic-ref", "--quiet", "HEAD"], signal).catch(() => "")).trim();
|
||||
const otherTips = owned.filter(([, ref]) => ref !== current).map(([sha]) => sha);
|
||||
if (otherTips.length && await hasCommitsMissingUpstream(checkout, otherTips, repoUrl, resolveGitAuth, signal, [head])) {
|
||||
@@ -393,7 +396,9 @@ async function provenNestedWorktreeHistory(linked: LinkedNestedCheckout, repoUrl
|
||||
throw new Error("unpushed_managed_checkout_branches");
|
||||
}
|
||||
const headOnRemote = !await hasCommitsMissingUpstream(checkout, [head], repoUrl, resolveGitAuth, signal);
|
||||
return { headOnRemote, preserve: [] };
|
||||
const preserve = remoteTracked.length
|
||||
? await tipsMissingUpstream(checkout, remoteTracked, repoUrl, resolveGitAuth, signal, [...covered, ...recorded]) : [];
|
||||
return { headOnRemote, preserve };
|
||||
}
|
||||
|
||||
async function tipsMissingUpstream(checkout: string, tips: string[], repoUrl: string,
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { chatEndpoints, type Db } from "@paperclipai/db";
|
||||
import { githubConnectionEventService } from "./github-connection-events.js";
|
||||
import type { PaperclipCloudConnector } from "./paperclip-cloud-connector.js";
|
||||
|
||||
function fixture() {
|
||||
let time = Date.parse("2026-09-04T12:00:00Z");
|
||||
const bindings = [{
|
||||
grant: { id: "grant", companyId: "company", kind: "user", subjectUserId: "owner", status: "active",
|
||||
providerTenant: { github: { installationIds: ["101"] } } },
|
||||
connection: { id: "connection", config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } } },
|
||||
}];
|
||||
const bots: Array<{ companyId: string }> = [];
|
||||
// Only SQL is stubbed. Backoff, lease admission and concurrent execution use
|
||||
// the real service; relational persistence has its own Postgres suite.
|
||||
const select = vi.fn(() => ({ from: (table: unknown) => table === chatEndpoints
|
||||
? { where: async () => bots }
|
||||
: { innerJoin: () => ({ where: async () => bindings }) } }));
|
||||
const leaseEvents = vi.fn<PaperclipCloudConnector["leaseEvents"]>().mockResolvedValue(null);
|
||||
const connector = { leaseEvents } as unknown as PaperclipCloudConnector;
|
||||
const service = githubConnectionEventService({ select } as unknown as Db, { connector, now: () => new Date(time) });
|
||||
return { service, leaseEvents, select, bots, advance: (ms: number) => { time += ms; } };
|
||||
}
|
||||
|
||||
describe("GitHub event polling during active App setup", () => {
|
||||
it("reaches Cloud during an old empty-queue backoff without expanding idle polling", async () => {
|
||||
const f = fixture();
|
||||
for (const delay of [0, 10_000, 20_000, 40_000, 80_000]) {
|
||||
f.advance(delay);
|
||||
await f.service.pollOnce();
|
||||
}
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(5);
|
||||
const queries = f.select.mock.calls.length;
|
||||
f.bots.push({ companyId: "company" });
|
||||
f.advance(5_000);
|
||||
for (let index = 0; index < 3; index++) await f.service.pollOnce();
|
||||
expect(f.select).toHaveBeenCalledTimes(queries);
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(5);
|
||||
await f.service.pollOnce({ forSetup: true });
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(6);
|
||||
const setupQueries = f.select.mock.calls.length;
|
||||
await f.service.pollOnce({ forSetup: true });
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(6);
|
||||
expect(f.select).toHaveBeenCalledTimes(setupQueries);
|
||||
f.advance(5_000);
|
||||
await f.service.pollOnce({ forSetup: true });
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(7);
|
||||
});
|
||||
|
||||
it("shares one lease between setup and scheduled callers while Cloud is slow", async () => {
|
||||
const f = fixture();
|
||||
let complete!: (value: null) => void;
|
||||
f.leaseEvents.mockImplementationOnce(() => new Promise(resolve => { complete = resolve; }));
|
||||
const scheduled = f.service.pollOnce();
|
||||
const setup = f.service.pollOnce({ forSetup: true });
|
||||
await vi.waitFor(() => expect(f.leaseEvents).toHaveBeenCalledTimes(1));
|
||||
expect(complete).toBeTypeOf("function");
|
||||
complete(null);
|
||||
await Promise.all([scheduled, setup]);
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("releases the shared attempt after an error and retries only after the setup throttle", async () => {
|
||||
const f = fixture();
|
||||
f.leaseEvents.mockRejectedValueOnce(new Error("Cloud unavailable"));
|
||||
await expect(f.service.pollOnce({ forSetup: true })).rejects.toThrow("Cloud unavailable");
|
||||
await f.service.pollOnce({ forSetup: true });
|
||||
await f.service.pollOnce();
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(1);
|
||||
f.advance(5_000);
|
||||
await f.service.pollOnce({ forSetup: true });
|
||||
expect(f.leaseEvents).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -221,6 +221,8 @@ export function githubConnectionEventService(
|
||||
const now = options.now ?? (() => new Date());
|
||||
let nextPollAt = 0;
|
||||
let emptyPolls = 0;
|
||||
let nextSetupPollAt = 0;
|
||||
let inFlight: Promise<GitHubConnectionEventPollResult> | null = null;
|
||||
|
||||
async function activeBindings() {
|
||||
const rows = await db.select({ grant: connectionGrants, connection: toolConnections })
|
||||
@@ -458,88 +460,101 @@ export function githubConnectionEventService(
|
||||
return "processed" as const;
|
||||
}
|
||||
|
||||
return {
|
||||
async pollOnce(): Promise<GitHubConnectionEventPollResult> {
|
||||
if (now().getTime() < nextPollAt) {
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
const bindings = await activeBindings();
|
||||
const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} <> 'archived'`));
|
||||
if (bindings.length === 0 && bots.length === 0) {
|
||||
nextPollAt = now().getTime() + 5_000;
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
const config = options.connector ? null : paperclipCloudConnectorConfigFromEnv(options.env);
|
||||
const connector = options.connector ?? (config ? createPaperclipCloudConnector({ config }) : null);
|
||||
if (!connector) {
|
||||
nextPollAt = now().getTime() + 5 * 60_000;
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
const first = bindings[0] ?? { subject: "github-bot-events", companyId: bots[0]!.companyId };
|
||||
const lease = await connector.leaseEvents({ subject: first.subject, companyId: first.companyId });
|
||||
if (!lease) {
|
||||
emptyPolls += 1;
|
||||
nextPollAt = now().getTime() + Math.min(bots.length ? 15_000 : 5 * 60_000, 5_000 * (2 ** Math.min(emptyPolls, 6)));
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
emptyPolls = 0;
|
||||
async function runPoll(): Promise<GitHubConnectionEventPollResult> {
|
||||
const bindings = await activeBindings();
|
||||
const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} <> 'archived'`));
|
||||
if (bindings.length === 0 && bots.length === 0) {
|
||||
nextPollAt = now().getTime() + 5_000;
|
||||
const result: GitHubConnectionEventPollResult = {
|
||||
leased: lease.events.length + (lease.failedEventIds?.length ?? 0),
|
||||
processed: 0,
|
||||
duplicate: 0,
|
||||
ignored: 0,
|
||||
failed: lease.failedEventIds?.length ?? 0,
|
||||
};
|
||||
const acknowledge: string[] = [];
|
||||
for (const event of lease.events) {
|
||||
if (event.payload.githubApp !== undefined) {
|
||||
try { await dispatchGitHubBotCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; }
|
||||
catch { result.failed += 1; }
|
||||
continue;
|
||||
}
|
||||
const matched = bindings.filter((binding) => event.bindingIds.includes(binding.id));
|
||||
if (matched.length === 0) {
|
||||
result.ignored += 1;
|
||||
acknowledge.push(event.id);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const companies = new Map<string, GitHubBinding[]>();
|
||||
for (const binding of matched) companies.set(binding.companyId, [...(companies.get(binding.companyId) ?? []), binding]);
|
||||
for (const [companyId, companyBindings] of companies) {
|
||||
const status = await processForCompany(companyId, companyBindings, event);
|
||||
result[status] += 1;
|
||||
}
|
||||
acknowledge.push(event.id);
|
||||
if (event.event === "installation" && (event.action === "deleted" || event.action === "suspend")) {
|
||||
await Promise.all(matched.map((binding) => connector.setWebhookBinding({
|
||||
subject: binding.subject,
|
||||
companyId: binding.companyId,
|
||||
id: binding.id,
|
||||
installationId: binding.installationId,
|
||||
connectionId: binding.connectionId,
|
||||
grantId: binding.grantId,
|
||||
active: false,
|
||||
})));
|
||||
}
|
||||
} catch {
|
||||
result.failed += 1;
|
||||
}
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
const config = options.connector ? null : paperclipCloudConnectorConfigFromEnv(options.env);
|
||||
const connector = options.connector ?? (config ? createPaperclipCloudConnector({ config }) : null);
|
||||
if (!connector) {
|
||||
nextPollAt = now().getTime() + 5 * 60_000;
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
const first = bindings[0] ?? { subject: "github-bot-events", companyId: bots[0]!.companyId };
|
||||
const lease = await connector.leaseEvents({ subject: first.subject, companyId: first.companyId });
|
||||
if (!lease) {
|
||||
emptyPolls += 1;
|
||||
nextPollAt = now().getTime() + Math.min(bots.length ? 15_000 : 5 * 60_000, 5_000 * (2 ** Math.min(emptyPolls, 6)));
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
emptyPolls = 0;
|
||||
nextPollAt = now().getTime() + 5_000;
|
||||
const result: GitHubConnectionEventPollResult = {
|
||||
leased: lease.events.length + (lease.failedEventIds?.length ?? 0),
|
||||
processed: 0,
|
||||
duplicate: 0,
|
||||
ignored: 0,
|
||||
failed: lease.failedEventIds?.length ?? 0,
|
||||
};
|
||||
const acknowledge: string[] = [];
|
||||
for (const event of lease.events) {
|
||||
if (event.payload.githubApp !== undefined) {
|
||||
try { await dispatchGitHubBotCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; }
|
||||
catch { result.failed += 1; }
|
||||
continue;
|
||||
}
|
||||
if (acknowledge.length > 0) {
|
||||
await connector.acknowledgeEvents({
|
||||
subject: first.subject,
|
||||
companyId: first.companyId,
|
||||
leaseId: lease.leaseId,
|
||||
deliveryIds: acknowledge,
|
||||
});
|
||||
const matched = bindings.filter((binding) => event.bindingIds.includes(binding.id));
|
||||
if (matched.length === 0) {
|
||||
result.ignored += 1;
|
||||
acknowledge.push(event.id);
|
||||
continue;
|
||||
}
|
||||
for (const deliveryId of lease.failedEventIds ?? []) {
|
||||
logger.warn({ deliveryId, reason: "inner_event_authentication_failed" },
|
||||
"GitHub connector event retained for retry");
|
||||
try {
|
||||
const companies = new Map<string, GitHubBinding[]>();
|
||||
for (const binding of matched) companies.set(binding.companyId, [...(companies.get(binding.companyId) ?? []), binding]);
|
||||
for (const [companyId, companyBindings] of companies) {
|
||||
const status = await processForCompany(companyId, companyBindings, event);
|
||||
result[status] += 1;
|
||||
}
|
||||
acknowledge.push(event.id);
|
||||
if (event.event === "installation" && (event.action === "deleted" || event.action === "suspend")) {
|
||||
await Promise.all(matched.map((binding) => connector.setWebhookBinding({
|
||||
subject: binding.subject,
|
||||
companyId: binding.companyId,
|
||||
id: binding.id,
|
||||
installationId: binding.installationId,
|
||||
connectionId: binding.connectionId,
|
||||
grantId: binding.grantId,
|
||||
active: false,
|
||||
})));
|
||||
}
|
||||
} catch {
|
||||
result.failed += 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
if (acknowledge.length > 0) {
|
||||
await connector.acknowledgeEvents({
|
||||
subject: first.subject,
|
||||
companyId: first.companyId,
|
||||
leaseId: lease.leaseId,
|
||||
deliveryIds: acknowledge,
|
||||
});
|
||||
}
|
||||
for (const deliveryId of lease.failedEventIds ?? []) {
|
||||
logger.warn({ deliveryId, reason: "inner_event_authentication_failed" },
|
||||
"GitHub connector event retained for retry");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
return {
|
||||
pollOnce(input: { forSetup?: boolean } = {}): Promise<GitHubConnectionEventPollResult> {
|
||||
// A manager actively awaiting a dedicated App's signed ping must not
|
||||
// inherit a personal connection's old empty-queue backoff. This is a
|
||||
// bounded readiness probe, not an additional background polling loop.
|
||||
if (inFlight) return inFlight;
|
||||
const time = now().getTime();
|
||||
if (time < (input.forSetup ? nextSetupPollAt : nextPollAt)) {
|
||||
return Promise.resolve({ leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 });
|
||||
}
|
||||
nextSetupPollAt = time + 5_000;
|
||||
nextPollAt = Math.max(nextPollAt, nextSetupPollAt);
|
||||
const work = runPoll().finally(() => { if (inFlight === work) inFlight = null; });
|
||||
inFlight = work;
|
||||
return work;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -112,7 +112,7 @@ export async function resolveGitHubOperationCredentials(
|
||||
runId: string;
|
||||
},
|
||||
) {
|
||||
const { run, context } = await captureRunIdentity(db, input);
|
||||
const { run, context } = await captureRunIdentity(db, input, { readScope: "github_credentials" });
|
||||
if (!context) throw forbidden("This run predates managed GitHub credentials");
|
||||
let summary: GitHubCredentialSummary;
|
||||
let env: Record<string, string> = {};
|
||||
|
||||
@@ -202,18 +202,36 @@ export async function appendHeartbeatRunEvent(
|
||||
}
|
||||
|
||||
async function contiguousCursor(db: Db, runId: string, sourceInstanceId: string): Promise<number> {
|
||||
const rows = await db
|
||||
const scope = and(
|
||||
eq(heartbeatRunEvents.runId, runId),
|
||||
eq(heartbeatRunEvents.sourceInstanceId, sourceInstanceId),
|
||||
sql`${heartbeatRunEvents.sourceSeq} >= 1`,
|
||||
);
|
||||
const [range] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)`.mapWith(Number),
|
||||
max: sql<number | null>`max(${heartbeatRunEvents.sourceSeq})`.mapWith(Number),
|
||||
})
|
||||
.from(heartbeatRunEvents)
|
||||
.where(scope);
|
||||
if (!range || !range.count) return 0;
|
||||
if (range.max === range.count) return range.count;
|
||||
const [first] = await db
|
||||
.select({ sourceSeq: heartbeatRunEvents.sourceSeq })
|
||||
.from(heartbeatRunEvents)
|
||||
.where(and(
|
||||
eq(heartbeatRunEvents.runId, runId),
|
||||
eq(heartbeatRunEvents.sourceInstanceId, sourceInstanceId),
|
||||
))
|
||||
.orderBy(asc(heartbeatRunEvents.sourceSeq));
|
||||
let cursor = 0;
|
||||
for (const row of rows) {
|
||||
if (row.sourceSeq === cursor + 1) cursor += 1;
|
||||
else if (row.sourceSeq !== null && row.sourceSeq > cursor + 1) break;
|
||||
}
|
||||
return cursor;
|
||||
.where(and(scope, eq(heartbeatRunEvents.sourceSeq, 1)))
|
||||
.limit(1);
|
||||
if (!first) return 0;
|
||||
const [last] = await db
|
||||
.select({ sourceSeq: heartbeatRunEvents.sourceSeq })
|
||||
.from(heartbeatRunEvents)
|
||||
.where(and(scope, sql`not exists (
|
||||
select 1 from heartbeat_run_events next_event
|
||||
where next_event.run_id = heartbeat_run_events.run_id
|
||||
and next_event.source_instance_id = heartbeat_run_events.source_instance_id
|
||||
and next_event.source_seq = heartbeat_run_events.source_seq + 1
|
||||
)`))
|
||||
.orderBy(asc(heartbeatRunEvents.sourceSeq))
|
||||
.limit(1);
|
||||
return last?.sourceSeq ?? 0;
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { issues, heartbeatRuns } from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { canActorReadExecutionWorkspace, type AuthorizationActor, type AuthorizationDecision } from "./authorization.js";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { canActorReadExecutionWorkspace, type AuthorizationActor, type AuthorizationDecision, type authorizationService } from "./authorization.js";
|
||||
import { retryIdempotentDatabaseOperation } from "../database-retry.js";
|
||||
|
||||
type RunIssueBinding = {
|
||||
@@ -13,6 +13,7 @@ type RunIssueBinding = {
|
||||
export async function canActorReadHeartbeatRun(
|
||||
db: Db,
|
||||
access: {
|
||||
decideReads?: ReturnType<typeof authorizationService>["decideReads"];
|
||||
decide(input: {
|
||||
actor: AuthorizationActor;
|
||||
action: "issue:read";
|
||||
@@ -66,6 +67,44 @@ export async function getHeartbeatRunReadDecisions(
|
||||
actor: AuthorizationActor,
|
||||
runs: readonly RunIssueBinding[],
|
||||
): Promise<boolean[]> {
|
||||
const decideReads = access.decideReads;
|
||||
if (decideReads) {
|
||||
const bindings = runs.map(run => ({ companyId: run.companyId, scopeKind: run.scopeKind, issueId: run.issueId }));
|
||||
const key = (companyId: string, issueId: string) => JSON.stringify([companyId, issueId]);
|
||||
const issuesByCompany = new Map<string, Set<string>>();
|
||||
for (const run of bindings) {
|
||||
if (run.scopeKind === "company" || !run.issueId) continue;
|
||||
let ids = issuesByCompany.get(run.companyId);
|
||||
if (!ids) { ids = new Set(); issuesByCompany.set(run.companyId, ids); }
|
||||
ids.add(run.issueId);
|
||||
}
|
||||
const resources: Parameters<typeof decideReads>[1] = [];
|
||||
for (const [companyId, issueIds] of issuesByCompany) {
|
||||
const ids = [...issueIds];
|
||||
for (let offset = 0; offset < ids.length; offset += 500) {
|
||||
const chunk = ids.slice(offset, offset + 500);
|
||||
const existing = await retryIdempotentDatabaseOperation(() => db
|
||||
.select({ id: issues.id })
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), inArray(issues.id, chunk))));
|
||||
for (const { id: issueId } of existing) resources.push({
|
||||
type: "issue", companyId, issueId, projectId: null, parentIssueId: null,
|
||||
assigneeAgentId: null, assigneeUserId: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
const allowed = new Map<string, boolean>();
|
||||
if (resources.length) {
|
||||
const decisions = await retryIdempotentDatabaseOperation(() => decideReads(actor, resources));
|
||||
for (let index = 0; index < resources.length; index++) {
|
||||
const resource = resources[index];
|
||||
if (resource.type === "issue" && resource.issueId) {
|
||||
allowed.set(key(resource.companyId, resource.issueId), decisions[index]?.allowed === true);
|
||||
}
|
||||
}
|
||||
}
|
||||
return bindings.map(run => run.scopeKind === "company" || Boolean(run.issueId && allowed.get(key(run.companyId, run.issueId))));
|
||||
}
|
||||
const decisions = new Map<string, Promise<boolean>>();
|
||||
return Promise.all(runs.map(run => {
|
||||
const binding = { companyId: run.companyId, scopeKind: run.scopeKind, issueId: run.issueId };
|
||||
|
||||
@@ -298,7 +298,7 @@ export function readAcceptedTaskContinuationSummary(
|
||||
const result = record(resultJson);
|
||||
if (result.finalizationPhase !== "committed") return null;
|
||||
const kind = result.finalizationReasonCode === "live_continuation_registered" ? "same_agent"
|
||||
: ["scheduled_monitor_waiting", "scheduled_monitor_dispatched"].includes(String(result.finalizationReasonCode)) ? "monitor" : null;
|
||||
: ["scheduled_monitor_waiting", "scheduled_monitor_dispatched", "scheduled_monitor_preserved"].includes(String(result.finalizationReasonCode)) ? "monitor" : null;
|
||||
if (!kind) return null;
|
||||
for (const candidate of semanticResultCandidates(result)) {
|
||||
const continuation = record(candidate.continuation);
|
||||
|
||||
@@ -811,6 +811,9 @@ export function heartbeatService(
|
||||
getAgent,
|
||||
resolveSessionBeforeForWakeup,
|
||||
getRun,
|
||||
getRunCleanupState,
|
||||
getRunExecutionWait,
|
||||
getRunWorkspaceSettlement,
|
||||
getRunStatus,
|
||||
ensureRuntimeState,
|
||||
getTaskSession,
|
||||
@@ -823,6 +826,7 @@ export function heartbeatService(
|
||||
hasResolvablePriorSessionWorkspaceForWake,
|
||||
getRunReadAccess,
|
||||
getRunLogAccess,
|
||||
getRunWorkspaceAccess,
|
||||
listRuns,
|
||||
getRuntimeStateWithSessions,
|
||||
listTaskSessions,
|
||||
@@ -830,6 +834,8 @@ export function heartbeatService(
|
||||
listEvents,
|
||||
getRetryExhaustedReason,
|
||||
getRunIssueSummary,
|
||||
getRunIssueControl,
|
||||
getActiveRunIssueControlForAgent,
|
||||
getActiveRunForAgent,
|
||||
getActiveRunIssueSummaryForAgent,
|
||||
} = createHeartbeatRunState(db);
|
||||
@@ -1014,6 +1020,7 @@ export function heartbeatService(
|
||||
environmentRuntime,
|
||||
scheduleNativeSessionResumeDispatch,
|
||||
getRun,
|
||||
getRunWorkspaceSettlement,
|
||||
executeRun,
|
||||
activeRunExecutionPromises,
|
||||
cancelHeartbeatNativeRun,
|
||||
@@ -1386,6 +1393,8 @@ export function heartbeatService(
|
||||
|
||||
const runControl = createHeartbeatRunControl(db, {
|
||||
getRun,
|
||||
getRunCleanupState,
|
||||
getRunExecutionWait,
|
||||
options,
|
||||
envOrchestrator,
|
||||
activeRunExecutions,
|
||||
@@ -3987,6 +3996,7 @@ export function heartbeatService(
|
||||
|
||||
getRunReadAccess,
|
||||
getRunLogAccess,
|
||||
getRunWorkspaceAccess,
|
||||
|
||||
getRuntimeState: getRuntimeStateWithSessions,
|
||||
|
||||
@@ -4007,7 +4017,7 @@ export function heartbeatService(
|
||||
logStore: string | null;
|
||||
logRef: string | null;
|
||||
},
|
||||
opts?: { offset?: number; limitBytes?: number },
|
||||
opts?: { offset?: number; limitBytes?: number; alignStartToRecord?: boolean; signal?: AbortSignal },
|
||||
) => {
|
||||
const run =
|
||||
typeof runOrLookup === "string"
|
||||
@@ -4149,6 +4159,10 @@ export function heartbeatService(
|
||||
|
||||
getRunIssueSummary,
|
||||
|
||||
getRunIssueControl,
|
||||
|
||||
getActiveRunIssueControlForAgent,
|
||||
|
||||
getActiveRunForAgent,
|
||||
|
||||
getActiveRunIssueSummaryForAgent,
|
||||
|
||||
@@ -13,6 +13,7 @@ import type { HeartbeatRunControlDependencies } from "../run-control.js";
|
||||
type TestCallbacks = HeartbeatRunControlDependencies & ReturnType<typeof createHeartbeatLifecycle> & {
|
||||
getRun: Mock<HeartbeatRunControlDependencies["getRun"]>;
|
||||
getAgent: Mock<HeartbeatRunControlDependencies["getAgent"]>;
|
||||
getRunCleanupState: Mock<HeartbeatRunControlDependencies["getRunCleanupState"]>;
|
||||
setRunStatusFromLive: Mock<HeartbeatRunControlDependencies["setRunStatusFromLive"]>;
|
||||
getSchedulingSuppression: Mock<HeartbeatRunControlDependencies["getSchedulingSuppression"]>;
|
||||
options: { closeWarmNativeSessionsForRun: Mock<NonNullable<HeartbeatRunControlDependencies["options"]["closeWarmNativeSessionsForRun"]>> };
|
||||
@@ -43,6 +44,7 @@ export function callbacks(db: Db): TestCallbacks {
|
||||
...state,
|
||||
...lifecycle,
|
||||
getRun: vi.fn(state.getRun),
|
||||
getRunCleanupState: vi.fn(state.getRunCleanupState),
|
||||
getAgent: vi.fn(state.getAgent),
|
||||
setRunStatusFromLive: vi.fn(lifecycle.setRunStatusFromLive),
|
||||
setRunStatus: vi.fn(lifecycle.setRunStatus),
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { heartbeatRunEvents } from "@paperclipai/db";
|
||||
|
||||
/**
|
||||
* Canonical PRP conversation data, excluding historical tool activity.
|
||||
* Keep request lifecycles so a cold open cannot hide a question or approval.
|
||||
* Item identity may occur on started/completed rather than each text delta.
|
||||
*/
|
||||
export function conversationEventCondition(runId: string) {
|
||||
return sql`(
|
||||
${heartbeatRunEvents.eventType} like 'runtime_request.%'
|
||||
or ${heartbeatRunEvents.eventType} in ('run.result.accepted', 'run.terminal', 'turn.completed', 'turn.failed', 'turn.cancelled', 'turn.interrupted')
|
||||
or (
|
||||
${heartbeatRunEvents.eventType} in ('item.started', 'item.delta', 'item.completed')
|
||||
and (lower(replace(coalesce(${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}', ${heartbeatRunEvents.payload} #>> '{prpEvent,payload,item,kind}', ${heartbeatRunEvents.payload} #>> '{prpEvent,payload,item,type}', ''), '_', '')) in ('agentmessage', 'assistantmessage')
|
||||
or coalesce(
|
||||
${heartbeatRunEvents.payload} #>> '{prpEvent,itemId}',
|
||||
${heartbeatRunEvents.payload} #>> '{prpEvent,payload,itemId}',
|
||||
${heartbeatRunEvents.payload} #>> '{prpEvent,payload,item,id}',
|
||||
${heartbeatRunEvents.payload} #>> '{prpEvent,payload,id}'
|
||||
) in (
|
||||
select coalesce(e.payload #>> '{prpEvent,itemId}', e.payload #>> '{prpEvent,payload,itemId}', e.payload #>> '{prpEvent,payload,item,id}', e.payload #>> '{prpEvent,payload,id}')
|
||||
from heartbeat_run_events e
|
||||
where e.run_id = ${runId}
|
||||
and e.event_type in ('item.started', 'item.delta', 'item.completed')
|
||||
and lower(replace(coalesce(e.payload #>> '{prpEvent,payload,kind}', e.payload #>> '{prpEvent,payload,item,kind}', e.payload #>> '{prpEvent,payload,item,type}', ''), '_', '')) in ('agentmessage', 'assistantmessage')
|
||||
))
|
||||
)
|
||||
)`;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
|
||||
import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js";
|
||||
import { readNativeStopExecutionEvidence } from "./native-stop-evidence-read.js";
|
||||
import { createHeartbeatRunControl } from "./run-control.js";
|
||||
|
||||
vi.mock("../local-service-supervisor.js", async importOriginal => ({
|
||||
...await importOriginal<typeof import("../local-service-supervisor.js")>(),
|
||||
terminateLocalService: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence reads", () => {
|
||||
let db: Db;
|
||||
const { fixture } = setupRunControlDatabase("native-stop-evidence-", value => { db = value; });
|
||||
|
||||
it("reads current receipts below the byte budget without changing saved data", async () => {
|
||||
const { company, agent } = await fixture();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Stop evidence fixture" }).returning();
|
||||
const id = randomUUID(), bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const receipt = { schema: "paperclip.native-cancellation.v1", runId: id, companyId: company.id, issueId: issue.id,
|
||||
scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: true,
|
||||
intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID(), futureProof: { version: 2 } };
|
||||
const result = { nativeCancellation: receipt, cancelledByActorType: "user", cancelledByUserId: randomUUID(),
|
||||
stdout: bulk, stderr: bulk, futureExecutionEvidence: { savedInput: bulk } };
|
||||
const [before] = await db.insert(heartbeatRuns).values({ id, companyId: company.id, agentId: agent.id,
|
||||
nativeIssueId: issue.id, runtimeMode: "native", status: "cancelled", processPid: 123, processGroupId: 456,
|
||||
contextSnapshot: { savedInput: bulk }, runnerProfileJson: { savedInput: bulk }, stdoutExcerpt: bulk,
|
||||
stderrExcerpt: bulk, resultJson: result }).returning();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let evidence: Awaited<ReturnType<typeof readNativeStopExecutionEvidence>>;
|
||||
try { evidence = await readNativeStopExecutionEvidence(db, company.id, id); }
|
||||
finally { capture.flush(); }
|
||||
expect(evidence!).toEqual({ id, companyId: company.id, nativeIssueId: issue.id, status: "cancelled", processPid: 123,
|
||||
processGroupId: 456, resultJson: { nativeCancellation: receipt, cancelledByActorType: "user", cancelledByUserId: result.cancelledByUserId } });
|
||||
expect(isAcknowledgedNativeStop(evidence!)).toBe(true);
|
||||
expect(events.at(-1)).toMatchObject({ completedQueries: 1, groups: 1 });
|
||||
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(2048);
|
||||
expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64));
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]).toEqual(before);
|
||||
await db.update(heartbeatRuns).set({ resultJson: { ...result, nativeCancellation: { ...receipt, dispatchState: "pending" } } })
|
||||
.where(eq(heartbeatRuns.id, id));
|
||||
expect(isAcknowledgedNativeStop((await readNativeStopExecutionEvidence(db, company.id, id))!)).toBe(false);
|
||||
expect(await readNativeStopExecutionEvidence(db, randomUUID(), id)).toBeNull();
|
||||
expect(await readNativeStopExecutionEvidence(db, company.id, randomUUID())).toBeNull();
|
||||
});
|
||||
|
||||
it.each([null, [], false, 42, "not a receipt", { nativeCancellation: null }, { nativeCancellation: { dispatched: "true" } }])(
|
||||
"keeps malformed receipts fail-closed: %j", async result => {
|
||||
const { company, agent } = await fixture();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
|
||||
runtimeMode: "native", status: "cancelled" }).returning();
|
||||
await db.execute(sql`update ${heartbeatRuns} set result_json = ${JSON.stringify(result)}::jsonb where id = ${run.id}`);
|
||||
expect(isAcknowledgedNativeStop((await readNativeStopExecutionEvidence(db, company.id, run.id))!)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["string", "array"])("does not transfer a large malformed %s result", async kind => {
|
||||
const { company, agent } = await fixture();
|
||||
const bulk = randomBytes(128 * 1024).toString("base64");
|
||||
const saved = kind === "array" ? [bulk] : bulk;
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
|
||||
runtimeMode: "native", status: "cancelled" }).returning();
|
||||
await db.execute(sql`update ${heartbeatRuns} set result_json = ${JSON.stringify(saved)}::jsonb where id = ${run.id}`);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const evidence = await readNativeStopExecutionEvidence(db, company.id, run.id);
|
||||
expect(evidence?.resultJson).toBeNull();
|
||||
expect(isAcknowledgedNativeStop(evidence!)).toBe(false);
|
||||
} finally { capture.flush(); }
|
||||
expect(events.at(-1)).toMatchObject({ completedQueries: 1, groups: 1 });
|
||||
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(1024);
|
||||
expect((await db.select({ result: heartbeatRuns.resultJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].result).toEqual(saved);
|
||||
});
|
||||
|
||||
it("uses the scoped receipt reader for the native continuation recheck", async () => {
|
||||
const { company, agent } = await fixture();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Native continuation fixture" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
|
||||
nativeIssueId: issue.id, runtimeMode: "native", status: "succeeded" }).returning();
|
||||
const deps = callbacks(db);
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
await control.resumeRemoteStopComments(run);
|
||||
expect(deps.getRun).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
|
||||
export async function readNativeStopExecutionEvidence(db: Db, companyId: string, runId: string) {
|
||||
const [run] = await db.select({
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
status: heartbeatRuns.status,
|
||||
processPid: heartbeatRuns.processPid,
|
||||
processGroupId: heartbeatRuns.processGroupId,
|
||||
resultJson: sql<typeof heartbeatRuns.$inferSelect.resultJson>`case
|
||||
when jsonb_typeof(${heartbeatRuns.resultJson}) = 'object' then (
|
||||
select coalesce(jsonb_object_agg(key, value), '{}'::jsonb)
|
||||
from jsonb_each(${heartbeatRuns.resultJson})
|
||||
where key = any(array['nativeCancellation', 'cancelledByActorType', 'cancelledByUserId'])
|
||||
) else null end`,
|
||||
}).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))).limit(1);
|
||||
return run ?? null;
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
import { claimNativeSessionResumptions } from "../native-runtime/native-finalization-reconciler.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agents, agentWakeupRequests, caseHandoffs, cases, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueComments, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { eq, inArray, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js";
|
||||
import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js";
|
||||
import { createHeartbeatQueue, type HeartbeatQueueDependencies } from "./queue.js";
|
||||
import { createHeartbeatRunControl } from "./run-control.js";
|
||||
import { callbacks as runControlCallbacks } from "./__tests__/run-control-fixtures.js";
|
||||
|
||||
import { createHeartbeatQueueTestDependencies as callbacks } from "../../__tests__/helpers/heartbeat-queue.js";
|
||||
|
||||
@@ -113,6 +115,116 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
return { ...seeded, recipient, pending };
|
||||
}
|
||||
|
||||
async function addStrandedMessages(seeded: Awaited<ReturnType<typeof strandedMention>>) {
|
||||
const messages = [seeded.pending];
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const [comment] = await db.insert(issueComments).values({ companyId: seeded.company.id, issueId: seeded.issue.id,
|
||||
body: `Additional saved feedback ${index}`, authorAgentId: seeded.agent.id }).returning();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: seeded.company.id, agentId: seeded.recipient.id,
|
||||
source: "automation", reason: "issue_execution_deferred", status: "deferred_issue_execution",
|
||||
payload: { issueId: seeded.issue.id, _paperclipWakeContext: { issueId: seeded.issue.id,
|
||||
wakeReason: "issue_comment_mentioned", wakeCommentIds: [comment.id] } } }).returning();
|
||||
messages.push(wake);
|
||||
}
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: new Date(0) }).where(inArray(agentWakeupRequests.id, messages.map(row => row.id)));
|
||||
return messages;
|
||||
}
|
||||
|
||||
async function expectSavedMessages(messages: Awaited<ReturnType<typeof addStrandedMessages>>) {
|
||||
const rows = await db.select().from(agentWakeupRequests).where(inArray(agentWakeupRequests.id, messages.map(row => row.id)));
|
||||
expect(rows).toHaveLength(messages.length);
|
||||
for (const before of messages) {
|
||||
const row = rows.find(row => row.id === before.id)!;
|
||||
expect(row).toMatchObject({ payload: before.payload, status: "deferred_issue_execution", runId: null,
|
||||
finishedAt: null, error: null });
|
||||
expect(row.updatedAt.getTime()).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
|
||||
it.each(["legacy", "native"])("keeps saved %s messages while avoiding repeated native source release", async runtimeMode => {
|
||||
const seeded = await strandedMention(runtimeMode);
|
||||
const messages = await addStrandedMessages(seeded);
|
||||
const deps = callbacks(db), queue = createHeartbeatQueue(db, deps);
|
||||
await queue.resumeQueuedRuns();
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(runtimeMode === "native" ? 1 : 3);
|
||||
await expectSavedMessages(messages);
|
||||
await queue.resumeQueuedRuns();
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(runtimeMode === "native" ? 2 : 6);
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries a rejected native release on the next saved message", async () => {
|
||||
const seeded = await strandedMention("native"), messages = await addStrandedMessages(seeded);
|
||||
const deps = callbacks(db);
|
||||
deps.releaseIssueExecutionAndPromote.mockRejectedValueOnce(new Error("Fixture release remains pending"));
|
||||
await createHeartbeatQueue(db, deps).resumeQueuedRuns();
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(2);
|
||||
await expectSavedMessages(messages);
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("joins the native release before it suppresses another message's duplicate check", async () => {
|
||||
const seeded = await strandedMention("native"), messages = await addStrandedMessages(seeded);
|
||||
const deps = callbacks(db);
|
||||
let finish!: () => void;
|
||||
deps.releaseIssueExecutionAndPromote.mockImplementationOnce(() => new Promise<void>(resolve => { finish = resolve; }));
|
||||
let settled = false;
|
||||
const pending = createHeartbeatQueue(db, deps).resumeQueuedRuns().then(() => { settled = true; });
|
||||
try {
|
||||
await vi.waitFor(() => expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(1));
|
||||
expect(settled).toBe(false);
|
||||
} finally { finish?.(); }
|
||||
await pending;
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(1);
|
||||
await expectSavedMessages(messages);
|
||||
});
|
||||
|
||||
it("rechecks a different native source that completes during the pass", async () => {
|
||||
const seeded = await strandedMention("native"), messages = await addStrandedMessages(seeded);
|
||||
await db.update(heartbeatRuns).set({ createdAt: new Date(0) }).where(eq(heartbeatRuns.id, seeded.run.id));
|
||||
const deps = callbacks(db);
|
||||
let nextId: string | undefined;
|
||||
deps.releaseIssueExecutionAndPromote.mockImplementationOnce(async () => {
|
||||
const [next] = await db.insert(heartbeatRuns).values({ companyId: seeded.company.id, agentId: seeded.agent.id,
|
||||
runtimeMode: "native", nativeIssueId: seeded.issue.id, status: "succeeded", finishedAt: new Date(),
|
||||
contextSnapshot: { issueId: seeded.issue.id } }).returning();
|
||||
nextId = next.id;
|
||||
});
|
||||
await createHeartbeatQueue(db, deps).resumeQueuedRuns();
|
||||
expect(deps.releaseIssueExecutionAndPromote.mock.calls.map(([run]) => run.id)).toEqual([seeded.run.id, nextId]);
|
||||
await expectSavedMessages(messages);
|
||||
});
|
||||
|
||||
it("rechecks changed native evidence on the same source run", async () => {
|
||||
const seeded = await strandedMention("native"), messages = await addStrandedMessages(seeded);
|
||||
const deps = callbacks(db);
|
||||
deps.releaseIssueExecutionAndPromote.mockImplementationOnce(async () => {
|
||||
await db.update(heartbeatRuns).set({ resultJson: { observedReceipt: "new evidence" } }).where(eq(heartbeatRuns.id, seeded.run.id));
|
||||
});
|
||||
await createHeartbeatQueue(db, deps).resumeQueuedRuns();
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(2);
|
||||
expect(deps.releaseIssueExecutionAndPromote.mock.calls.map(([run]) => run.id)).toEqual([seeded.run.id, seeded.run.id]);
|
||||
await expectSavedMessages(messages);
|
||||
});
|
||||
|
||||
it("checks each unrelated native task in the same pass", async () => {
|
||||
const seeded = await strandedMention("native"), firstMessages = await addStrandedMessages(seeded);
|
||||
const [issue] = await db.insert(issues).values({ companyId: seeded.company.id, title: "Another saved task", status: "todo", assigneeAgentId: seeded.agent.id }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: seeded.company.id, agentId: seeded.agent.id,
|
||||
runtimeMode: "native", nativeIssueId: issue.id, status: "succeeded", finishedAt: new Date(), contextSnapshot: { issueId: issue.id } }).returning();
|
||||
const [comment] = await db.insert(issueComments).values({ companyId: seeded.company.id, issueId: issue.id,
|
||||
body: "Other task feedback", authorAgentId: seeded.agent.id }).returning();
|
||||
const [pending] = await db.insert(agentWakeupRequests).values({ companyId: seeded.company.id, agentId: seeded.recipient.id,
|
||||
source: "automation", reason: "issue_execution_deferred", status: "deferred_issue_execution",
|
||||
payload: { issueId: issue.id, _paperclipWakeContext: { issueId: issue.id, wakeReason: "issue_comment_mentioned", wakeCommentIds: [comment.id] } } }).returning();
|
||||
const secondMessages = await addStrandedMessages({ ...seeded, issue, run, pending });
|
||||
const deps = callbacks(db);
|
||||
await createHeartbeatQueue(db, deps).resumeQueuedRuns();
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(2);
|
||||
expect(new Set(deps.releaseIssueExecutionAndPromote.mock.calls.map(([row]) => row.id))).toEqual(new Set([seeded.run.id, run.id]));
|
||||
await expectSavedMessages([...firstMessages, ...secondMessages]);
|
||||
});
|
||||
|
||||
it("rechecks a handoff's assignment version under the issue claim lock", async () => {
|
||||
const { company, agent, issue, wake, run } = await fixture("queued");
|
||||
const [record] = await db.insert(cases).values({ companyId: company.id, caseNumber: 1, identifier: "QM-CASE-1",
|
||||
@@ -137,7 +249,9 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
});
|
||||
|
||||
it.each(["legacy", "native"])("periodic recovery preserves a paused recipient through the real %s release adapter", async runtimeMode => {
|
||||
const { company, issue, recipient, pending } = await strandedMention(runtimeMode);
|
||||
const seeded = await strandedMention(runtimeMode);
|
||||
const { company, issue, recipient } = seeded;
|
||||
const messages = await addStrandedMessages(seeded);
|
||||
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, recipient.id));
|
||||
await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, issue.id));
|
||||
const deps = callbacks(db);
|
||||
@@ -146,16 +260,16 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }),
|
||||
resolveSessionBeforeForWakeup: async () => null,
|
||||
}), recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined } });
|
||||
deps.releaseIssueExecutionAndPromote.mockImplementation(async (run, options) => {
|
||||
await release({ companyId: run.companyId, runId: run.id, now: new Date(), ...options });
|
||||
});
|
||||
const controlDeps = runControlCallbacks(db);
|
||||
vi.mocked(controlDeps.wakeQueue.releaseIssueExecution).mockImplementation(release);
|
||||
const control = createHeartbeatRunControl(db, controlDeps);
|
||||
deps.releaseIssueExecutionAndPromote.mockImplementation(control.releaseIssueExecutionAndPromote);
|
||||
for (let pass = 0; pass < 2; pass++) {
|
||||
await createHeartbeatQueue(db, deps).resumeQueuedRuns();
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, pending.id)))[0])
|
||||
.toMatchObject({ status: "deferred_issue_execution", runId: null, finishedAt: null, error: null, payload: pending.payload });
|
||||
await expectSavedMessages(messages);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, company.id))).toHaveLength(1);
|
||||
}
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(2);
|
||||
expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledTimes(runtimeMode === "native" ? 2 : 6);
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -1834,6 +1834,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
AND live_issue_run.status IN ('queued', 'running', 'scheduled_retry'))`,
|
||||
cutoff ? gte(agentWakeupRequests.requestedAt, cutoff) : undefined))
|
||||
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
|
||||
const releasedNativeSources = new Set<string>();
|
||||
for (const { wake } of strandedQueues) {
|
||||
if (!queuedCommentIdsFromWakePayload(wake.payload).length &&
|
||||
!await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue;
|
||||
@@ -1845,6 +1846,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
startedAt: heartbeatRuns.startedAt,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
sourceVersion: sql<string>`${heartbeatRuns}.xmin::text`,
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, wake.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${String(wake.payload?.issueId)}`,
|
||||
@@ -1855,6 +1857,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
));
|
||||
if (!latest || !latest.finishedAt || !isHeartbeatRunTerminalStatus(latest.status)) continue;
|
||||
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
|
||||
const releaseSource = `${latest.companyId}:${String(wake.payload?.issueId)}:${latest.id}:${latest.sourceVersion}`;
|
||||
if (latest.runtimeMode === "native" && releasedNativeSources.has(releaseSource)) continue;
|
||||
if (latest.runtimeMode === "legacy") {
|
||||
const cancelledAdmission = latest.status === "cancelled" && !latest.startedAt &&
|
||||
latest.errorCode === "execution_reconciliation_required";
|
||||
@@ -1864,7 +1868,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
});
|
||||
}
|
||||
}
|
||||
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).catch(err => {
|
||||
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
|
||||
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
|
||||
}).catch(err => {
|
||||
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro
|
||||
import { classifyEnvironmentCapabilities, environmentRuntimeService } from "../environment-runtime.js";
|
||||
import { environmentService } from "../environments.js";
|
||||
import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "./recovery.js";
|
||||
import { createHeartbeatRunState } from "./run-state.js";
|
||||
import type { HotRestartIntent } from "../hot-restart.js";
|
||||
import type { NativeRestartRecoveryClaim, NativeRestartRecoveryDisposition } from "../native-runtime/index.js";
|
||||
import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "../native-runtime/native-runner-ownership.js";
|
||||
@@ -48,6 +49,7 @@ function callbacks(db: Db) {
|
||||
return {
|
||||
enterShutdown: vi.fn(),
|
||||
getRun: vi.fn<HeartbeatRecoveryDependencies["getRun"]>(async () => null),
|
||||
getRunWorkspaceSettlement: vi.fn(createHeartbeatRunState(db).getRunWorkspaceSettlement),
|
||||
getAgent: vi.fn<HeartbeatRecoveryDependencies["getAgent"]>(async () => null),
|
||||
appendRunEvent: vi.fn<HeartbeatRecoveryDependencies["appendRunEvent"]>(async () => undefined),
|
||||
setRunStatusIfRunning: vi.fn<HeartbeatRecoveryDependencies["setRunStatusIfRunning"]>(async () => ({ updated: false, run: null })),
|
||||
|
||||
@@ -101,6 +101,7 @@ export interface HeartbeatRecoveryDependencies extends Pick<HeartbeatRetryDepend
|
||||
getRun: (runId: string, options?: Parameters<ReturnType<typeof createHeartbeatRunState>["getRun"]>[1]) => Promise<
|
||||
Awaited<ReturnType<ReturnType<typeof createHeartbeatRunState>["getRun"]>> | null
|
||||
>;
|
||||
getRunWorkspaceSettlement: ReturnType<typeof createHeartbeatRunState>["getRunWorkspaceSettlement"];
|
||||
environmentRuntime: ReturnType<typeof environmentRuntimeService>;
|
||||
pluginWorkerManager?: WorkspaceWorkers;
|
||||
environmentsSvc: Pick<ReturnType<typeof environmentService>, "getById" | "getLeaseById" | "releaseLease">;
|
||||
@@ -360,6 +361,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
environmentRuntime,
|
||||
scheduleNativeSessionResumeDispatch,
|
||||
getRun,
|
||||
getRunWorkspaceSettlement,
|
||||
executeRun,
|
||||
activeRunExecutionPromises,
|
||||
cancelHeartbeatNativeRun,
|
||||
@@ -1858,7 +1860,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
agentId: string;
|
||||
succeeded: boolean;
|
||||
}) {
|
||||
const settledRun = await getRun(input.runId);
|
||||
const settledRun = await getRunWorkspaceSettlement(input.runId);
|
||||
const workspaceSyncReference = readNativeWorkspaceSyncReference(
|
||||
parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,
|
||||
);
|
||||
|
||||
@@ -9,6 +9,7 @@ import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
import { UnresolvedWorkspaceBaseRefError } from "../workspace-runtime.js";
|
||||
import { LegacyControllerLeaseLostError } from "../legacy-controller-lease.js";
|
||||
import { requestedRunCancellation } from "../run-cancellation.js";
|
||||
import { ProjectAccessDeniedError } from "../project-access-denial.js";
|
||||
import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";
|
||||
import { createHeartbeatRunPreparation, ConfigurationIncompleteFailure } from "./run-preparation.js";
|
||||
import { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js";
|
||||
@@ -77,15 +78,19 @@ it("constructs completion handlers without querying or starting work", () => {
|
||||
});
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("heartbeat run completion boundary", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)("heartbeat run completion boundary", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: Db;
|
||||
beforeAll(async () => {
|
||||
// Keep the accounting transaction real without reading provider receipt files.
|
||||
vi.spyOn(usageReceipts, "indexPendingUsageReceipts").mockResolvedValue(new Map());
|
||||
vi.spyOn(usageReceipts, "recoverPendingRunUsageReceipts").mockResolvedValue([]);
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-completion");
|
||||
db = createDb(database.connectionString);
|
||||
const external = process.env.PAPERCLIP_TEST_DATABASE_URL;
|
||||
if (external) db = createDb(external);
|
||||
else {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-completion");
|
||||
db = createDb(database.connectionString);
|
||||
}
|
||||
await db.execute(sql`set client_min_messages = warning`);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); vi.restoreAllMocks(); });
|
||||
@@ -334,6 +339,18 @@ describe.skipIf(!support.supported)("heartbeat run completion boundary", () => {
|
||||
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), { suppressImmediateRecovery: true });
|
||||
});
|
||||
|
||||
it("persists a project denial without provider work or an automatic retry", async () => {
|
||||
const f = await fixture();
|
||||
const projectId = randomUUID();
|
||||
await f.completion.failRunSetup(f.setup(new ProjectAccessDeniedError(projectId)));
|
||||
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "project_access_denied",
|
||||
resultJson: { projectAccessDenied: { projectId }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });
|
||||
expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.enqueueWakeup).not.toHaveBeenCalled();
|
||||
expect(f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible).not.toHaveBeenCalled();
|
||||
expect(f.deps.finalizeIssueCommentPolicy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("records a setup configuration gap without inventing provider work or applying comment policy", async () => {
|
||||
const f = await fixture();
|
||||
await f.completion.failRunSetup(f.setup(new ConfigurationIncompleteFailure("missing credential", { configurationIncomplete: { reason: "missing_binding" } })));
|
||||
|
||||
Loaded 100 of 313 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user