chore(runtime): harden worktree and heartbeat flows

This commit is contained in:
dotta authored and Dotta committed 2026-04-10 10:11:31 -05:00
1 parent 0e87fdbe35
commit dcb807cb2f
48 files changed
+15603 -670

No files matched your search

+5 -4
View File
@@ -32,15 +32,16 @@
"skills"
],
"scripts": {
"dev": "tsx src/index.ts",
"dev:watch": "cross-env PAPERCLIP_MIGRATION_PROMPT=never PAPERCLIP_MIGRATION_AUTO_APPLY=true tsx ./scripts/dev-watch.ts",
"preflight:workspace-links": "tsx ../scripts/ensure-workspace-package-links.ts",
"dev": "pnpm run preflight:workspace-links && tsx src/index.ts",
"dev:watch": "pnpm run preflight:workspace-links && cross-env PAPERCLIP_MIGRATION_PROMPT=never PAPERCLIP_MIGRATION_AUTO_APPLY=true tsx ./scripts/dev-watch.ts",
"prepare:ui-dist": "bash ../scripts/prepare-server-ui-dist.sh",
"build": "tsc && mkdir -p dist/onboarding-assets && cp -R src/onboarding-assets/. dist/onboarding-assets/",
"build": "pnpm run preflight:workspace-links && tsc && mkdir -p dist/onboarding-assets && cp -R src/onboarding-assets/. dist/onboarding-assets/",
"prepack": "pnpm run prepare:ui-dist",
"postpack": "rm -rf ui-dist",
"clean": "rm -rf dist",
"start": "node dist/index.js",
"typecheck": "pnpm --filter @paperclipai/plugin-sdk build && tsc --noEmit"
"typecheck": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk build && tsc --noEmit"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.888.0",
@@ -19,16 +19,19 @@ describe("adapter session codecs", () => {
const parsed = claudeSessionCodec.deserialize({
session_id: "claude-session-1",
folder: "/tmp/workspace",
prompt_bundle_key: "bundle-1",
});
expect(parsed).toEqual({
sessionId: "claude-session-1",
cwd: "/tmp/workspace",
promptBundleKey: "bundle-1",
});
const serialized = claudeSessionCodec.serialize(parsed);
expect(serialized).toEqual({
sessionId: "claude-session-1",
cwd: "/tmp/workspace",
promptBundleKey: "bundle-1",
});
expect(claudeSessionCodec.getDisplayId?.(serialized ?? null)).toBe("claude-session-1");
});
@@ -0,0 +1,117 @@
import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { agentRoutes } from "../routes/agents.js";
import { errorHandler } from "../middleware/index.js";
const mockAgentService = vi.hoisted(() => ({
getById: vi.fn(),
}));
const mockHeartbeatService = vi.hoisted(() => ({
getRunIssueSummary: vi.fn(),
getActiveRunIssueSummaryForAgent: vi.fn(),
}));
const mockIssueService = vi.hoisted(() => ({
getById: vi.fn(),
getByIdentifier: vi.fn(),
}));
vi.mock("../services/index.js", () => ({
agentService: () => mockAgentService,
agentInstructionsService: () => ({}),
accessService: () => ({}),
approvalService: () => ({}),
companySkillService: () => ({ listRuntimeSkillEntries: vi.fn() }),
budgetService: () => ({}),
heartbeatService: () => mockHeartbeatService,
issueApprovalService: () => ({}),
issueService: () => mockIssueService,
logActivity: vi.fn(),
secretService: () => ({}),
syncInstructionsBundleConfigFromFilePath: vi.fn((_agent, config) => config),
workspaceOperationService: () => ({}),
}));
vi.mock("../adapters/index.js", () => ({
findServerAdapter: vi.fn(),
listAdapterModels: vi.fn(),
detectAdapterModel: vi.fn(),
findActiveServerAdapter: vi.fn(),
requireServerAdapter: vi.fn(),
}));
function createApp() {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
(req as any).actor = {
type: "board",
userId: "local-board",
companyIds: ["company-1"],
source: "local_implicit",
isInstanceAdmin: false,
};
next();
});
app.use("/api", agentRoutes({} as any));
app.use(errorHandler);
return app;
}
describe("agent live run routes", () => {
beforeEach(() => {
vi.clearAllMocks();
mockIssueService.getByIdentifier.mockResolvedValue({
id: "issue-1",
companyId: "company-1",
executionRunId: "run-1",
assigneeAgentId: "agent-1",
status: "in_progress",
});
mockIssueService.getById.mockResolvedValue(null);
mockAgentService.getById.mockResolvedValue({
id: "agent-1",
companyId: "company-1",
name: "Builder",
adapterType: "codex_local",
});
mockHeartbeatService.getRunIssueSummary.mockResolvedValue({
id: "run-1",
status: "running",
invocationSource: "on_demand",
triggerDetail: "manual",
startedAt: new Date("2026-04-10T09:30:00.000Z"),
finishedAt: null,
createdAt: new Date("2026-04-10T09:29:59.000Z"),
agentId: "agent-1",
issueId: "issue-1",
});
mockHeartbeatService.getActiveRunIssueSummaryForAgent.mockResolvedValue(null);
});
it("returns a compact active run payload for issue polling", async () => {
const res = await request(createApp()).get("/api/issues/PAP-1295/active-run");
expect(res.status, JSON.stringify(res.body)).toBe(200);
expect(mockIssueService.getByIdentifier).toHaveBeenCalledWith("PAP-1295");
expect(mockHeartbeatService.getRunIssueSummary).toHaveBeenCalledWith("run-1");
expect(res.body).toEqual({
id: "run-1",
status: "running",
invocationSource: "on_demand",
triggerDetail: "manual",
startedAt: "2026-04-10T09:30:00.000Z",
finishedAt: null,
createdAt: "2026-04-10T09:29:59.000Z",
agentId: "agent-1",
issueId: "issue-1",
agentName: "Builder",
adapterType: "codex_local",
});
expect(res.body).not.toHaveProperty("resultJson");
expect(res.body).not.toHaveProperty("contextSnapshot");
expect(res.body).not.toHaveProperty("logRef");
});
});
+274 -306
View File
@@ -7,16 +7,22 @@ import { execute } from "@paperclipai/adapter-claude-local/server";
async function writeFakeClaudeCommand(commandPath: string): Promise<void> {
const script = `#!/usr/bin/env node
const fs = require("node:fs");
const path = require("node:path");
const argv = process.argv.slice(2);
const addDirIndex = argv.indexOf("--add-dir");
const addDir = addDirIndex >= 0 ? argv[addDirIndex + 1] : null;
const instructionsIndex = argv.indexOf("--append-system-prompt-file");
const instructionsFilePath = instructionsIndex >= 0 ? argv[instructionsIndex + 1] : null;
const capturePath = process.env.PAPERCLIP_TEST_CAPTURE_PATH;
const promptFileFlagIndex = process.argv.indexOf("--append-system-prompt-file");
const appendedSystemPromptFilePath = promptFileFlagIndex >= 0 ? process.argv[promptFileFlagIndex + 1] : null;
const payload = {
argv: process.argv.slice(2),
argv,
prompt: fs.readFileSync(0, "utf8"),
addDir,
instructionsFilePath,
instructionsContents: instructionsFilePath ? fs.readFileSync(instructionsFilePath, "utf8") : null,
skillEntries: addDir ? fs.readdirSync(path.join(addDir, ".claude", "skills")).sort() : [],
claudeConfigDir: process.env.CLAUDE_CONFIG_DIR || null,
appendedSystemPromptFilePath,
appendedSystemPromptFileContents: appendedSystemPromptFilePath ? fs.readFileSync(appendedSystemPromptFilePath, "utf8") : null,
};
if (capturePath) {
fs.writeFileSync(capturePath, JSON.stringify(payload), "utf8");
@@ -29,310 +35,17 @@ console.log(JSON.stringify({ type: "result", session_id: "claude-session-1", res
await fs.chmod(commandPath, 0o755);
}
async function writeRetryThenSucceedClaudeCommand(commandPath: string): Promise<void> {
const script = `#!/usr/bin/env node
const fs = require("node:fs");
const capturePath = process.env.PAPERCLIP_TEST_CAPTURE_PATH;
const statePath = process.env.PAPERCLIP_TEST_STATE_PATH;
const promptFileFlagIndex = process.argv.indexOf("--append-system-prompt-file");
const appendedSystemPromptFilePath = promptFileFlagIndex >= 0 ? process.argv[promptFileFlagIndex + 1] : null;
const payload = {
argv: process.argv.slice(2),
prompt: fs.readFileSync(0, "utf8"),
claudeConfigDir: process.env.CLAUDE_CONFIG_DIR || null,
appendedSystemPromptFilePath,
appendedSystemPromptFileContents: appendedSystemPromptFilePath ? fs.readFileSync(appendedSystemPromptFilePath, "utf8") : null,
type CapturePayload = {
argv: string[];
prompt: string;
addDir: string | null;
instructionsFilePath: string | null;
instructionsContents: string | null;
skillEntries: string[];
claudeConfigDir: string | null;
};
if (capturePath) {
const entries = fs.existsSync(capturePath) ? JSON.parse(fs.readFileSync(capturePath, "utf8")) : [];
entries.push(payload);
fs.writeFileSync(capturePath, JSON.stringify(entries), "utf8");
}
const resumed = process.argv.includes("--resume");
const shouldFailResume = resumed && statePath && !fs.existsSync(statePath);
if (shouldFailResume) {
fs.writeFileSync(statePath, "retried", "utf8");
console.log(JSON.stringify({
type: "result",
subtype: "error",
session_id: "claude-session-1",
result: "No conversation found with session id claude-session-1",
errors: ["No conversation found with session id claude-session-1"],
}));
process.exit(1);
}
console.log(JSON.stringify({ type: "system", subtype: "init", session_id: "claude-session-2", model: "claude-sonnet" }));
console.log(JSON.stringify({ type: "assistant", session_id: "claude-session-2", message: { content: [{ type: "text", text: "hello" }] } }));
console.log(JSON.stringify({ type: "result", session_id: "claude-session-2", result: "hello", usage: { input_tokens: 1, cache_read_input_tokens: 0, output_tokens: 1 } }));
`;
await fs.writeFile(commandPath, script, "utf8");
await fs.chmod(commandPath, 0o755);
}
async function setupExecuteEnv(
root: string,
options?: { commandWriter?: (commandPath: string) => Promise<void> },
) {
const workspace = path.join(root, "workspace");
const binDir = path.join(root, "bin");
const commandPath = path.join(binDir, "claude");
const capturePath = path.join(root, "capture.json");
const statePath = path.join(root, "state.txt");
await fs.mkdir(workspace, { recursive: true });
await fs.mkdir(binDir, { recursive: true });
await (options?.commandWriter ?? writeFakeClaudeCommand)(commandPath);
const previousHome = process.env.HOME;
const previousPath = process.env.PATH;
process.env.HOME = root;
process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ""}`;
return {
workspace, commandPath, capturePath, statePath,
restore: () => {
if (previousHome === undefined) delete process.env.HOME;
else process.env.HOME = previousHome;
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
},
};
}
describe("claude execute", () => {
/**
* Regression tests for https://github.com/paperclipai/paperclip/issues/2848
*
* --append-system-prompt-file should only be passed on fresh sessions.
* On resumed sessions the instructions are already in the session cache;
* re-injecting them wastes tokens and may be rejected by the CLI.
*/
it("passes --append-system-prompt-file on a fresh session when instructionsFile is set", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-fresh-"));
const { workspace, commandPath, capturePath, restore } = await setupExecuteEnv(root);
const instructionsFile = path.join(root, "instructions.md");
await fs.writeFile(instructionsFile, "# Agent instructions", "utf-8");
try {
await execute({
runId: "run-fresh",
agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },
runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null },
config: {
command: commandPath,
cwd: workspace,
env: { PAPERCLIP_TEST_CAPTURE_PATH: capturePath },
promptTemplate: "Do work.",
instructionsFilePath: instructionsFile,
},
context: {},
authToken: "tok",
onLog: async () => {},
onMeta: async () => {},
});
const captured = JSON.parse(await fs.readFile(capturePath, "utf-8"));
expect(captured.argv).toContain("--append-system-prompt-file");
} finally {
restore();
await fs.rm(root, { recursive: true, force: true });
}
});
it("omits --append-system-prompt-file on a resumed session even when instructionsFile is set", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-resume-"));
const { workspace, commandPath, capturePath, restore } = await setupExecuteEnv(root);
const instructionsFile = path.join(root, "instructions.md");
await fs.writeFile(instructionsFile, "# Agent instructions", "utf-8");
try {
await execute({
runId: "run-resume",
agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },
runtime: { sessionId: "claude-session-1", sessionParams: null, sessionDisplayId: null, taskKey: null },
config: {
command: commandPath,
cwd: workspace,
env: { PAPERCLIP_TEST_CAPTURE_PATH: capturePath },
promptTemplate: "Do work.",
instructionsFilePath: instructionsFile,
},
context: {},
authToken: "tok",
onLog: async () => {},
onMeta: async () => {},
});
const captured = JSON.parse(await fs.readFile(capturePath, "utf-8"));
expect(captured.argv).not.toContain("--append-system-prompt-file");
expect(captured.argv).toContain("--resume");
} finally {
restore();
await fs.rm(root, { recursive: true, force: true });
}
});
/**
* Regression tests for commandNotes accuracy (Greptile P2).
*
* commandNotes should only claim instructions were injected when the flag
* was actually passed — i.e. on fresh sessions, not resumed ones.
*/
it("commandNotes reports injection on a fresh session with instructionsFile", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-notes-fresh-"));
const { workspace, commandPath, restore } = await setupExecuteEnv(root);
const instructionsFile = path.join(root, "instructions.md");
await fs.writeFile(instructionsFile, "# Agent instructions", "utf-8");
let capturedNotes: string[] = [];
try {
await execute({
runId: "run-notes-fresh",
agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },
runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null },
config: {
command: commandPath,
cwd: workspace,
env: {},
promptTemplate: "Do work.",
instructionsFilePath: instructionsFile,
},
context: {},
authToken: "tok",
onLog: async () => {},
onMeta: async (meta) => { capturedNotes = (meta.commandNotes as string[]) ?? []; },
});
expect(capturedNotes.some((n) => n.includes("--append-system-prompt-file"))).toBe(true);
} finally {
restore();
await fs.rm(root, { recursive: true, force: true });
}
});
it("commandNotes is empty on a resumed session even when instructionsFile is set", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-notes-resume-"));
const { workspace, commandPath, restore } = await setupExecuteEnv(root);
const instructionsFile = path.join(root, "instructions.md");
await fs.writeFile(instructionsFile, "# Agent instructions", "utf-8");
let capturedNotes: string[] = ["sentinel"];
try {
await execute({
runId: "run-notes-resume",
agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },
runtime: { sessionId: "claude-session-1", sessionParams: null, sessionDisplayId: null, taskKey: null },
config: {
command: commandPath,
cwd: workspace,
env: {},
promptTemplate: "Do work.",
instructionsFilePath: instructionsFile,
},
context: {},
authToken: "tok",
onLog: async () => {},
onMeta: async (meta) => { capturedNotes = (meta.commandNotes as string[]) ?? []; },
});
expect(capturedNotes).toHaveLength(0);
} finally {
restore();
await fs.rm(root, { recursive: true, force: true });
}
});
/**
* Regression test for unnecessary file I/O on resumed sessions (Greptile P2).
*
* The combined agent-instructions.md temp file must NOT be written when
* resuming, since the instructions are already baked into the session cache.
*/
it("does not write agent-instructions temp file on a resumed session", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-io-resume-"));
const { workspace, commandPath, restore } = await setupExecuteEnv(root);
const instructionsFile = path.join(root, "instructions.md");
await fs.writeFile(instructionsFile, "# Agent instructions", "utf-8");
try {
await execute({
runId: "run-io-resume",
agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },
runtime: { sessionId: "claude-session-1", sessionParams: null, sessionDisplayId: null, taskKey: null },
config: {
command: commandPath,
cwd: workspace,
env: {},
promptTemplate: "Do work.",
instructionsFilePath: instructionsFile,
},
context: {},
authToken: "tok",
onLog: async () => {},
onMeta: async () => {},
});
// The skills dir lives under HOME/.paperclip/skills — verify no combined
// agent-instructions.md was written anywhere under root on a resume.
const allFiles = await fs.readdir(root, { recursive: true });
const tempInstructionsWritten = (allFiles as string[]).some((f) =>
f.includes("agent-instructions.md"),
);
expect(tempInstructionsWritten).toBe(false);
} finally {
restore();
await fs.rm(root, { recursive: true, force: true });
}
});
it("rebuilds the combined instructions file when an unknown resumed session falls back to fresh", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-resume-fallback-"));
const { workspace, commandPath, capturePath, statePath, restore } = await setupExecuteEnv(root, {
commandWriter: writeRetryThenSucceedClaudeCommand,
});
const instructionsFile = path.join(root, "instructions.md");
await fs.writeFile(instructionsFile, "# Agent instructions", "utf-8");
const metaEvents: Array<{ commandArgs: string[]; commandNotes: string[] }> = [];
try {
const result = await execute({
runId: "run-resume-fallback",
agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },
runtime: { sessionId: "claude-session-1", sessionParams: null, sessionDisplayId: null, taskKey: null },
config: {
command: commandPath,
cwd: workspace,
env: {
PAPERCLIP_TEST_CAPTURE_PATH: capturePath,
PAPERCLIP_TEST_STATE_PATH: statePath,
},
promptTemplate: "Do work.",
instructionsFilePath: instructionsFile,
},
context: {},
authToken: "tok",
onLog: async () => {},
onMeta: async (meta) => {
metaEvents.push({
commandArgs: ((meta.commandArgs as string[]) ?? []).slice(),
commandNotes: ((meta.commandNotes as string[]) ?? []).slice(),
});
},
});
const captured = JSON.parse(await fs.readFile(capturePath, "utf-8")) as Array<{
argv: string[];
appendedSystemPromptFilePath: string | null;
appendedSystemPromptFileContents: string | null;
}>;
expect(captured).toHaveLength(2);
expect(captured[0]?.argv).toContain("--resume");
expect(captured[0]?.argv).not.toContain("--append-system-prompt-file");
expect(captured[1]?.argv).not.toContain("--resume");
expect(captured[1]?.argv).toContain("--append-system-prompt-file");
expect(captured[1]?.appendedSystemPromptFilePath).toContain("agent-instructions.md");
expect(captured[1]?.appendedSystemPromptFilePath).not.toBe(instructionsFile);
expect(captured[1]?.appendedSystemPromptFileContents).toContain("# Agent instructions");
expect(captured[1]?.appendedSystemPromptFileContents).toContain(
`The above agent instructions were loaded from ${instructionsFile}. Resolve any relative file references from ${path.dirname(instructionsFile)}/.`,
);
expect(metaEvents).toHaveLength(2);
expect(metaEvents[0]?.commandNotes).toHaveLength(0);
expect(metaEvents[1]?.commandNotes.some((note) => note.includes("--append-system-prompt-file"))).toBe(true);
expect(result.sessionId).toBe("claude-session-2");
expect(result.clearSession).toBe(false);
} finally {
restore();
await fs.rm(root, { recursive: true, force: true });
}
});
it("logs HOME, CLAUDE_CONFIG_DIR, and the resolved executable path in invocation metadata", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-meta-"));
const workspace = path.join(root, "workspace");
@@ -403,4 +116,259 @@ describe("claude execute", () => {
await fs.rm(root, { recursive: true, force: true });
}
});
it("reuses a stable Paperclip-managed Claude prompt bundle across equivalent runs", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-bundle-"));
const workspace = path.join(root, "workspace");
const commandPath = path.join(root, "claude");
const capturePath1 = path.join(root, "capture-1.json");
const capturePath2 = path.join(root, "capture-2.json");
const instructionsPath = path.join(root, "AGENTS.md");
const paperclipHome = path.join(root, "paperclip-home");
await fs.mkdir(workspace, { recursive: true });
await fs.writeFile(instructionsPath, "You are managed instructions.\n", "utf8");
await writeFakeClaudeCommand(commandPath);
const previousHome = process.env.HOME;
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
process.env.HOME = root;
process.env.PAPERCLIP_HOME = paperclipHome;
try {
const first = await execute({
runId: "run-1",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Claude Coder",
adapterType: "claude_local",
adapterConfig: {},
},
runtime: {
sessionId: null,
sessionParams: null,
sessionDisplayId: null,
taskKey: null,
},
config: {
command: commandPath,
cwd: workspace,
instructionsFilePath: instructionsPath,
env: {
PAPERCLIP_TEST_CAPTURE_PATH: capturePath1,
},
promptTemplate: "Follow the paperclip heartbeat.",
},
context: {},
authToken: "run-jwt-token",
onLog: async () => {},
});
expect(first.exitCode).toBe(0);
expect(first.errorMessage).toBeNull();
expect(first.sessionParams).toMatchObject({
sessionId: "claude-session-1",
cwd: workspace,
});
expect(typeof first.sessionParams?.promptBundleKey).toBe("string");
const second = await execute({
runId: "run-2",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Claude Coder",
adapterType: "claude_local",
adapterConfig: {},
},
runtime: {
sessionId: null,
sessionParams: first.sessionParams ?? null,
sessionDisplayId: null,
taskKey: null,
},
config: {
command: commandPath,
cwd: workspace,
instructionsFilePath: instructionsPath,
env: {
PAPERCLIP_TEST_CAPTURE_PATH: capturePath2,
},
promptTemplate: "Follow the paperclip heartbeat.",
},
context: {
issueId: "issue-1",
taskId: "issue-1",
wakeReason: "issue_commented",
wakeCommentId: "comment-2",
paperclipWake: {
reason: "issue_commented",
issue: {
id: "issue-1",
identifier: "PAP-874",
title: "chat-speed issues",
status: "in_progress",
priority: "medium",
},
commentIds: ["comment-2"],
latestCommentId: "comment-2",
comments: [
{
id: "comment-2",
issueId: "issue-1",
body: "Second comment",
bodyTruncated: false,
createdAt: "2026-03-28T14:35:10.000Z",
author: { type: "user", id: "user-1" },
},
],
commentWindow: {
requestedCount: 1,
includedCount: 1,
missingCount: 0,
},
truncated: false,
fallbackFetchNeeded: false,
},
},
authToken: "run-jwt-token",
onLog: async () => {},
});
expect(second.exitCode).toBe(0);
expect(second.errorMessage).toBeNull();
const capture1 = JSON.parse(await fs.readFile(capturePath1, "utf8")) as CapturePayload;
const capture2 = JSON.parse(await fs.readFile(capturePath2, "utf8")) as CapturePayload;
const expectedRoot = path.join(
paperclipHome,
"instances",
"default",
"companies",
"company-1",
"claude-prompt-cache",
);
expect(capture1.addDir).toBeTruthy();
expect(capture1.addDir).toBe(capture2.addDir);
expect(capture1.instructionsFilePath).toBeTruthy();
expect(capture1.instructionsFilePath).toBe(capture2.instructionsFilePath);
expect(capture1.addDir?.startsWith(expectedRoot)).toBe(true);
expect(capture1.instructionsFilePath?.startsWith(expectedRoot)).toBe(true);
expect(capture1.instructionsContents).toContain("You are managed instructions.");
expect(capture1.instructionsContents).toContain(`The above agent instructions were loaded from ${instructionsPath}.`);
expect(capture1.skillEntries).toContain("paperclip");
expect(capture2.argv).toContain("--resume");
expect(capture2.argv).toContain("claude-session-1");
expect(capture2.prompt).toContain("## Paperclip Resume Delta");
expect(capture2.prompt).not.toContain("Follow the paperclip heartbeat.");
} finally {
if (previousHome === undefined) delete process.env.HOME;
else process.env.HOME = previousHome;
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
await fs.rm(root, { recursive: true, force: true });
}
});
it("starts a fresh Claude session when the stable prompt bundle changes", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-reset-"));
const workspace = path.join(root, "workspace");
const commandPath = path.join(root, "claude");
const capturePath1 = path.join(root, "capture-before.json");
const capturePath2 = path.join(root, "capture-after.json");
const instructionsPath = path.join(root, "AGENTS.md");
const paperclipHome = path.join(root, "paperclip-home");
const logs: string[] = [];
await fs.mkdir(workspace, { recursive: true });
await fs.writeFile(instructionsPath, "Version one instructions.\n", "utf8");
await writeFakeClaudeCommand(commandPath);
const previousHome = process.env.HOME;
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
process.env.HOME = root;
process.env.PAPERCLIP_HOME = paperclipHome;
try {
const first = await execute({
runId: "run-before",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Claude Coder",
adapterType: "claude_local",
adapterConfig: {},
},
runtime: {
sessionId: null,
sessionParams: null,
sessionDisplayId: null,
taskKey: null,
},
config: {
command: commandPath,
cwd: workspace,
instructionsFilePath: instructionsPath,
env: {
PAPERCLIP_TEST_CAPTURE_PATH: capturePath1,
},
promptTemplate: "Follow the paperclip heartbeat.",
},
context: {},
authToken: "run-jwt-token",
onLog: async () => {},
});
await fs.writeFile(instructionsPath, "Version two instructions.\n", "utf8");
const second = await execute({
runId: "run-after",
agent: {
id: "agent-1",
companyId: "company-1",
name: "Claude Coder",
adapterType: "claude_local",
adapterConfig: {},
},
runtime: {
sessionId: null,
sessionParams: first.sessionParams ?? null,
sessionDisplayId: null,
taskKey: null,
},
config: {
command: commandPath,
cwd: workspace,
instructionsFilePath: instructionsPath,
env: {
PAPERCLIP_TEST_CAPTURE_PATH: capturePath2,
},
promptTemplate: "Follow the paperclip heartbeat.",
},
context: {},
authToken: "run-jwt-token",
onLog: async (_stream, chunk) => {
logs.push(chunk);
},
});
expect(first.exitCode).toBe(0);
expect(second.exitCode).toBe(0);
expect(second.errorMessage).toBeNull();
const before = JSON.parse(await fs.readFile(capturePath1, "utf8")) as CapturePayload;
const after = JSON.parse(await fs.readFile(capturePath2, "utf8")) as CapturePayload;
expect(before.instructionsFilePath).not.toBe(after.instructionsFilePath);
expect(after.argv).not.toContain("--resume");
expect(after.prompt).toContain("Follow the paperclip heartbeat.");
expect(logs.join("")).toContain("will not be resumed with");
} finally {
if (previousHome === undefined) delete process.env.HOME;
else process.env.HOME = previousHome;
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
await fs.rm(root, { recursive: true, force: true });
}
});
});
@@ -123,7 +123,9 @@ describe("company portability routes", () => {
.send({ include: { company: true, agents: true, projects: true } });
expect(res.status).toBe(200);
expect(res.body.rootPath).toBe("paperclip");
expect(mockCompanyPortabilityService.previewExport).toHaveBeenCalledWith("11111111-1111-4111-8111-111111111111", {
include: { company: true, agents: true, projects: true },
});
});
it("rejects replace collision strategy on CEO-safe import routes", async () => {
@@ -0,0 +1,91 @@
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { heartbeatService } from "../services/heartbeat.ts";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres heartbeat list tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
describeEmbeddedPostgres("heartbeat list", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-list-");
db = createDb(tempDb.connectionString);
}, 20_000);
afterEach(async () => {
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(companies);
});
afterAll(async () => {
await tempDb?.cleanup();
});
it("returns runs even when the linked db schema lacks processGroupId", async () => {
const companyId = randomUUID();
const agentId = randomUUID();
const runId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
id: agentId,
companyId,
name: "CodexCoder",
role: "engineer",
status: "running",
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {},
permissions: {},
});
await db.insert(heartbeatRuns).values({
id: runId,
companyId,
agentId,
invocationSource: "assignment",
status: "running",
contextSnapshot: { issueId: randomUUID() },
});
const originalDescriptor = Object.getOwnPropertyDescriptor(heartbeatRuns, "processGroupId");
Object.defineProperty(heartbeatRuns, "processGroupId", {
value: undefined,
configurable: true,
writable: true,
});
try {
const runs = await heartbeatService(db).list(companyId, agentId, 5);
expect(runs).toHaveLength(1);
expect(runs[0]?.id).toBe(runId);
expect(runs[0]?.processGroupId ?? null).toBeNull();
} finally {
if (originalDescriptor) {
Object.defineProperty(heartbeatRuns, "processGroupId", originalDescriptor);
} else {
delete (heartbeatRuns as Record<string, unknown>).processGroupId;
}
}
});
});
@@ -49,10 +49,70 @@ function spawnAliveProcess() {
});
}
function isPidAlive(pid: number | null | undefined) {
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false;
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
async function waitForPidExit(pid: number, timeoutMs = 2_000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (!isPidAlive(pid)) return true;
await new Promise((resolve) => setTimeout(resolve, 50));
}
return !isPidAlive(pid);
}
async function spawnOrphanedProcessGroup() {
const leader = spawn(
process.execPath,
[
"-e",
[
"const { spawn } = require('node:child_process');",
"const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });",
"process.stdout.write(String(child.pid));",
"setTimeout(() => process.exit(0), 25);",
].join(" "),
],
{
detached: true,
stdio: ["ignore", "pipe", "ignore"],
},
);
let stdout = "";
leader.stdout?.on("data", (chunk) => {
stdout += String(chunk);
});
await new Promise<void>((resolve, reject) => {
leader.once("error", reject);
leader.once("exit", () => resolve());
});
const descendantPid = Number.parseInt(stdout.trim(), 10);
if (!Number.isInteger(descendantPid) || descendantPid <= 0) {
throw new Error(`Failed to capture orphaned descendant pid from detached process group: ${stdout}`);
}
return {
processPid: leader.pid ?? null,
processGroupId: leader.pid ?? null,
descendantPid,
};
}
describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
const childProcesses = new Set<ChildProcess>();
const cleanupPids = new Set<number>();
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-recovery-");
@@ -66,6 +126,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
child.kill("SIGKILL");
}
childProcesses.clear();
for (const pid of cleanupPids) {
try {
process.kill(pid, "SIGKILL");
} catch {
// Ignore already-dead cleanup targets.
}
}
cleanupPids.clear();
await db.delete(issues);
await db.delete(heartbeatRunEvents);
await db.delete(heartbeatRuns);
@@ -79,6 +147,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
child.kill("SIGKILL");
}
childProcesses.clear();
for (const pid of cleanupPids) {
try {
process.kill(pid, "SIGKILL");
} catch {
// Ignore already-dead cleanup targets.
}
}
cleanupPids.clear();
runningProcesses.clear();
await tempDb?.cleanup();
});
@@ -88,6 +164,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
agentStatus?: "paused" | "idle" | "running";
runStatus?: "running" | "queued" | "failed";
processPid?: number | null;
processGroupId?: number | null;
processLossRetryCount?: number;
includeIssue?: boolean;
runErrorCode?: string | null;
@@ -143,6 +220,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
wakeupRequestId,
contextSnapshot: input?.includeIssue === false ? {} : { issueId },
processPid: input?.processPid ?? null,
processGroupId: input?.processGroupId ?? null,
processLossRetryCount: input?.processLossRetryCount ?? 0,
errorCode: input?.runErrorCode ?? null,
error: input?.runError ?? null,
@@ -228,6 +306,45 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(issue?.checkoutRunId).toBe(runId);
});
it.skipIf(process.platform === "win32")("reaps orphaned descendant process groups when the parent pid is already gone", async () => {
const orphan = await spawnOrphanedProcessGroup();
cleanupPids.add(orphan.descendantPid);
expect(isPidAlive(orphan.descendantPid)).toBe(true);
const { agentId, runId, issueId } = await seedRunFixture({
processPid: orphan.processPid,
processGroupId: orphan.processGroupId,
});
const heartbeat = heartbeatService(db);
const result = await heartbeat.reapOrphanedRuns();
expect(result.reaped).toBe(1);
expect(result.runIds).toEqual([runId]);
expect(await waitForPidExit(orphan.descendantPid, 2_000)).toBe(true);
const runs = await db
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.agentId, agentId));
expect(runs).toHaveLength(2);
const failedRun = runs.find((row) => row.id === runId);
expect(failedRun?.status).toBe("failed");
expect(failedRun?.errorCode).toBe("process_lost");
expect(failedRun?.error).toContain("descendant process group");
const retryRun = runs.find((row) => row.id !== runId);
expect(retryRun?.status).toBe("queued");
const issue = await db
.select()
.from(issues)
.where(eq(issues.id, issueId))
.then((rows) => rows[0] ?? null);
expect(issue?.executionRunId).toBe(retryRun?.id ?? null);
});
it("does not queue a second retry after the first process-loss retry was already used", async () => {
const { agentId, runId, issueId } = await seedRunFixture({
processPid: 999_999_999,
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
summarizeHeartbeatRunResultJson,
buildHeartbeatRunIssueComment,
mergeHeartbeatRunResultJson,
} from "../services/heartbeat-run-summary.js";
describe("summarizeHeartbeatRunResultJson", () => {
@@ -55,3 +56,35 @@ describe("buildHeartbeatRunIssueComment", () => {
expect(buildHeartbeatRunIssueComment({ costUsd: 1.2 })).toBeNull();
});
});
describe("mergeHeartbeatRunResultJson", () => {
it("adds adapter summaries into stored result json for comment posting", () => {
const merged = mergeHeartbeatRunResultJson(
{ stdout: "raw stdout", stderr: "" },
"## Summary\n\n1. first thing\n2. second thing",
);
expect(merged).toEqual({
stdout: "raw stdout",
stderr: "",
summary: "## Summary\n\n1. first thing\n2. second thing",
});
expect(buildHeartbeatRunIssueComment(merged)).toBe("## Summary\n\n1. first thing\n2. second thing");
});
it("creates a result payload when only a summary exists", () => {
expect(mergeHeartbeatRunResultJson(null, "done")).toEqual({ summary: "done" });
});
it("does not overwrite an explicit summary already returned by the adapter", () => {
expect(
mergeHeartbeatRunResultJson(
{ summary: "adapter result", stdout: "raw stdout" },
"fallback summary",
),
).toEqual({
summary: "adapter result",
stdout: "raw stdout",
});
});
});
@@ -413,33 +413,45 @@ describe("issue execution policy transitions", () => {
const policy = twoStagePolicy();
const reviewStageId = policy.stages[0].id;
it("non-participant cannot advance the active stage", () => {
expect(() =>
applyIssueExecutionPolicyTransition({
issue: {
status: "in_review",
assigneeAgentId: qaAgentId,
assigneeUserId: null,
executionPolicy: policy,
executionState: {
status: "pending",
currentStageId: reviewStageId,
currentStageIndex: 0,
currentStageType: "review",
currentParticipant: { type: "agent", agentId: qaAgentId },
returnAssignee: { type: "agent", agentId: coderAgentId },
completedStageIds: [],
lastDecisionId: null,
lastDecisionOutcome: null,
},
it("non-participant stage updates are coerced back to the active stage", () => {
const result = applyIssueExecutionPolicyTransition({
issue: {
status: "in_review",
assigneeAgentId: qaAgentId,
assigneeUserId: null,
executionPolicy: policy,
executionState: {
status: "pending",
currentStageId: reviewStageId,
currentStageIndex: 0,
currentStageType: "review",
currentParticipant: { type: "agent", agentId: qaAgentId },
returnAssignee: { type: "agent", agentId: coderAgentId },
completedStageIds: [],
lastDecisionId: null,
lastDecisionOutcome: null,
},
policy,
requestedStatus: "done",
requestedAssigneePatch: { assigneeUserId: boardUserId },
actor: { agentId: coderAgentId },
commentBody: "Trying to bypass review",
}),
).toThrow("Only the active reviewer or approver can advance");
},
policy,
requestedStatus: "done",
requestedAssigneePatch: { assigneeUserId: boardUserId },
actor: { agentId: coderAgentId },
commentBody: "Trying to bypass review",
});
expect(result.patch).toMatchObject({
status: "in_review",
assigneeAgentId: qaAgentId,
assigneeUserId: null,
executionState: {
status: "pending",
currentStageId: reviewStageId,
currentStageType: "review",
currentParticipant: { type: "agent", agentId: qaAgentId },
returnAssignee: { type: "agent", agentId: coderAgentId },
},
});
expect(result.decision).toBeUndefined();
});
it("non-participant can still post non-advancing updates", () => {
@@ -76,6 +76,21 @@ describe("TelemetryClient periodic flush", () => {
expect(fetch).not.toHaveBeenCalled();
});
it("falls back to the api gateway ingest url when the default hostname fails", async () => {
vi.mocked(fetch)
.mockRejectedValueOnce(new TypeError("getaddrinfo ENOTFOUND telemetry.paperclip.ing"))
.mockResolvedValueOnce({ ok: true });
const client = makeClient({ endpoint: undefined });
client.track("install.started");
await client.flush();
expect(fetch).toHaveBeenCalledTimes(2);
expect(vi.mocked(fetch).mock.calls[0]?.[0]).toBe("https://telemetry.paperclip.ing/ingest");
expect(vi.mocked(fetch).mock.calls[1]?.[0]).toBe("https://rusqrrg391.execute-api.us-east-1.amazonaws.com/ingest");
});
it("startPeriodicFlush is idempotent", () => {
const client = makeClient();
client.startPeriodicFlush(1000);
+1 -1
View File
@@ -13,7 +13,7 @@ type BuildInvocationEnvForLogsOptions = {
resolvedCommandEnvKey?: string;
};
export const runningProcesses: Map<string, { child: ChildProcess; graceSec: number }> =
export const runningProcesses: Map<string, { child: ChildProcess; graceSec: number; processGroupId: number | null }> =
serverUtils.runningProcesses;
export const MAX_CAPTURE_BYTES = serverUtils.MAX_CAPTURE_BYTES;
export const MAX_EXCERPT_BYTES = serverUtils.MAX_EXCERPT_BYTES;
+1 -7
View File
@@ -67,13 +67,7 @@ export function deriveAuthTrustedOrigins(config: Config): string[] {
export function createBetterAuthInstance(db: Db, config: Config, trustedOrigins?: string[]): BetterAuthInstance {
const baseUrl = config.authBaseUrlMode === "explicit" ? config.authPublicBaseUrl : undefined;
const secret = process.env.BETTER_AUTH_SECRET ?? process.env.PAPERCLIP_AGENT_JWT_SECRET;
if (!secret) {
throw new Error(
"BETTER_AUTH_SECRET (or PAPERCLIP_AGENT_JWT_SECRET) must be set. " +
"For local development, set BETTER_AUTH_SECRET=paperclip-dev-secret in your .env file.",
);
}
const secret = process.env.BETTER_AUTH_SECRET ?? process.env.PAPERCLIP_AGENT_JWT_SECRET ?? "paperclip-dev-secret";
const effectiveTrustedOrigins = trustedOrigins ?? deriveAuthTrustedOrigins(config);
const publicUrl = process.env.PAPERCLIP_PUBLIC_URL ?? baseUrl;
+7
View File
@@ -475,6 +475,13 @@ export async function startServer(): Promise<StartedServer> {
resolveBetterAuthSession,
resolveBetterAuthSessionFromHeaders,
} = await import("./auth/better-auth.js");
const betterAuthSecret =
process.env.BETTER_AUTH_SECRET?.trim() ?? process.env.PAPERCLIP_AGENT_JWT_SECRET?.trim();
if (!betterAuthSecret) {
throw new Error(
"authenticated mode requires BETTER_AUTH_SECRET (or PAPERCLIP_AGENT_JWT_SECRET) to be set",
);
}
const derivedTrustedOrigins = deriveAuthTrustedOrigins(config);
const envTrustedOrigins = (process.env.BETTER_AUTH_TRUSTED_ORIGINS ?? "")
.split(",")
+4 -5
View File
@@ -2437,15 +2437,14 @@ export function agentRoutes(db: Db) {
}
assertCompanyAccess(req, issue.companyId);
let run = issue.executionRunId ? await heartbeat.getRun(issue.executionRunId) : null;
let run = issue.executionRunId ? await heartbeat.getRunIssueSummary(issue.executionRunId) : null;
if (run && run.status !== "queued" && run.status !== "running") {
run = null;
}
if (!run && issue.assigneeAgentId && issue.status === "in_progress") {
const candidateRun = await heartbeat.getActiveRunForAgent(issue.assigneeAgentId);
const candidateContext = asRecord(candidateRun?.contextSnapshot);
const candidateIssueId = asNonEmptyString(candidateContext?.issueId);
const candidateRun = await heartbeat.getActiveRunIssueSummaryForAgent(issue.assigneeAgentId);
const candidateIssueId = asNonEmptyString(candidateRun?.issueId);
if (candidateRun && candidateIssueId === issue.id) {
run = candidateRun;
}
@@ -2462,7 +2461,7 @@ export function agentRoutes(db: Db) {
}
res.json({
...redactCurrentUserValue(run, await getCurrentUserRedactionOptions()),
...run,
agentId: agent.id,
agentName: agent.name,
adapterType: agent.adapterType,
@@ -13,6 +13,34 @@ function readCommentText(value: unknown) {
return trimmed.length > 0 ? trimmed : null;
}
export function mergeHeartbeatRunResultJson(
resultJson: Record<string, unknown> | null | undefined,
summary: string | null | undefined,
): Record<string, unknown> | null {
const normalizedSummary = readCommentText(summary);
const baseResult =
resultJson && typeof resultJson === "object" && !Array.isArray(resultJson)
? resultJson
: null;
if (!baseResult) {
return normalizedSummary ? { summary: normalizedSummary } : null;
}
if (!normalizedSummary) {
return baseResult;
}
if (readCommentText(baseResult.summary)) {
return baseResult;
}
return {
...baseResult,
summary: normalizedSummary,
};
}
export function summarizeHeartbeatRunResultJson(
resultJson: Record<string, unknown> | null | undefined,
): Record<string, unknown> | null {
+139 -41
View File
@@ -32,7 +32,11 @@ import { companySkillService } from "./company-skills.js";
import { budgetService, type BudgetEnforcementScope } from "./budgets.js";
import { secretService } from "./secrets.js";
import { resolveDefaultAgentWorkspaceDir, resolveManagedProjectWorkspaceDir } from "../home-paths.js";
import { buildHeartbeatRunIssueComment, summarizeHeartbeatRunResultJson } from "./heartbeat-run-summary.js";
import {
buildHeartbeatRunIssueComment,
mergeHeartbeatRunResultJson,
summarizeHeartbeatRunResultJson,
} from "./heartbeat-run-summary.js";
import {
buildWorkspaceReadyComment,
cleanupExecutionWorkspaceArtifacts,
@@ -47,6 +51,7 @@ import {
import { issueService } from "./issues.js";
import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js";
import { workspaceOperationService } from "./workspace-operations.js";
import { isProcessGroupAlive, terminateLocalService } from "./local-service-supervisor.js";
import {
buildExecutionWorkspaceAdapterConfig,
gateProjectExecutionWorkspacePolicy,
@@ -261,6 +266,9 @@ async function ensureManagedProjectWorkspace(input: {
}
}
const heartbeatRunProcessGroupIdColumn =
heartbeatRuns.processGroupId ?? sql<number | null>`NULL`.as("processGroupId");
const heartbeatRunListColumns = {
id: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
@@ -288,6 +296,7 @@ const heartbeatRunListColumns = {
errorCode: heartbeatRuns.errorCode,
externalRunId: heartbeatRuns.externalRunId,
processPid: heartbeatRuns.processPid,
processGroupId: heartbeatRunProcessGroupIdColumn,
processStartedAt: heartbeatRuns.processStartedAt,
retryOfRunId: heartbeatRuns.retryOfRunId,
processLossRetryCount: heartbeatRuns.processLossRetryCount,
@@ -296,6 +305,18 @@ const heartbeatRunListColumns = {
updatedAt: heartbeatRuns.updatedAt,
} as const;
const heartbeatRunIssueSummaryColumns = {
id: heartbeatRuns.id,
status: heartbeatRuns.status,
invocationSource: heartbeatRuns.invocationSource,
triggerDetail: heartbeatRuns.triggerDetail,
startedAt: heartbeatRuns.startedAt,
finishedAt: heartbeatRuns.finishedAt,
createdAt: heartbeatRuns.createdAt,
agentId: heartbeatRuns.agentId,
issueId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("issueId"),
} as const;
function appendExcerpt(prev: string, chunk: string) {
return appendWithCap(prev, chunk, MAX_EXCERPT_BYTES);
}
@@ -707,18 +728,6 @@ export function shouldResetTaskSessionForWake(
return false;
}
function shouldRequireIssueCommentForWake(
contextSnapshot: Record<string, unknown> | null | undefined,
) {
const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
return (
wakeReason === "issue_assigned" ||
wakeReason === "execution_review_requested" ||
wakeReason === "execution_approval_requested" ||
wakeReason === "execution_changes_requested"
);
}
export function formatRuntimeWorkspaceWarningLog(warning: string) {
return {
stream: "stdout" as const,
@@ -1026,6 +1035,46 @@ function isProcessAlive(pid: number | null | undefined) {
}
}
async function terminateHeartbeatRunProcess(input: {
pid: number | null | undefined;
processGroupId: number | null | undefined;
graceMs?: number;
}) {
const pid = input.pid ?? null;
const processGroupId = input.processGroupId ?? null;
if (typeof pid !== "number" && typeof processGroupId !== "number") return;
await terminateLocalService(
{
pid:
typeof pid === "number" && Number.isInteger(pid) && pid > 0
? pid
: (processGroupId ?? 0),
processGroupId:
typeof processGroupId === "number" && Number.isInteger(processGroupId) && processGroupId > 0
? processGroupId
: null,
},
input.graceMs ? { forceAfterMs: input.graceMs } : undefined,
);
}
function buildProcessLossMessage(run: {
processPid: number | null;
processGroupId: number | null;
}, options?: { descendantOnly?: boolean }) {
if (options?.descendantOnly && run.processGroupId) {
return `Process lost -- parent pid ${run.processPid ?? "unknown"} exited, but descendant process group ${run.processGroupId} was still alive and was terminated`;
}
if (run.processPid) {
return `Process lost -- child pid ${run.processPid} is no longer running`;
}
if (run.processGroupId) {
return `Process lost -- process group ${run.processGroupId} is no longer running`;
}
return "Process lost -- server may have restarted";
}
function truncateDisplayId(value: string | null | undefined, max = 128) {
if (!value) return null;
return value.length > max ? value.slice(0, max) : value;
@@ -1824,13 +1873,14 @@ export function heartbeatService(db: Db) {
async function persistRunProcessMetadata(
runId: string,
meta: { pid: number; startedAt: string },
meta: { pid: number; processGroupId: number | null; startedAt: string },
) {
const startedAt = new Date(meta.startedAt);
return db
.update(heartbeatRuns)
.set({
processPid: meta.pid,
processGroupId: meta.processGroupId,
processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt,
updatedAt: new Date(),
})
@@ -2047,17 +2097,6 @@ export function heartbeatService(db: Db) {
return { outcome: "retry_exhausted" as const, queuedRun: null };
}
if (!shouldRequireIssueCommentForWake(contextSnapshot)) {
if (run.issueCommentStatus !== "not_applicable") {
await patchRunIssueCommentStatus(run.id, {
issueCommentStatus: "not_applicable",
issueCommentSatisfiedByCommentId: null,
issueCommentRetryQueuedAt: null,
});
}
return { outcome: "not_applicable" as const, queuedRun: null };
}
const queuedRun = await enqueueMissingIssueCommentRetry(run, agent, issueId);
if (queuedRun) {
await appendRunEvent(run, await nextRunEventSeq(run.id), {
@@ -2356,7 +2395,9 @@ export function heartbeatService(db: Db) {
}
const tracksLocalChild = isTrackedLocalChildProcessAdapter(adapterType);
if (tracksLocalChild && run.processPid && isProcessAlive(run.processPid)) {
const processPidAlive = tracksLocalChild && run.processPid && isProcessAlive(run.processPid);
const processGroupAlive = tracksLocalChild && run.processGroupId && isProcessGroupAlive(run.processGroupId);
if (processPidAlive) {
if (run.errorCode !== DETACHED_PROCESS_ERROR_CODE) {
const detachedMessage = `Lost in-memory process handle, but child pid ${run.processPid} is still alive`;
const detachedRun = await setRunStatus(run.id, "running", {
@@ -2378,10 +2419,17 @@ export function heartbeatService(db: Db) {
continue;
}
const shouldRetry = tracksLocalChild && !!run.processPid && (run.processLossRetryCount ?? 0) < 1;
const baseMessage = run.processPid
? `Process lost -- child pid ${run.processPid} is no longer running`
: "Process lost -- server may have restarted";
let descendantOnlyCleanup = false;
if (processGroupAlive) {
descendantOnlyCleanup = true;
await terminateHeartbeatRunProcess({
pid: run.processPid,
processGroupId: run.processGroupId,
});
}
const shouldRetry = tracksLocalChild && (!!run.processPid || !!run.processGroupId) && (run.processLossRetryCount ?? 0) < 1;
const baseMessage = buildProcessLossMessage(run, descendantOnlyCleanup ? { descendantOnly: true } : undefined);
let finalizedRun = await setRunStatus(run.id, "failed", {
error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
@@ -2414,6 +2462,8 @@ export function heartbeatService(db: Db) {
: baseMessage,
payload: {
...(run.processPid ? { processPid: run.processPid } : {}),
...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),
...(descendantOnlyCleanup ? { descendantOnlyCleanup: true } : {}),
...(retriedRun ? { retryRunId: retriedRun.id } : {}),
},
});
@@ -3179,7 +3229,14 @@ export function heartbeatService(db: Db) {
onLog,
onMeta: onAdapterMeta,
onSpawn: async (meta) => {
await persistRunProcessMetadata(run.id, meta);
await persistRunProcessMetadata(run.id, {
pid: meta.pid,
processGroupId:
"processGroupId" in meta && typeof meta.processGroupId === "number"
? meta.processGroupId
: null,
startedAt: meta.startedAt,
});
},
authToken: authToken ?? undefined,
});
@@ -3299,6 +3356,11 @@ export function heartbeatService(db: Db) {
} as Record<string, unknown>)
: null;
const persistedResultJson = mergeHeartbeatRunResultJson(
adapterResult.resultJson ?? null,
adapterResult.summary ?? null,
);
await setRunStatus(run.id, status, {
finishedAt: new Date(),
error:
@@ -3319,7 +3381,7 @@ export function heartbeatService(db: Db) {
exitCode: adapterResult.exitCode,
signal: adapterResult.signal,
usageJson,
resultJson: adapterResult.resultJson ?? null,
resultJson: persistedResultJson,
sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId,
stdoutExcerpt,
stderrExcerpt,
@@ -3347,7 +3409,7 @@ export function heartbeatService(db: Db) {
});
if (issueId && outcome === "succeeded") {
try {
const issueComment = buildHeartbeatRunIssueComment(adapterResult.resultJson ?? null);
const issueComment = buildHeartbeatRunIssueComment(persistedResultJson);
if (issueComment) {
await issuesSvc.addComment(issueId, issueComment, { agentId: agent.id, runId: finalizedRun.id });
}
@@ -4242,13 +4304,16 @@ export function heartbeatService(db: Db) {
const running = runningProcesses.get(run.id);
if (running) {
running.child.kill("SIGTERM");
const graceMs = Math.max(1, running.graceSec) * 1000;
setTimeout(() => {
if (!running.child.killed) {
running.child.kill("SIGKILL");
}
}, graceMs);
await terminateHeartbeatRunProcess({
pid: running.child.pid ?? run.processPid,
processGroupId: running.processGroupId ?? run.processGroupId,
graceMs: Math.max(1, running.graceSec) * 1000,
});
} else if (run.processPid || run.processGroupId) {
await terminateHeartbeatRunProcess({
pid: run.processPid,
processGroupId: run.processGroupId,
});
}
const cancelled = await setRunStatus(run.id, "cancelled", {
@@ -4298,8 +4363,17 @@ export function heartbeatService(db: Db) {
const running = runningProcesses.get(run.id);
if (running) {
running.child.kill("SIGTERM");
await terminateHeartbeatRunProcess({
pid: running.child.pid ?? run.processPid,
processGroupId: running.processGroupId ?? run.processGroupId,
graceMs: Math.max(1, running.graceSec) * 1000,
});
runningProcesses.delete(run.id);
} else if (run.processPid || run.processGroupId) {
await terminateHeartbeatRunProcess({
pid: run.processPid,
processGroupId: run.processGroupId,
});
}
await releaseIssueExecutionAndPromote(run);
}
@@ -4515,6 +4589,15 @@ export function heartbeatService(db: Db) {
cancelBudgetScopeWork,
getRunIssueSummary: async (runId: string) => {
const [run] = await db
.select(heartbeatRunIssueSummaryColumns)
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.limit(1);
return run ?? null;
},
getActiveRunForAgent: async (agentId: string) => {
const [run] = await db
.select()
@@ -4529,5 +4612,20 @@ export function heartbeatService(db: Db) {
.limit(1);
return run ?? null;
},
getActiveRunIssueSummaryForAgent: async (agentId: string) => {
const [run] = await db
.select(heartbeatRunIssueSummaryColumns)
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.agentId, agentId),
eq(heartbeatRuns.status, "running"),
),
)
.orderBy(desc(heartbeatRuns.startedAt))
.limit(1);
return run ?? null;
},
};
}
+5 -11
View File
@@ -393,19 +393,13 @@ export function applyIssueExecutionPolicyTransition(input: TransitionInput): Tra
}
}
const attemptedStageAdvance =
(requestedStatus !== undefined && requestedStatus !== "in_review") ||
(requestedAssigneePatchProvided && !principalsEqual(explicitAssignee, currentParticipant));
const stageStateDrifted =
if (
input.issue.status !== "in_review" ||
!principalsEqual(currentAssignee, currentParticipant) ||
!principalsEqual(existingState?.currentParticipant ?? null, currentParticipant);
if (attemptedStageAdvance && !stageStateDrifted) {
throw unprocessable("Only the active reviewer or approver can advance the current execution stage");
}
if (stageStateDrifted) {
!principalsEqual(existingState?.currentParticipant ?? null, currentParticipant) ||
(requestedStatus !== undefined && requestedStatus !== "in_review") ||
(requestedAssigneePatchProvided && !principalsEqual(explicitAssignee, currentParticipant))
) {
buildPendingStagePatch({
patch,
previous: existingState,
@@ -221,6 +221,17 @@ export function isPidAlive(pid: number) {
}
}
export function isProcessGroupAlive(processGroupId: number | null | undefined) {
if (process.platform === "win32") return false;
if (typeof processGroupId !== "number" || !Number.isInteger(processGroupId) || processGroupId <= 0) return false;
try {
process.kill(-processGroupId, 0);
return true;
} catch {
return false;
}
}
async function isLikelyMatchingCommand(record: LocalServiceRegistryRecord) {
if (process.platform === "win32") return true;
try {
@@ -296,13 +307,19 @@ export async function terminateLocalService(
const deadline = Date.now() + (opts?.forceAfterMs ?? 2_000);
while (Date.now() < deadline) {
if (!isPidAlive(record.pid)) {
const targetAlive = targetProcessGroup
? isProcessGroupAlive(record.processGroupId)
: isPidAlive(record.pid);
if (!targetAlive) {
return;
}
await delay(100);
}
if (!isPidAlive(record.pid)) return;
const stillAlive = targetProcessGroup
? isProcessGroupAlive(record.processGroupId)
: isPidAlive(record.pid);
if (!stillAlive) return;
try {
if (targetProcessGroup) {
process.kill(-record.processGroupId!, "SIGKILL");