mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 16:27:18 +02:00
Ensure worktree execution starts only after activation (#9374)
## Thinking Path > - Paperclip is the open-source control plane people use to manage AI agents and their work. > - Its scheduler, routines, and heartbeat services decide when agents automatically begin work. > - Experimental per-worktree execution is useful for isolated development, but enabling it previously allowed automatic services to consider an existing backlog. > - A worktree activation must therefore create a durable eligibility boundary rather than merely toggle execution on. > - This pull request records an activation cutoff and applies it consistently to automatic routine and heartbeat dispatch. > - The result is that an enabled worktree executes only work created after its own activation, while non-worktree behavior remains unchanged. ## Linked Issues or Issue Description **Problem type:** Bug / safety regression **Summary:** Enabling experimental run execution in an existing worktree could start automatic scheduler, routine, watchdog, and heartbeat activity for work created before that worktree was explicitly armed. **Expected behavior:** A worktree that has execution enabled only considers automatically dispatched work created on or after its activation timestamp. Ambiguous activation state fails closed. Non-worktree instances keep their existing behavior. **Related public work:** Refs #8275 (runtime worktree policy gating); this PR adds an activation-time boundary for automatic execution rather than changing the general runtime policy. ## What Changed - Persist a worktree execution activation timestamp and originating instance ID; stamp them only when the experimental toggle changes from disabled to enabled. - Resolve activation state fail-closed when the cutoff is missing, invalid, disabled, or belongs to another instance. - Gate automatic routine scheduling, webhooks, watchdog activity, and heartbeat selection at the activation cutoff; manual runs remain available. - Share the canonical worktree truthy-environment helper across routine dispatch and agent inbox filtering. - Add cutoff and truthy-runtime regression coverage, plus experimental-settings UI states that explain armed and suppressed execution. ## Verification - `pnpm exec vitest run server/src/__tests__/routines-service.test.ts server/src/__tests__/instance-settings-service.test.ts` — passes: 2 files, 60 tests. - `pnpm --filter @paperclipai/server typecheck` — passes. - Existing CI completed successfully before the follow-up review fixes; this branch was rebased onto the latest `origin/master` before retesting. ## Risks - **Behavioral:** Automatic worktree execution is intentionally more restrictive; pre-existing work is suppressed until newly created after activation. - **Operational:** A malformed or cross-instance activation record fails closed, requiring an operator to disable and re-enable the experimental toggle on the intended worktree. - **Compatibility:** The worktree environment now accepts all canonical truthy values (`1`, `true`, `yes`, and `on`) consistently; non-worktree instances are unaffected. - **Branch metadata:** This existing execution-workspace branch predates the current naming rule and cannot be renamed under this task's workspace contract; the code and PR title do not include internal ticket references. > `ROADMAP.md` was checked; this targeted execution-safety fix does not duplicate planned core work. ## Model Used - Anthropic Claude Code — assisted with the original implementation; exact model identifier and context window were not recorded in the repository metadata. - OpenAI Codex CLI — assisted with PR preparation and review fixes; exact model identifier and context window are not exposed in this execution environment. Used with terminal tooling, code editing, and targeted test execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [ ] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
22 files changed
+1082
-69
No files matched your search
@@ -1249,4 +1249,14 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
.where(eq(issueRelations.relatedIssueId, blockedIssueId));
|
||||
expect(blockers.some((row) => row.blockerIssueId === escalations[0]!.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("handles an armed cutoff when no liveness findings exist", async () => {
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness({
|
||||
issueCreatedAtGte: new Date(),
|
||||
});
|
||||
|
||||
expect(result.findings).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -133,6 +133,16 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {
|
||||
return { companyId, agentId, issueId };
|
||||
}
|
||||
|
||||
async function armWorktreeRunExecution(cutoff: Date) {
|
||||
await instanceSettingsService(db, {
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "test-worktree",
|
||||
},
|
||||
now: () => cutoff,
|
||||
}).updateExperimental({ enableWorktreeRunExecution: true });
|
||||
}
|
||||
|
||||
async function waitForCompletedRun(runId: string, agentId: string) {
|
||||
let latestStatus: string | null = null;
|
||||
let latestLastRunId: string | null = null;
|
||||
@@ -243,6 +253,48 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {
|
||||
expect(runningCount).toBe(0);
|
||||
});
|
||||
|
||||
it("skips pre-cutoff system wakes but allows user wakes in an armed worktree", async () => {
|
||||
const { agentId, issueId } = await insertAgentAndIssue();
|
||||
await armWorktreeRunExecution(new Date(Date.now() + 1_000));
|
||||
const heartbeat = heartbeatService(db, {
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "test-worktree",
|
||||
},
|
||||
});
|
||||
|
||||
const systemRun = await heartbeat.wakeup(agentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
payload: { issueId },
|
||||
contextSnapshot: { issueId },
|
||||
requestedByActorType: "system",
|
||||
});
|
||||
expect(systemRun).toBeNull();
|
||||
|
||||
const skippedWake = await db
|
||||
.select({ reason: agentWakeupRequests.reason, payload: agentWakeupRequests.payload })
|
||||
.from(agentWakeupRequests)
|
||||
.orderBy(sql`${agentWakeupRequests.createdAt} desc`)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(skippedWake).toMatchObject({ reason: "heartbeat.worktree_execution_cutoff" });
|
||||
expect(skippedWake?.payload).toMatchObject({
|
||||
heartbeatSkip: { reason: "worktree_execution_cutoff", issueId },
|
||||
});
|
||||
|
||||
const userRun = await heartbeat.wakeup(agentId, {
|
||||
source: "on_demand",
|
||||
triggerDetail: "user",
|
||||
payload: { issueId },
|
||||
contextSnapshot: { issueId, skipIssueComment: true },
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "operator",
|
||||
});
|
||||
expect(userRun).not.toBeNull();
|
||||
await heartbeat.waitForRunExecutionDrain(userRun!.id);
|
||||
}, 10_000);
|
||||
|
||||
it("still creates live-plane assignment runs when suppression is not active", async () => {
|
||||
const { agentId, issueId } = await insertAgentAndIssue();
|
||||
await db
|
||||
|
||||
@@ -89,6 +89,9 @@ describe("instance settings routes", () => {
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
enableWorkspaceBranchReconcileForward: true,
|
||||
enableWorkspaceDirtyQuarantineRepair: true,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
},
|
||||
createdAt: "2026-06-20T00:00:00.000Z",
|
||||
@@ -114,6 +117,9 @@ describe("instance settings routes", () => {
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
enableWorkspaceBranchReconcileForward: true,
|
||||
enableWorkspaceDirtyQuarantineRepair: true,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
});
|
||||
mockInstanceSettingsService.update.mockResolvedValue({
|
||||
@@ -138,6 +144,9 @@ describe("instance settings routes", () => {
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
enableWorkspaceBranchReconcileForward: true,
|
||||
enableWorkspaceDirtyQuarantineRepair: true,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
},
|
||||
createdAt: "2026-06-20T00:00:00.000Z",
|
||||
@@ -168,6 +177,9 @@ describe("instance settings routes", () => {
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
enableWorkspaceBranchReconcileForward: true,
|
||||
enableWorkspaceDirtyQuarantineRepair: true,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
},
|
||||
});
|
||||
@@ -226,6 +238,9 @@ describe("instance settings routes", () => {
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
enableWorkspaceBranchReconcileForward: true,
|
||||
enableWorkspaceDirtyQuarantineRepair: true,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
});
|
||||
|
||||
@@ -240,6 +255,28 @@ describe("instance settings routes", () => {
|
||||
expect(mockLogActivity).toHaveBeenCalledTimes(2);
|
||||
}, 10_000);
|
||||
|
||||
it("strips server-managed worktree run execution fields before updating experimental settings", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
source: "local_implicit",
|
||||
isInstanceAdmin: true,
|
||||
});
|
||||
|
||||
await request(app)
|
||||
.patch("/api/instance/settings/experimental")
|
||||
.send({
|
||||
enableWorktreeRunExecution: true,
|
||||
worktreeRunExecutionActivatedAt: "2026-07-10T12:00:00.000Z",
|
||||
worktreeRunExecutionActivationInstanceId: "copied-instance",
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
expect(mockInstanceSettingsService.updateExperimental).toHaveBeenCalledWith({
|
||||
enableWorktreeRunExecution: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("allows local board users to read and update the instance default environment", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeExperimentalSettings } from "../services/instance-settings.js";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { InstanceExperimentalSettings } from "@paperclipai/shared";
|
||||
import {
|
||||
applyExperimentalSettingsPatch,
|
||||
normalizeExperimentalSettings,
|
||||
resolveWorktreeRunExecutionActivationState,
|
||||
} from "../services/instance-settings.js";
|
||||
|
||||
describe("instance settings service", () => {
|
||||
it("ignores retired experimental flags without resetting current settings", () => {
|
||||
@@ -39,6 +44,8 @@ describe("instance settings service", () => {
|
||||
enableWorkspaceBranchReconcileForward: true,
|
||||
enableWorkspaceDirtyQuarantineRepair: false,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 48,
|
||||
});
|
||||
});
|
||||
@@ -115,4 +122,208 @@ describe("instance settings service", () => {
|
||||
expect(normalizeExperimentalSettings({}).enableBuiltInAgents).toBe(false);
|
||||
expect(normalizeExperimentalSettings({ enableExternalObjects: true }).enableBuiltInAgents).toBe(false);
|
||||
});
|
||||
|
||||
it("sets worktree run execution activation fields on a false to true transition", () => {
|
||||
const activatedAt = new Date("2026-07-10T12:00:00.000Z");
|
||||
|
||||
const next = applyExperimentalSettingsPatch(
|
||||
{ enableWorktreeRunExecution: false },
|
||||
{ enableWorktreeRunExecution: true },
|
||||
{
|
||||
now: () => activatedAt,
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(next.enableWorktreeRunExecution).toBe(true);
|
||||
expect(next.worktreeRunExecutionActivatedAt).toBe("2026-07-10T12:00:00.000Z");
|
||||
expect(next.worktreeRunExecutionActivationInstanceId).toBe("worktree-instance");
|
||||
});
|
||||
|
||||
it("clears worktree run execution activation fields on a true to false transition", () => {
|
||||
const next = applyExperimentalSettingsPatch(
|
||||
{
|
||||
enableWorktreeRunExecution: true,
|
||||
worktreeRunExecutionActivatedAt: "2026-07-10T12:00:00.000Z",
|
||||
worktreeRunExecutionActivationInstanceId: "worktree-instance",
|
||||
},
|
||||
{ enableWorktreeRunExecution: false },
|
||||
{
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(next.enableWorktreeRunExecution).toBe(false);
|
||||
expect(next.worktreeRunExecutionActivatedAt).toBeNull();
|
||||
expect(next.worktreeRunExecutionActivationInstanceId).toBeNull();
|
||||
});
|
||||
|
||||
it("refreshes the activation cutoff when worktree run execution is re-toggled", () => {
|
||||
const firstActivation = applyExperimentalSettingsPatch(
|
||||
{ enableWorktreeRunExecution: false },
|
||||
{ enableWorktreeRunExecution: true },
|
||||
{
|
||||
now: () => new Date("2026-07-10T12:00:00.000Z"),
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
},
|
||||
);
|
||||
const disabled = applyExperimentalSettingsPatch(
|
||||
firstActivation,
|
||||
{ enableWorktreeRunExecution: false },
|
||||
{
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const secondActivation = applyExperimentalSettingsPatch(
|
||||
disabled,
|
||||
{ enableWorktreeRunExecution: true },
|
||||
{
|
||||
now: () => new Date("2026-07-10T12:05:00.000Z"),
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(secondActivation.worktreeRunExecutionActivatedAt).toBe("2026-07-10T12:05:00.000Z");
|
||||
expect(secondActivation.worktreeRunExecutionActivatedAt).not.toBe(
|
||||
firstActivation.worktreeRunExecutionActivatedAt,
|
||||
);
|
||||
});
|
||||
|
||||
it("strips client-supplied activation fields before applying experimental patches", () => {
|
||||
const next = applyExperimentalSettingsPatch(
|
||||
{ enableWorktreeRunExecution: false },
|
||||
{
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: "2026-07-10T12:00:00.000Z",
|
||||
worktreeRunExecutionActivationInstanceId: "copied-instance",
|
||||
},
|
||||
{
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(next.worktreeRunExecutionActivatedAt).toBeNull();
|
||||
expect(next.worktreeRunExecutionActivationInstanceId).toBeNull();
|
||||
});
|
||||
|
||||
it("resolves worktree run execution as armed only when the cutoff matches the current instance", async () => {
|
||||
const experimental = normalizeExperimentalSettings({
|
||||
enableWorktreeRunExecution: true,
|
||||
worktreeRunExecutionActivatedAt: "2026-07-10T12:00:00.000Z",
|
||||
worktreeRunExecutionActivationInstanceId: "worktree-instance",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: async () => experimental,
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
armed: true,
|
||||
cutoff: "2026-07-10T12:00:00.000Z",
|
||||
activationInstanceId: "worktree-instance",
|
||||
reason: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when worktree run execution is missing a cutoff", async () => {
|
||||
const experimental = normalizeExperimentalSettings({
|
||||
enableWorktreeRunExecution: true,
|
||||
worktreeRunExecutionActivationInstanceId: "worktree-instance",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: async () => experimental,
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
armed: false,
|
||||
cutoff: null,
|
||||
reason: "missing_cutoff",
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when worktree run execution was activated by another instance", async () => {
|
||||
const experimental = normalizeExperimentalSettings({
|
||||
enableWorktreeRunExecution: true,
|
||||
worktreeRunExecutionActivatedAt: "2026-07-10T12:00:00.000Z",
|
||||
worktreeRunExecutionActivationInstanceId: "source-instance",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: async () => experimental,
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "target-instance",
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
armed: false,
|
||||
cutoff: null,
|
||||
activationInstanceId: "source-instance",
|
||||
reason: "instance_id_mismatch",
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed on settings read errors and avoids reads outside worktree runtimes", async () => {
|
||||
await expect(
|
||||
resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: async () => {
|
||||
throw new Error("settings unavailable");
|
||||
},
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
armed: false,
|
||||
cutoff: null,
|
||||
reason: "settings_read_error",
|
||||
});
|
||||
|
||||
const getExperimental = vi.fn<() => Promise<InstanceExperimentalSettings>>();
|
||||
await expect(
|
||||
resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental,
|
||||
runtimeEnv: {
|
||||
PAPERCLIP_IN_WORKTREE: "false",
|
||||
PAPERCLIP_INSTANCE_ID: "worktree-instance",
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
armed: false,
|
||||
cutoff: null,
|
||||
reason: "not_worktree_runtime",
|
||||
});
|
||||
expect(getExperimental).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
});
|
||||
@@ -88,6 +88,7 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => {
|
||||
});
|
||||
|
||||
async function seedFixture(opts?: {
|
||||
runtimeEnv?: Record<string, string | undefined>;
|
||||
wakeup?: (
|
||||
agentId: string,
|
||||
wakeupOpts: {
|
||||
@@ -147,6 +148,7 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => {
|
||||
});
|
||||
|
||||
const svc = routineService(db, {
|
||||
runtimeEnv: opts?.runtimeEnv,
|
||||
heartbeat: {
|
||||
wakeup: async (wakeupAgentId, wakeupOpts) => {
|
||||
wakeups.push({ agentId: wakeupAgentId, opts: wakeupOpts });
|
||||
@@ -204,6 +206,18 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => {
|
||||
return { companyId, agentId, issueSvc, projectId, routine, svc, wakeups };
|
||||
}
|
||||
|
||||
async function armWorktreeExecution(cutoff: Date, instanceId = "worktree-routines-test") {
|
||||
await db.insert(instanceSettings).values({
|
||||
singletonKey: "default",
|
||||
general: {},
|
||||
experimental: {
|
||||
enableWorktreeRunExecution: true,
|
||||
worktreeRunExecutionActivatedAt: cutoff.toISOString(),
|
||||
worktreeRunExecutionActivationInstanceId: instanceId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
it("filters listed routines by project", async () => {
|
||||
const { companyId, agentId, projectId, routine, svc } = await seedFixture();
|
||||
const otherProjectId = randomUUID();
|
||||
@@ -1746,6 +1760,80 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => {
|
||||
expect(run.status).toBe("issue_created");
|
||||
});
|
||||
|
||||
it("records suppressed automatic runs when worktree execution is disabled while allowing manual runs", async () => {
|
||||
const runtimeEnv = { PAPERCLIP_IN_WORKTREE: "yes", PAPERCLIP_INSTANCE_ID: "worktree-routines-test" };
|
||||
const { companyId, routine, svc } = await seedFixture({ runtimeEnv });
|
||||
const { trigger: scheduleTrigger } = await svc.createTrigger(
|
||||
routine.id,
|
||||
{ kind: "schedule", cronExpression: "0 0 * * *", timezone: "UTC" },
|
||||
{},
|
||||
);
|
||||
const { trigger: webhookTrigger } = await svc.createTrigger(
|
||||
routine.id,
|
||||
{ kind: "webhook", signingMode: "none" },
|
||||
{},
|
||||
);
|
||||
const pastDue = new Date("2020-01-01T00:00:00.000Z");
|
||||
await db.update(routineTriggers).set({ nextRunAt: pastDue }).where(eq(routineTriggers.id, scheduleTrigger.id));
|
||||
|
||||
expect(await svc.tickScheduledTriggers(new Date())).toEqual({ triggered: 0 });
|
||||
const webhookRun = await svc.firePublicTrigger(webhookTrigger.publicId!, { payload: { event: "created" } });
|
||||
expect(webhookRun).toMatchObject({ source: "webhook", status: "skipped", failureReason: "worktree_execution_cutoff" });
|
||||
|
||||
const manualRun = await svc.runRoutine(routine.id, { source: "manual" });
|
||||
expect(manualRun.status).toBe("issue_created");
|
||||
|
||||
const automatedRuns = await db.select().from(routineRuns).where(eq(routineRuns.routineId, routine.id));
|
||||
expect(automatedRuns.filter((run) => run.failureReason === "worktree_execution_cutoff")).toHaveLength(2);
|
||||
expect(automatedRuns.filter((run) => run.linkedIssueId)).toHaveLength(1);
|
||||
const scheduleAfter = await db.select().from(routineTriggers).where(eq(routineTriggers.id, scheduleTrigger.id)).then((rows) => rows[0]);
|
||||
expect(scheduleAfter!.nextRunAt!.getTime()).toBeGreaterThan(pastDue.getTime());
|
||||
expect((await db.select().from(issues).where(eq(issues.companyId, companyId))).filter((issue) => issue.originKind === "routine_execution")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("dispatches only post-cutoff scheduled routines in an armed worktree", async () => {
|
||||
const runtimeEnv = { PAPERCLIP_IN_WORKTREE: "true", PAPERCLIP_INSTANCE_ID: "worktree-routines-test" };
|
||||
const { companyId, agentId, projectId, routine: oldRoutine, svc } = await seedFixture({ runtimeEnv });
|
||||
const cutoff = new Date("2025-01-01T00:00:00.000Z");
|
||||
await armWorktreeExecution(cutoff);
|
||||
const newRoutine = await svc.create(companyId, {
|
||||
projectId,
|
||||
goalId: null,
|
||||
parentIssueId: null,
|
||||
title: "new routine",
|
||||
description: null,
|
||||
assigneeAgentId: agentId,
|
||||
priority: "medium",
|
||||
status: "active",
|
||||
concurrencyPolicy: "coalesce_if_active",
|
||||
catchUpPolicy: "skip_missed",
|
||||
}, {});
|
||||
await db.update(routines).set({ createdAt: new Date("2024-12-31T23:59:59.000Z") }).where(eq(routines.id, oldRoutine.id));
|
||||
await db.update(routines).set({ createdAt: new Date("2025-01-01T00:00:01.000Z") }).where(eq(routines.id, newRoutine.id));
|
||||
const { trigger: oldTrigger } = await svc.createTrigger(oldRoutine.id, { kind: "schedule", cronExpression: "0 0 * * *", timezone: "UTC" }, {});
|
||||
const { trigger: newTrigger } = await svc.createTrigger(newRoutine.id, { kind: "schedule", cronExpression: "0 0 * * *", timezone: "UTC" }, {});
|
||||
await db.update(routineTriggers).set({ nextRunAt: new Date("2020-01-01T00:00:00.000Z") }).where(eq(routineTriggers.id, oldTrigger.id));
|
||||
await db.update(routineTriggers).set({ nextRunAt: new Date("2020-01-01T00:00:00.000Z") }).where(eq(routineTriggers.id, newTrigger.id));
|
||||
|
||||
expect(await svc.tickScheduledTriggers(new Date())).toEqual({ triggered: 1 });
|
||||
const oldRuns = await db.select().from(routineRuns).where(eq(routineRuns.routineId, oldRoutine.id));
|
||||
expect(oldRuns).toMatchObject([{ status: "skipped", failureReason: "worktree_execution_cutoff", linkedIssueId: null }]);
|
||||
const newRuns = await db.select().from(routineRuns).where(eq(routineRuns.routineId, newRoutine.id));
|
||||
expect(newRuns).toMatchObject([{ status: "issue_created" }]);
|
||||
});
|
||||
|
||||
it("applies the armed cutoff to webhook dispatch but not manual API runs", async () => {
|
||||
const runtimeEnv = { PAPERCLIP_IN_WORKTREE: "true", PAPERCLIP_INSTANCE_ID: "worktree-routines-test" };
|
||||
const { routine, svc } = await seedFixture({ runtimeEnv });
|
||||
await armWorktreeExecution(new Date("2025-01-01T00:00:00.000Z"));
|
||||
await db.update(routines).set({ createdAt: new Date("2024-12-31T23:59:59.000Z") }).where(eq(routines.id, routine.id));
|
||||
const { trigger } = await svc.createTrigger(routine.id, { kind: "webhook", signingMode: "none" }, {});
|
||||
|
||||
const webhookRun = await svc.firePublicTrigger(trigger.publicId!, { payload: { event: "created" } });
|
||||
expect(webhookRun).toMatchObject({ status: "skipped", failureReason: "worktree_execution_cutoff", linkedIssueId: null });
|
||||
expect((await svc.runRoutine(routine.id, { source: "api" })).status).toBe("issue_created");
|
||||
});
|
||||
|
||||
it("suppresses scheduled ticks while the routine project is paused, then resumes when unpaused", async () => {
|
||||
const { companyId, projectId, routine, svc } = await seedFixture();
|
||||
const { trigger } = await svc.createTrigger(
|
||||
|
||||
@@ -663,4 +663,16 @@ describeEmbeddedPostgres("task watchdog scheduler", () => {
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "task_watchdog")));
|
||||
expect(watchdogIssues).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("handles an armed cutoff when no watchdogs are active", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const { service } = createService();
|
||||
|
||||
const result = await service.reconcileTaskWatchdogs({
|
||||
companyId,
|
||||
issueCreatedAtGte: new Date(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ checked: 0, triggered: 0 });
|
||||
});
|
||||
});
|
||||
@@ -65,6 +65,30 @@ describe("ui branding", () => {
|
||||
expect(meta).toContain('name="paperclip-worktree-color"');
|
||||
});
|
||||
|
||||
it("surfaces the runtime instance id so the UI can fail closed on copied rows", () => {
|
||||
const branding = getWorktreeUiBranding({
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_WORKTREE_NAME: "paperclip-pr-432",
|
||||
PAPERCLIP_WORKTREE_COLOR: "#4f86f7",
|
||||
PAPERCLIP_INSTANCE_ID: "inst-abc123",
|
||||
});
|
||||
expect(branding.instanceId).toBe("inst-abc123");
|
||||
|
||||
const meta = renderRuntimeBrandingMeta(branding);
|
||||
expect(meta).toContain('name="paperclip-instance-id"');
|
||||
expect(meta).toContain('content="inst-abc123"');
|
||||
});
|
||||
|
||||
it("omits the instance-id meta when the runtime id is unset", () => {
|
||||
const branding = getWorktreeUiBranding({
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
PAPERCLIP_WORKTREE_NAME: "paperclip-pr-432",
|
||||
PAPERCLIP_WORKTREE_COLOR: "#4f86f7",
|
||||
});
|
||||
expect(branding.instanceId).toBeNull();
|
||||
expect(renderRuntimeBrandingMeta(branding)).not.toContain('name="paperclip-instance-id"');
|
||||
});
|
||||
|
||||
it("rewrites the favicon and runtime branding blocks for worktree instances only", () => {
|
||||
const branded = applyUiBranding(TEMPLATE, {
|
||||
PAPERCLIP_IN_WORKTREE: "true",
|
||||
|
||||
@@ -48,6 +48,7 @@ import {
|
||||
reconcilePersistedRuntimeServicesOnStartup,
|
||||
routineService,
|
||||
} from "./services/index.js";
|
||||
import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js";
|
||||
import {
|
||||
parseAdapterRegistryEnv,
|
||||
reconcileAdapterAvailability,
|
||||
@@ -838,6 +839,16 @@ export async function startServer(): Promise<StartedServer> {
|
||||
drainHeartbeatRunsForShutdown = heartbeat.drainRunningRunsForShutdown;
|
||||
const environmentCustomImages = environmentCustomImageService(db as any, { pluginWorkerManager });
|
||||
const routines = routineService(db as any, { pluginWorkerManager });
|
||||
const worktreeRunExecutionActivation = await resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: () => instanceSettingsService(db).getExperimental(),
|
||||
});
|
||||
logger.info(
|
||||
{
|
||||
state: worktreeRunExecutionActivation.armed ? "armed" : "disarmed",
|
||||
cutoff: worktreeRunExecutionActivation.cutoff,
|
||||
},
|
||||
"worktree run-execution cutoff state",
|
||||
);
|
||||
const heartbeatSchedulingSuppression = await heartbeat.resolveSchedulingSuppression();
|
||||
|
||||
// Reap orphaned runs before timer ticks start so wakeups cannot coalesce
|
||||
|
||||
@@ -83,7 +83,11 @@ import {
|
||||
import { redactEventPayload } from "../redaction.js";
|
||||
import { redactCurrentUserValue } from "../log-redaction.js";
|
||||
import { renderOrgChartSvg, renderOrgChartPng, type OrgNode, type OrgChartStyle, ORG_CHART_STYLES } from "./org-chart-svg.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import {
|
||||
instanceSettingsService,
|
||||
isTruthyRuntimeEnvValue,
|
||||
resolveWorktreeRunExecutionActivationState,
|
||||
} from "../services/instance-settings.js";
|
||||
import { runClaudeLogin } from "@paperclipai/adapter-claude-local/server";
|
||||
import { DEFAULT_CODEX_LOCAL_BYPASS_APPROVALS_AND_SANDBOX } from "@paperclipai/adapter-codex-local";
|
||||
import { DEFAULT_CURSOR_LOCAL_MODEL } from "@paperclipai/adapter-cursor-local";
|
||||
@@ -2042,14 +2046,23 @@ export function agentRoutes(
|
||||
includeRoutineExecutions: true,
|
||||
limit: ISSUE_LIST_DEFAULT_LIMIT,
|
||||
});
|
||||
const issueIds = rows.map((issue) => issue.id);
|
||||
const worktreeActivation = await resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: () => instanceSettingsService(db).getExperimental(),
|
||||
});
|
||||
const isWorktreeRuntime = isTruthyRuntimeEnvValue(process.env.PAPERCLIP_IN_WORKTREE);
|
||||
const eligibleRows = !isWorktreeRuntime
|
||||
? rows
|
||||
: worktreeActivation.armed
|
||||
? rows.filter((issue) => new Date(issue.createdAt) >= new Date(worktreeActivation.cutoff))
|
||||
: [];
|
||||
const issueIds = eligibleRows.map((issue) => issue.id);
|
||||
const [dependencyReadiness, recoveryActionByIssue] = await Promise.all([
|
||||
issuesSvc.listDependencyReadiness(req.actor.companyId, issueIds),
|
||||
recoveryActionsSvc.listActiveForIssues(req.actor.companyId, issueIds),
|
||||
]);
|
||||
|
||||
res.json(
|
||||
rows.map((issue) => ({
|
||||
eligibleRows.map((issue) => ({
|
||||
id: issue.id,
|
||||
identifier: issue.identifier,
|
||||
title: issue.title,
|
||||
|
||||
@@ -161,7 +161,10 @@ import {
|
||||
WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE,
|
||||
WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION,
|
||||
} from "./execution-workspace-policy.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import {
|
||||
instanceSettingsService,
|
||||
resolveWorktreeRunExecutionActivation,
|
||||
} from "./instance-settings.js";
|
||||
import {
|
||||
evaluateExecutionAllowlist,
|
||||
isExecutionForcedToKubernetes,
|
||||
@@ -5018,30 +5021,45 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
// Only worktree runtimes ever read the setting; a short TTL keeps the hot-path
|
||||
// suppression checks off the DB, and a read failure falls back to prior/default
|
||||
// (fail closed to suppression).
|
||||
let cachedWorktreeRunExecutionOverride: { value: boolean; at: number } = { value: false, at: 0 };
|
||||
let cachedWorktreeRunExecutionOverride: { allowed: boolean; cutoff: Date | null; at: number } = {
|
||||
allowed: false,
|
||||
cutoff: null,
|
||||
at: 0,
|
||||
};
|
||||
const WORKTREE_RUN_EXECUTION_OVERRIDE_TTL_MS = 3_000;
|
||||
const resolveWorktreeRunExecutionOverride = async (): Promise<boolean> => {
|
||||
if (!inWorktreeRuntime) return false;
|
||||
const resolveWorktreeRunExecutionOverride = async () => {
|
||||
if (!inWorktreeRuntime) return { allowed: false, cutoff: null };
|
||||
const now = Date.now();
|
||||
if (now - cachedWorktreeRunExecutionOverride.at < WORKTREE_RUN_EXECUTION_OVERRIDE_TTL_MS) {
|
||||
return cachedWorktreeRunExecutionOverride.value;
|
||||
return cachedWorktreeRunExecutionOverride;
|
||||
}
|
||||
try {
|
||||
const experimental = await instanceSettings.getExperimental();
|
||||
const activation = resolveWorktreeRunExecutionActivation(
|
||||
await instanceSettings.getExperimental(),
|
||||
runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || null,
|
||||
);
|
||||
const cutoff = activation.armed ? new Date(activation.cutoff) : null;
|
||||
cachedWorktreeRunExecutionOverride = {
|
||||
value: experimental.enableWorktreeRunExecution === true,
|
||||
allowed: Boolean(activation.armed && cutoff && !Number.isNaN(cutoff.getTime())),
|
||||
cutoff: cutoff && !Number.isNaN(cutoff.getTime()) ? cutoff : null,
|
||||
at: now,
|
||||
};
|
||||
} catch {
|
||||
// Keep the prior (default-false) value so a settings read failure fails
|
||||
// closed to the safe suppressed state.
|
||||
}
|
||||
return cachedWorktreeRunExecutionOverride.value;
|
||||
return cachedWorktreeRunExecutionOverride;
|
||||
};
|
||||
const getSchedulingSuppression = async () =>
|
||||
resolveHeartbeatSchedulingSuppression(runtimeEnv, {
|
||||
allowWorktreeRunExecution: await resolveWorktreeRunExecutionOverride(),
|
||||
const getSchedulingSuppression = async () => {
|
||||
const override = await resolveWorktreeRunExecutionOverride();
|
||||
return resolveHeartbeatSchedulingSuppression(runtimeEnv, {
|
||||
allowWorktreeRunExecution: override.allowed,
|
||||
});
|
||||
};
|
||||
const getWorktreeExecutionCutoff = async () => {
|
||||
const override = await resolveWorktreeRunExecutionOverride();
|
||||
return override.allowed ? override.cutoff : null;
|
||||
};
|
||||
|
||||
const runLogStore = getRunLogStore();
|
||||
const secretsSvc = secretService(db);
|
||||
@@ -9462,6 +9480,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
}
|
||||
|
||||
async function promoteDueScheduledRetries(now = new Date()) {
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const dueRuns = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
@@ -9469,6 +9488,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
and(
|
||||
eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
lte(heartbeatRuns.scheduledRetryAt, now),
|
||||
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
|
||||
),
|
||||
)
|
||||
.orderBy(asc(heartbeatRuns.scheduledRetryAt), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
|
||||
@@ -10745,6 +10765,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
|
||||
const queuedRuns = await db
|
||||
.select({ agentId: heartbeatRuns.agentId })
|
||||
@@ -10753,6 +10774,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
.where(and(
|
||||
eq(heartbeatRuns.status, "queued"),
|
||||
eq(companies.status, "active"),
|
||||
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
|
||||
));
|
||||
|
||||
const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
|
||||
@@ -10762,7 +10784,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
}
|
||||
|
||||
async function reconcileStrandedAssignedIssues() {
|
||||
return recovery.reconcileStrandedAssignedIssues();
|
||||
return recovery.reconcileStrandedAssignedIssues({ issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
||||
}
|
||||
|
||||
async function sweepStaleIssueLocks() {
|
||||
@@ -10783,15 +10805,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
}
|
||||
|
||||
async function scanSilentActiveRuns(opts?: { now?: Date; companyId?: string }) {
|
||||
return recovery.scanSilentActiveRuns(opts);
|
||||
return recovery.scanSilentActiveRuns({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
||||
}
|
||||
|
||||
async function reconcileProductivityReviews(opts?: { now?: Date; companyId?: string }) {
|
||||
return productivityReviews.reconcileProductivityReviews(opts);
|
||||
return productivityReviews.reconcileProductivityReviews({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
||||
}
|
||||
|
||||
async function reconcileTaskWatchdogs(opts?: { companyId?: string | null; runId?: string | null }) {
|
||||
return taskWatchdogs.reconcileTaskWatchdogs(opts);
|
||||
return taskWatchdogs.reconcileTaskWatchdogs({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
||||
}
|
||||
|
||||
async function buildRunOutputSilence(
|
||||
@@ -10813,7 +10835,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
force?: boolean;
|
||||
lookbackHours?: number;
|
||||
}) {
|
||||
return recovery.reconcileIssueGraphLiveness(opts);
|
||||
return recovery.reconcileIssueGraphLiveness({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
||||
}
|
||||
|
||||
async function updateRuntimeState(
|
||||
@@ -10873,6 +10895,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
|
||||
async function startNextQueuedRunForAgent(agentId: string) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return [];
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
|
||||
return withAgentStartLock(agentId, async () => {
|
||||
const agent = await getAgent(agentId);
|
||||
@@ -10892,7 +10915,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
const queuedRuns = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued")))
|
||||
.where(and(
|
||||
eq(heartbeatRuns.agentId, agentId),
|
||||
eq(heartbeatRuns.status, "queued"),
|
||||
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
|
||||
))
|
||||
.orderBy(asc(heartbeatRuns.createdAt));
|
||||
if (queuedRuns.length === 0) return [];
|
||||
|
||||
@@ -14304,6 +14331,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
return null;
|
||||
}
|
||||
|
||||
const worktreeExecutionCutoff = opts.requestedByActorType === "user"
|
||||
? null
|
||||
: await getWorktreeExecutionCutoff();
|
||||
|
||||
const company = await db
|
||||
.select({ status: companies.status })
|
||||
.from(companies)
|
||||
@@ -14369,11 +14400,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
? or(eq(issues.id, issueId), eq(issues.identifier, issueId.toUpperCase()))
|
||||
: eq(issues.identifier, issueId.toUpperCase());
|
||||
const resolvedIssue = await db
|
||||
.select({ id: issues.id, projectId: issues.projectId })
|
||||
.select({ id: issues.id, projectId: issues.projectId, createdAt: issues.createdAt })
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, agent.companyId), idMatch))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (resolvedIssue) {
|
||||
if (worktreeExecutionCutoff && resolvedIssue.createdAt < worktreeExecutionCutoff) {
|
||||
await writeSkippedHeartbeatRequest("heartbeat.worktree_execution_cutoff", {
|
||||
reason: "worktree_execution_cutoff",
|
||||
cutoff: worktreeExecutionCutoff.toISOString(),
|
||||
issueId: resolvedIssue.id,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
projectId = resolvedIssue.projectId ?? null;
|
||||
// Canonicalize context to the UUID so downstream lookups always use UUID
|
||||
if (resolvedIssue.id !== issueId) {
|
||||
@@ -14548,6 +14587,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
createdAt: issues.createdAt,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId)))
|
||||
@@ -14570,6 +14610,30 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
return { kind: "skipped" as const };
|
||||
}
|
||||
|
||||
if (worktreeExecutionCutoff && issue.createdAt < worktreeExecutionCutoff) {
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
companyId: agent.companyId,
|
||||
agentId,
|
||||
source,
|
||||
triggerDetail,
|
||||
reason: "heartbeat.worktree_execution_cutoff",
|
||||
payload: {
|
||||
...(payload ?? {}),
|
||||
heartbeatSkip: {
|
||||
reason: "worktree_execution_cutoff",
|
||||
cutoff: worktreeExecutionCutoff.toISOString(),
|
||||
issueId: issue.id,
|
||||
},
|
||||
},
|
||||
status: "skipped",
|
||||
requestedByActorType: opts.requestedByActorType ?? null,
|
||||
requestedByActorId: opts.requestedByActorId ?? null,
|
||||
idempotencyKey: opts.idempotencyKey ?? null,
|
||||
finishedAt: new Date(),
|
||||
});
|
||||
return { kind: "skipped" as const };
|
||||
}
|
||||
|
||||
const cancelStaleScheduledRetry = async (scheduledRun: typeof heartbeatRuns.$inferSelect) => {
|
||||
const issueCancelled = issue.status === "cancelled";
|
||||
if (
|
||||
@@ -15952,6 +16016,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
skipped: 0,
|
||||
};
|
||||
}
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
|
||||
const allAgents = await db
|
||||
.select({ ...getTableColumns(agents) })
|
||||
@@ -15969,6 +16034,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
const policy = parseHeartbeatPolicy(agent);
|
||||
if (!policy.enabled || policy.intervalSec <= 0) continue;
|
||||
|
||||
if (cutoff) {
|
||||
const eligibleIssue = await db
|
||||
.select({ id: issues.id })
|
||||
.from(issues)
|
||||
.where(and(
|
||||
eq(issues.companyId, agent.companyId),
|
||||
eq(issues.assigneeAgentId, agent.id),
|
||||
inArray(issues.status, ["todo", "in_progress"]),
|
||||
gte(issues.createdAt, cutoff),
|
||||
))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!eligibleIssue) continue;
|
||||
}
|
||||
|
||||
checked += 1;
|
||||
const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
|
||||
const elapsedMs = now.getTime() - baseline;
|
||||
|
||||
@@ -18,6 +18,164 @@ import { eq } from "drizzle-orm";
|
||||
const DEFAULT_SINGLETON_KEY = "default";
|
||||
const instanceGeneralSettingsStorageSchema = instanceGeneralSettingsSchema.strip();
|
||||
const instanceExperimentalSettingsStorageSchema = instanceExperimentalSettingsSchema.strip();
|
||||
const TRUTHY_RUNTIME_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
|
||||
|
||||
interface InstanceSettingsServiceOptions {
|
||||
runtimeEnv?: Record<string, string | undefined>;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
type WorktreeRunExecutionSuppressedReason =
|
||||
| "not_worktree_runtime"
|
||||
| "flag_disabled"
|
||||
| "missing_cutoff"
|
||||
| "missing_instance_id"
|
||||
| "instance_id_mismatch"
|
||||
| "settings_read_error";
|
||||
|
||||
export type WorktreeRunExecutionActivationState =
|
||||
| {
|
||||
armed: true;
|
||||
cutoff: string;
|
||||
activationInstanceId: string;
|
||||
reason: null;
|
||||
}
|
||||
| {
|
||||
armed: false;
|
||||
cutoff: null;
|
||||
activationInstanceId: string | null;
|
||||
reason: WorktreeRunExecutionSuppressedReason;
|
||||
};
|
||||
|
||||
export function isTruthyRuntimeEnvValue(value: string | undefined) {
|
||||
return typeof value === "string" && TRUTHY_RUNTIME_ENV_VALUES.has(value.trim().toLowerCase());
|
||||
}
|
||||
|
||||
function getRuntimeInstanceId(env: Record<string, string | undefined>) {
|
||||
const instanceId = env.PAPERCLIP_INSTANCE_ID?.trim();
|
||||
return instanceId ? instanceId : null;
|
||||
}
|
||||
|
||||
function stripServerManagedExperimentalPatchFields(
|
||||
patch: PatchInstanceExperimentalSettings | Record<string, unknown>,
|
||||
): PatchInstanceExperimentalSettings {
|
||||
const {
|
||||
worktreeRunExecutionActivatedAt: _ignoredActivatedAt,
|
||||
worktreeRunExecutionActivationInstanceId: _ignoredActivationInstanceId,
|
||||
...patchable
|
||||
} = patch as Record<string, unknown>;
|
||||
return patchable as PatchInstanceExperimentalSettings;
|
||||
}
|
||||
|
||||
export function applyExperimentalSettingsPatch(
|
||||
current: unknown,
|
||||
patch: PatchInstanceExperimentalSettings | Record<string, unknown>,
|
||||
options: InstanceSettingsServiceOptions = {},
|
||||
): InstanceExperimentalSettings {
|
||||
const previousExperimental = normalizeExperimentalSettings(current);
|
||||
const patchable = stripServerManagedExperimentalPatchFields(patch);
|
||||
const nextExperimental = normalizeExperimentalSettings({
|
||||
...previousExperimental,
|
||||
...patchable,
|
||||
});
|
||||
const hasWorktreeRunExecutionPatch = Object.prototype.hasOwnProperty.call(
|
||||
patchable,
|
||||
"enableWorktreeRunExecution",
|
||||
);
|
||||
|
||||
if (!hasWorktreeRunExecutionPatch) {
|
||||
return nextExperimental;
|
||||
}
|
||||
|
||||
if (nextExperimental.enableWorktreeRunExecution !== true) {
|
||||
return {
|
||||
...nextExperimental,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
};
|
||||
}
|
||||
|
||||
if (previousExperimental.enableWorktreeRunExecution === true) {
|
||||
return nextExperimental;
|
||||
}
|
||||
|
||||
const runtimeEnv = options.runtimeEnv ?? process.env;
|
||||
if (!isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)) {
|
||||
return nextExperimental;
|
||||
}
|
||||
|
||||
return {
|
||||
...nextExperimental,
|
||||
worktreeRunExecutionActivatedAt: (options.now ?? (() => new Date()))().toISOString(),
|
||||
worktreeRunExecutionActivationInstanceId: getRuntimeInstanceId(runtimeEnv),
|
||||
};
|
||||
}
|
||||
|
||||
function suppressWorktreeRunExecution(
|
||||
reason: WorktreeRunExecutionSuppressedReason,
|
||||
activationInstanceId: string | null = null,
|
||||
): WorktreeRunExecutionActivationState {
|
||||
return {
|
||||
armed: false,
|
||||
cutoff: null,
|
||||
activationInstanceId,
|
||||
reason,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveWorktreeRunExecutionActivation(
|
||||
experimental: InstanceExperimentalSettings,
|
||||
currentInstanceId: string | null | undefined,
|
||||
): WorktreeRunExecutionActivationState {
|
||||
if (experimental.enableWorktreeRunExecution !== true) {
|
||||
return suppressWorktreeRunExecution(
|
||||
"flag_disabled",
|
||||
experimental.worktreeRunExecutionActivationInstanceId,
|
||||
);
|
||||
}
|
||||
if (!experimental.worktreeRunExecutionActivatedAt) {
|
||||
return suppressWorktreeRunExecution(
|
||||
"missing_cutoff",
|
||||
experimental.worktreeRunExecutionActivationInstanceId,
|
||||
);
|
||||
}
|
||||
if (!currentInstanceId) {
|
||||
return suppressWorktreeRunExecution(
|
||||
"missing_instance_id",
|
||||
experimental.worktreeRunExecutionActivationInstanceId,
|
||||
);
|
||||
}
|
||||
if (experimental.worktreeRunExecutionActivationInstanceId !== currentInstanceId) {
|
||||
return suppressWorktreeRunExecution(
|
||||
"instance_id_mismatch",
|
||||
experimental.worktreeRunExecutionActivationInstanceId,
|
||||
);
|
||||
}
|
||||
return {
|
||||
armed: true,
|
||||
cutoff: experimental.worktreeRunExecutionActivatedAt,
|
||||
activationInstanceId: currentInstanceId,
|
||||
reason: null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function resolveWorktreeRunExecutionActivationState(options: {
|
||||
getExperimental: () => Promise<InstanceExperimentalSettings>;
|
||||
runtimeEnv?: Record<string, string | undefined>;
|
||||
}): Promise<WorktreeRunExecutionActivationState> {
|
||||
const runtimeEnv = options.runtimeEnv ?? process.env;
|
||||
if (!isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)) {
|
||||
return suppressWorktreeRunExecution("not_worktree_runtime");
|
||||
}
|
||||
try {
|
||||
return resolveWorktreeRunExecutionActivation(
|
||||
await options.getExperimental(),
|
||||
getRuntimeInstanceId(runtimeEnv),
|
||||
);
|
||||
} catch {
|
||||
return suppressWorktreeRunExecution("settings_read_error");
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings {
|
||||
const parsed = instanceGeneralSettingsStorageSchema.safeParse(raw ?? {});
|
||||
@@ -63,6 +221,9 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
enableWorkspaceBranchReconcileForward: parsed.data.enableWorkspaceBranchReconcileForward ?? true,
|
||||
enableWorkspaceDirtyQuarantineRepair: parsed.data.enableWorkspaceDirtyQuarantineRepair ?? true,
|
||||
enableWorktreeRunExecution: parsed.data.enableWorktreeRunExecution ?? false,
|
||||
worktreeRunExecutionActivatedAt: parsed.data.worktreeRunExecutionActivatedAt ?? null,
|
||||
worktreeRunExecutionActivationInstanceId:
|
||||
parsed.data.worktreeRunExecutionActivationInstanceId ?? null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours:
|
||||
parsed.data.issueGraphLivenessAutoRecoveryLookbackHours ??
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
@@ -88,6 +249,8 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
enableWorkspaceBranchReconcileForward: true,
|
||||
enableWorkspaceDirtyQuarantineRepair: true,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours:
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
};
|
||||
@@ -104,7 +267,7 @@ function toInstanceSettings(row: typeof instanceSettings.$inferSelect): Instance
|
||||
} as InstanceSettings;
|
||||
}
|
||||
|
||||
export function instanceSettingsService(db: Db) {
|
||||
export function instanceSettingsService(db: Db, options: InstanceSettingsServiceOptions = {}) {
|
||||
async function getOrCreateRow() {
|
||||
const existing = await db
|
||||
.select()
|
||||
@@ -192,10 +355,7 @@ export function instanceSettingsService(db: Db) {
|
||||
|
||||
updateExperimental: async (patch: PatchInstanceExperimentalSettings): Promise<InstanceSettings> => {
|
||||
const current = await getOrCreateRow();
|
||||
const nextExperimental = normalizeExperimentalSettings({
|
||||
...normalizeExperimentalSettings(current.experimental),
|
||||
...patch,
|
||||
});
|
||||
const nextExperimental = applyExperimentalSettingsPatch(current.experimental, patch, options);
|
||||
const now = new Date();
|
||||
const [updated] = await db
|
||||
.update(instanceSettings)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, asc, desc, eq, gt, inArray, isNull, notInArray, sql } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, gt, gte, inArray, isNull, notInArray, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { clampIssueRequestDepth } from "@paperclipai/shared";
|
||||
import {
|
||||
@@ -763,6 +763,7 @@ export function productivityReviewService(db: Db, deps?: { enqueueWakeup?: Enque
|
||||
now?: Date;
|
||||
companyId?: string;
|
||||
thresholds?: Partial<ProductivityReviewThresholds>;
|
||||
issueCreatedAtGte?: Date | null;
|
||||
}) {
|
||||
const now = opts?.now ?? new Date();
|
||||
const thresholds = buildThresholds(opts?.thresholds);
|
||||
@@ -777,6 +778,7 @@ export function productivityReviewService(db: Db, deps?: { enqueueWakeup?: Enque
|
||||
inArray(issues.status, ["todo", "in_progress"]),
|
||||
sql`${issues.assigneeAgentId} is not null`,
|
||||
sql`${issues.originKind} <> ${PRODUCTIVITY_REVIEW_ORIGIN_KIND}`,
|
||||
opts?.issueCreatedAtGte ? gte(issues.createdAt, opts.issueCreatedAtGte) : undefined,
|
||||
),
|
||||
)
|
||||
.orderBy(asc(issues.updatedAt), asc(issues.id))
|
||||
|
||||
@@ -2009,10 +2009,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
return { kind: "created" as const, evaluationIssueId: evaluation.id };
|
||||
}
|
||||
|
||||
async function scanSilentActiveRuns(opts?: { now?: Date; companyId?: string }) {
|
||||
async function scanSilentActiveRuns(opts?: { now?: Date; companyId?: string; issueCreatedAtGte?: Date | null }) {
|
||||
const now = opts?.now ?? new Date();
|
||||
const suspicionBefore = new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS);
|
||||
const candidates = await db
|
||||
let candidates = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
@@ -2025,6 +2025,27 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
.orderBy(asc(heartbeatRuns.createdAt))
|
||||
.limit(100);
|
||||
|
||||
if (opts?.issueCreatedAtGte) {
|
||||
const issueIds = [...new Set(candidates.flatMap((run) => {
|
||||
const context = parseObject(run.contextSnapshot);
|
||||
const issueId = context.issueId ?? context.taskId;
|
||||
return typeof issueId === "string" && issueId.length > 0 ? [issueId] : [];
|
||||
}))];
|
||||
const eligibleIssueIds = new Set(
|
||||
issueIds.length > 0
|
||||
? (await db.select({ id: issues.id }).from(issues).where(and(
|
||||
inArray(issues.id, issueIds),
|
||||
gte(issues.createdAt, opts.issueCreatedAtGte),
|
||||
))).map((issue) => issue.id)
|
||||
: [],
|
||||
);
|
||||
candidates = candidates.filter((run) => {
|
||||
const context = parseObject(run.contextSnapshot);
|
||||
const issueId = context.issueId ?? context.taskId;
|
||||
return typeof issueId === "string" && eligibleIssueIds.has(issueId);
|
||||
});
|
||||
}
|
||||
|
||||
const result = {
|
||||
scanned: candidates.length,
|
||||
created: 0,
|
||||
@@ -2955,7 +2976,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
return updated;
|
||||
}
|
||||
|
||||
async function reconcileStrandedAssignedIssues() {
|
||||
async function reconcileStrandedAssignedIssues(opts?: { issueCreatedAtGte?: Date | null }) {
|
||||
const candidates = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
@@ -2967,6 +2988,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
sql`${issues.assigneeAgentId} is not null`,
|
||||
eq(issues.status, "in_review"),
|
||||
),
|
||||
opts?.issueCreatedAtGte ? gte(issues.createdAt, opts.issueCreatedAtGte) : undefined,
|
||||
),
|
||||
);
|
||||
|
||||
@@ -4488,8 +4510,25 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
runId?: string | null;
|
||||
force?: boolean;
|
||||
lookbackHours?: number;
|
||||
issueCreatedAtGte?: Date | null;
|
||||
}) {
|
||||
const findings = await collectIssueGraphLivenessFindings();
|
||||
let findings = await collectIssueGraphLivenessFindings();
|
||||
if (opts?.issueCreatedAtGte) {
|
||||
const findingIssueIds = [...new Set(findings.map((finding) => finding.recoveryIssueId))];
|
||||
const eligibleIssueIds = new Set(
|
||||
findingIssueIds.length === 0
|
||||
? []
|
||||
: (await db
|
||||
.select({ id: issues.id })
|
||||
.from(issues)
|
||||
.where(and(
|
||||
inArray(issues.id, findingIssueIds),
|
||||
gte(issues.createdAt, opts.issueCreatedAtGte),
|
||||
)))
|
||||
.map((issue) => issue.id),
|
||||
);
|
||||
findings = findings.filter((finding) => eligibleIssueIds.has(finding.recoveryIssueId));
|
||||
}
|
||||
const experimentalSettings = await instanceSettings.getExperimental();
|
||||
const autoRecoveryEnabled = asBoolean(
|
||||
experimentalSettings.enableIssueGraphLivenessAutoRecovery,
|
||||
|
||||
@@ -65,6 +65,12 @@ import { secretService } from "./secrets.js";
|
||||
import { getSecretProvider } from "../secrets/provider-registry.js";
|
||||
import { parseCron, validateCron } from "./cron.js";
|
||||
import { heartbeatService } from "./heartbeat.js";
|
||||
import {
|
||||
instanceSettingsService,
|
||||
isTruthyRuntimeEnvValue,
|
||||
resolveWorktreeRunExecutionActivationState,
|
||||
type WorktreeRunExecutionActivationState,
|
||||
} from "./instance-settings.js";
|
||||
import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
||||
@@ -592,10 +598,13 @@ export function routineService(
|
||||
deps: {
|
||||
heartbeat?: IssueAssignmentWakeupDeps;
|
||||
pluginWorkerManager?: PluginWorkerManager;
|
||||
runtimeEnv?: Record<string, string | undefined>;
|
||||
} = {},
|
||||
) {
|
||||
const issueSvc = issueService(db);
|
||||
const secretsSvc = secretService(db);
|
||||
const instanceSettings = instanceSettingsService(db);
|
||||
const runtimeEnv = deps.runtimeEnv ?? process.env;
|
||||
const heartbeat = deps.heartbeat ?? heartbeatService(db, {
|
||||
pluginWorkerManager: deps.pluginWorkerManager,
|
||||
});
|
||||
@@ -1155,14 +1164,32 @@ export function routineService(
|
||||
}
|
||||
}
|
||||
|
||||
// Records a skipped scheduled firing without creating an execution issue. Used when the
|
||||
// routine's project is paused: the tick is still claimed/advanced upstream (no backfill),
|
||||
// and run history + trigger audit reflect the pause-specific skip.
|
||||
async function recordSuppressedScheduleRun(input: {
|
||||
async function getAutomaticRoutineDispatchEligibility(
|
||||
routine: typeof routines.$inferSelect,
|
||||
activation?: WorktreeRunExecutionActivationState,
|
||||
) {
|
||||
if (!isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)) return { eligible: true };
|
||||
|
||||
const resolvedActivation = activation ?? await resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: instanceSettings.getExperimental,
|
||||
runtimeEnv,
|
||||
});
|
||||
if (!resolvedActivation.armed) return { eligible: false };
|
||||
|
||||
const cutoff = new Date(resolvedActivation.cutoff);
|
||||
if (Number.isNaN(cutoff.getTime()) || routine.createdAt < cutoff) return { eligible: false };
|
||||
return { eligible: true };
|
||||
}
|
||||
|
||||
// Records an automatic firing that was claimed but intentionally not dispatched. The
|
||||
// scheduler advances its tick before calling this helper, so suppressed work is never
|
||||
// replayed after a setting or project state changes.
|
||||
async function recordSuppressedAutomaticRun(input: {
|
||||
routine: typeof routines.$inferSelect;
|
||||
trigger: typeof routineTriggers.$inferSelect;
|
||||
source: "schedule" | "webhook";
|
||||
reason: string;
|
||||
nextRunAt: Date | null;
|
||||
nextRunAt?: Date | null;
|
||||
}) {
|
||||
const triggeredAt = new Date();
|
||||
const run = await db.transaction(async (tx) => {
|
||||
@@ -1173,7 +1200,7 @@ export function routineService(
|
||||
companyId: input.routine.companyId,
|
||||
routineId: input.routine.id,
|
||||
triggerId: input.trigger.id,
|
||||
source: "schedule",
|
||||
source: input.source,
|
||||
status: "skipped",
|
||||
triggeredAt,
|
||||
failureReason: input.reason,
|
||||
@@ -1187,7 +1214,7 @@ export function routineService(
|
||||
routineId: input.routine.id,
|
||||
triggerId: input.trigger.id,
|
||||
triggeredAt,
|
||||
status: "skipped_paused",
|
||||
status: input.reason === "paused" ? "skipped_paused" : "skipped_worktree_execution_cutoff",
|
||||
nextRunAt: input.nextRunAt,
|
||||
}, txDb);
|
||||
return createdRun;
|
||||
@@ -1197,14 +1224,14 @@ export function routineService(
|
||||
await logActivity(db, {
|
||||
companyId: input.routine.companyId,
|
||||
actorType: "system",
|
||||
actorId: "routine-scheduler",
|
||||
actorId: input.source === "schedule" ? "routine-scheduler" : "routine-webhook",
|
||||
action: "routine.run_skipped",
|
||||
entityType: "routine_run",
|
||||
entityId: run.id,
|
||||
details: {
|
||||
routineId: input.routine.id,
|
||||
triggerId: input.trigger.id,
|
||||
source: "schedule",
|
||||
source: input.source,
|
||||
status: "skipped",
|
||||
reason: input.reason,
|
||||
},
|
||||
@@ -2645,6 +2672,16 @@ export function routineService(
|
||||
if (!valid) throw unauthorized();
|
||||
}
|
||||
|
||||
const eligibility = await getAutomaticRoutineDispatchEligibility(routine);
|
||||
if (!eligibility.eligible) {
|
||||
return recordSuppressedAutomaticRun({
|
||||
routine,
|
||||
trigger,
|
||||
source: "webhook",
|
||||
reason: "worktree_execution_cutoff",
|
||||
});
|
||||
}
|
||||
|
||||
return dispatchRoutineRun({
|
||||
routine,
|
||||
trigger,
|
||||
@@ -2732,6 +2769,12 @@ export function routineService(
|
||||
},
|
||||
|
||||
tickScheduledTriggers: async (now: Date = new Date()) => {
|
||||
const worktreeActivation = isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)
|
||||
? await resolveWorktreeRunExecutionActivationState({
|
||||
getExperimental: instanceSettings.getExperimental,
|
||||
runtimeEnv,
|
||||
})
|
||||
: undefined;
|
||||
const due = await db
|
||||
.select({
|
||||
trigger: routineTriggers,
|
||||
@@ -2761,11 +2804,13 @@ export function routineService(
|
||||
// at the next cron boundary instead of replaying missed firings. Routines with no
|
||||
// project are never suppressed here.
|
||||
const projectPaused = !!(row.routine.projectId && row.projectPausedAt);
|
||||
const automaticEligibility = await getAutomaticRoutineDispatchEligibility(row.routine, worktreeActivation);
|
||||
const worktreeSuppressed = !automaticEligibility.eligible;
|
||||
|
||||
let runCount = 1;
|
||||
let claimedNextRunAt = nextCronTickInTimeZone(row.trigger.cronExpression, row.trigger.timezone, now);
|
||||
|
||||
if (!projectPaused && row.routine.catchUpPolicy === "enqueue_missed_with_cap") {
|
||||
if (!projectPaused && !worktreeSuppressed && row.routine.catchUpPolicy === "enqueue_missed_with_cap") {
|
||||
let cursor: Date | null = row.trigger.nextRunAt;
|
||||
runCount = 0;
|
||||
while (cursor && cursor <= now && runCount < MAX_CATCH_UP_RUNS) {
|
||||
@@ -2792,11 +2837,12 @@ export function routineService(
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!claimed) continue;
|
||||
|
||||
if (projectPaused) {
|
||||
await recordSuppressedScheduleRun({
|
||||
if (projectPaused || worktreeSuppressed) {
|
||||
await recordSuppressedAutomaticRun({
|
||||
routine: row.routine,
|
||||
trigger: row.trigger,
|
||||
reason: "paused",
|
||||
source: "schedule",
|
||||
reason: worktreeSuppressed ? "worktree_execution_cutoff" : "paused",
|
||||
nextRunAt: claimedNextRunAt,
|
||||
});
|
||||
continue;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { and, asc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agentWakeupRequests,
|
||||
@@ -1560,8 +1560,28 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {})
|
||||
return toIssueWatchdog(updated);
|
||||
},
|
||||
|
||||
reconcileTaskWatchdogs: async (opts: { companyId?: string | null; runId?: string | null } = {}) => {
|
||||
const rows = await listActiveWatchdogsForCompany(opts.companyId ?? null);
|
||||
reconcileTaskWatchdogs: async (opts: {
|
||||
companyId?: string | null;
|
||||
runId?: string | null;
|
||||
issueCreatedAtGte?: Date | null;
|
||||
} = {}) => {
|
||||
let rows = await listActiveWatchdogsForCompany(opts.companyId ?? null);
|
||||
if (opts.issueCreatedAtGte) {
|
||||
const watchdogIssueIds = [...new Set(rows.map((row) => row.issueId))];
|
||||
const eligibleIssueIds = new Set(
|
||||
watchdogIssueIds.length === 0
|
||||
? []
|
||||
: (await db
|
||||
.select({ id: issues.id })
|
||||
.from(issues)
|
||||
.where(and(
|
||||
inArray(issues.id, watchdogIssueIds),
|
||||
gte(issues.createdAt, opts.issueCreatedAtGte),
|
||||
)))
|
||||
.map((issue) => issue.id),
|
||||
);
|
||||
rows = rows.filter((row) => eligibleIssueIds.has(row.issueId));
|
||||
}
|
||||
const result = {
|
||||
checked: 0,
|
||||
triggered: 0,
|
||||
|
||||
@@ -16,6 +16,13 @@ export type WorktreeUiBranding = {
|
||||
color: string | null;
|
||||
textColor: string | null;
|
||||
faviconHref: string | null;
|
||||
/**
|
||||
* Runtime instance id for this worktree preview. Surfaced to the client so
|
||||
* the experimental "Run tasks in this worktree" card can fail closed when a
|
||||
* copied settings row was armed in a different instance. Null outside a
|
||||
* worktree or when the runtime id is unset.
|
||||
*/
|
||||
instanceId: string | null;
|
||||
};
|
||||
|
||||
function isTruthyEnvValue(value: string | undefined): boolean {
|
||||
@@ -152,6 +159,7 @@ export function getWorktreeUiBranding(env: NodeJS.ProcessEnv = process.env): Wor
|
||||
color: null,
|
||||
textColor: null,
|
||||
faviconHref: null,
|
||||
instanceId: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -165,6 +173,7 @@ export function getWorktreeUiBranding(env: NodeJS.ProcessEnv = process.env): Wor
|
||||
color,
|
||||
textColor,
|
||||
faviconHref: createFaviconDataUrl(color, textColor),
|
||||
instanceId: nonEmpty(env.PAPERCLIP_INSTANCE_ID),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -181,12 +190,16 @@ export function renderFaviconLinks(branding: WorktreeUiBranding): string {
|
||||
export function renderRuntimeBrandingMeta(branding: WorktreeUiBranding): string {
|
||||
if (!branding.enabled || !branding.name || !branding.color || !branding.textColor) return "";
|
||||
|
||||
return [
|
||||
const tags = [
|
||||
'<meta name="paperclip-worktree-enabled" content="true" />',
|
||||
`<meta name="paperclip-worktree-name" content="${escapeHtmlAttribute(branding.name)}" />`,
|
||||
`<meta name="paperclip-worktree-color" content="${escapeHtmlAttribute(branding.color)}" />`,
|
||||
`<meta name="paperclip-worktree-text-color" content="${escapeHtmlAttribute(branding.textColor)}" />`,
|
||||
].join("\n");
|
||||
];
|
||||
if (branding.instanceId) {
|
||||
tags.push(`<meta name="paperclip-instance-id" content="${escapeHtmlAttribute(branding.instanceId)}" />`);
|
||||
}
|
||||
return tags.join("\n");
|
||||
}
|
||||
|
||||
function replaceMarkedBlock(html: string, startMarker: string, endMarker: string, content: string): string {
|
||||
|
||||
Reference in new issue
Block a user