mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-29 21:07:40 +01:00
Merge current master and regenerate the delegated token migration
Preserve the upstream environment-skill runtime migration and regenerate the gateway responsible-user column as migration 0379. Keep the additive column idempotent. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
commit
db3190fe13
247 files changed
+14906
-803
No files matched your search
@@ -96,6 +96,8 @@ Create ordinary assigned tasks, never subtasks of this conversation. Give each t
|
||||
|
||||
For review or follow-up tasks, include the existing source materials the assignee needs, not just task identifiers or a rubric. Read the source task's saved documents and comments, then copy the relevant existing content into the handoff or attach resources the recipient can actually read. A new task's tools may only expose its own documents and history. Supplying existing source material is coordination, not creating the assigned deliverable.
|
||||
|
||||
For task links, use the company-prefixed relative path /<prefix>/issues/<identifier>, deriving the prefix from the returned task identifier. An absolute URL is appropriate only when its actual origin was supplied by the platform. Never invent a hostname for a Paperclip link.
|
||||
|
||||
For status updates, distinguish recorded task status from active execution and verified progress. Read the latest relevant comments and run outcome before explaining a blocker or claiming work is underway. Use the advertised API tools for another task's details when needed. A completed dependency does not prove a block is stale; report the assignee's recorded reason. Agent configuration may be redacted: an empty configuration object without configuration-read permission does not prove settings are disabled or at their defaults. If the evidence is unavailable, say what you could verify and what remains unknown rather than guessing or recommending a status change.
|
||||
|
||||
Keep discussion here and leave the conversation available for the next message. Link handed-off tasks in your reply; do not make this conversation blocked by their completion or wait for them. Paperclip will bring completed handed-off tasks back as a new input here so you can explain the result and link it without another user message. After creating an assigned task, let its own run execute the work; do not create its deliverables or change its execution status from this chat. Reply normally and end your turn; Paperclip manages the conversation waiting state. Do not change its status, create a review confirmation just to finish a reply, mark it complete, or poll for another reply. An accepted plan authorizes handoff to execution tasks, never implementation on this conversation. Honor normal approvals. Ask mode is non-mutating. Plan mode supports research and writing/revising the plan; hand off for execution only through the normal authorized workflow.`;
|
||||
|
||||
@@ -601,7 +601,7 @@ function uniqueImportedSkillKey(companyId: string, baseSlug: string, usedKeys: S
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function buildSkillRuntimeName(key: string, slug: string) {
|
||||
export function buildSkillRuntimeName(key: string, slug: string) {
|
||||
if (key.startsWith("paperclipai/paperclip/")) return slug;
|
||||
return `${slug}--${hashSkillValue(key)}`;
|
||||
}
|
||||
@@ -6238,6 +6238,8 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
|
||||
files: Record<string, string>,
|
||||
options?: {
|
||||
onConflict?: PackageSkillConflictStrategy;
|
||||
/** Managed plugin namespaces may share a slug while retaining separate canonical records. */
|
||||
allowSlugCollision?: boolean;
|
||||
},
|
||||
): Promise<ImportPackageSkillResult[]> {
|
||||
await ensureSkillInventoryCurrent(companyId);
|
||||
@@ -6286,7 +6288,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
|
||||
const normalizedSlug = normalizeSkillSlug(importedSkill.slug) ?? importedSkill.slug;
|
||||
const existingByIncomingKey = existingByKey.get(importedSkill.key) ?? null;
|
||||
const existingByIncomingSlug = existingBySlug.get(normalizedSlug) ?? null;
|
||||
const conflict = existingByIncomingKey ?? existingByIncomingSlug;
|
||||
const conflict = existingByIncomingKey ?? (options?.allowSlugCollision ? null : existingByIncomingSlug);
|
||||
|
||||
if (!conflict || conflictStrategy === "replace") {
|
||||
const skillToPersist = conflict && conflict.key !== importedSkill.key
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { and, eq, inArray, or, sql } from "drizzle-orm";
|
||||
import { companySkills, pluginCompanySettings, pluginManagedResources, plugins, type Db } from "@paperclipai/db";
|
||||
import { readPaperclipSkillSyncPreference, resolvePaperclipDesiredSkillNames, writePaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils";
|
||||
import { unprocessable } from "../errors.js";
|
||||
import { buildSkillRuntimeName } from "./company-skills.js";
|
||||
import { pluginManagedSkillService } from "./plugin-managed-skills.js";
|
||||
|
||||
/** Environment capabilities join the normal authorized, versioned skill selection. */
|
||||
export async function environmentSkillConfig(db: Db, companyId: string,
|
||||
environment: { driver: string; config: Record<string, unknown> | null } | null | undefined,
|
||||
config: Record<string, unknown>, installMissing = true): Promise<{ config: Record<string, unknown>; skillKeys: string[]; shadowedSkills?: Array<{ key: string; shadowedBy: string }> }> {
|
||||
if (environment?.driver !== "plugin") return { config, skillKeys: [] };
|
||||
const { pluginKey, driverKey } = environment.config ?? {};
|
||||
if (typeof pluginKey !== "string" || typeof driverKey !== "string") return { config, skillKeys: [] };
|
||||
const [plugin] = await db.select().from(plugins).where(eq(plugins.pluginKey, pluginKey));
|
||||
const keys = plugin?.manifestJson.environmentDrivers?.find(driver => driver.driverKey === driverKey)?.skillKeys ?? [];
|
||||
if (!plugin || !keys.length) return { config, skillKeys: [] };
|
||||
const [settings] = await db.select().from(pluginCompanySettings).where(and(
|
||||
eq(pluginCompanySettings.companyId, companyId), eq(pluginCompanySettings.pluginId, plugin.id)));
|
||||
if (plugin.status !== "ready" || settings?.enabled === false || !plugin.manifestJson.capabilities.includes("skills.managed")) {
|
||||
throw unprocessable("Environment skills require an enabled plugin with skills.managed capability");
|
||||
}
|
||||
// Existing bindings need only a bounded metadata query, not a file-by-file drift scan.
|
||||
const installed = await db.select({ resourceKey: pluginManagedResources.resourceKey, key: companySkills.key })
|
||||
.from(pluginManagedResources).innerJoin(companySkills, and(
|
||||
eq(companySkills.id, pluginManagedResources.resourceId), eq(companySkills.companyId, companyId)))
|
||||
.where(and(eq(pluginManagedResources.companyId, companyId), eq(pluginManagedResources.pluginId, plugin.id),
|
||||
eq(pluginManagedResources.resourceKind, "skill"), inArray(pluginManagedResources.resourceKey, keys)));
|
||||
const selected = new Map(installed.map(skill => [skill.resourceKey, skill.key]));
|
||||
for (const key of keys) {
|
||||
if (selected.has(key) || !installMissing) continue;
|
||||
const result = await db.transaction(async tx => {
|
||||
// Serialize first installation across concurrent agents in this company.
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`environment-skill:${companyId}:${plugin.id}:${key}`}, 0))`);
|
||||
return pluginManagedSkillService(tx as unknown as Db, { pluginId: plugin.id, pluginKey, manifest: plugin.manifestJson }).reconcile(key, companyId);
|
||||
});
|
||||
if (!result.skill) throw unprocessable("Environment skill is unavailable");
|
||||
selected.set(key, result.skill.key);
|
||||
}
|
||||
const preference = readPaperclipSkillSyncPreference(config);
|
||||
const explicit = new Set(preference.desiredSkillEntries.map(entry => entry.key));
|
||||
const skillKeys = keys.flatMap(key => selected.has(key) ? [selected.get(key)!] : []);
|
||||
const candidates = await db.select({ key: companySkills.key, slug: companySkills.slug, name: companySkills.name })
|
||||
.from(companySkills).where(and(eq(companySkills.companyId, companyId),
|
||||
or(inArray(sql<string>`lower(${companySkills.key})`, [...new Set([...explicit, ...skillKeys])].map(key => key.toLowerCase())),
|
||||
inArray(sql<string>`lower(${companySkills.slug})`, [...explicit].map(key => key.toLowerCase().split("/").at(-1)!.replace(/--[a-f0-9]{10}$/, ""))))));
|
||||
const names = candidates.map(skill => ({ key: skill.key, runtimeName: buildSkillRuntimeName(skill.key, skill.slug) }));
|
||||
const entries = preference.desiredSkillEntries.map(entry => ({ ...entry,
|
||||
key: resolvePaperclipDesiredSkillNames(writePaperclipSkillSyncPreference({}, [entry]), names)[0] ?? entry.key,
|
||||
}));
|
||||
const selectedKeys = new Set(entries.map(entry => entry.key));
|
||||
const required = candidates.filter(skill => skillKeys.includes(skill.key));
|
||||
const shadowedSkills = candidates.flatMap(skill => {
|
||||
if (skillKeys.includes(skill.key) || !selectedKeys.has(skill.key)) return [];
|
||||
const winner = required.find(candidate => candidate.slug === skill.slug || candidate.name === skill.name);
|
||||
return winner ? [{ key: skill.key, shadowedBy: winner.key }] : [];
|
||||
});
|
||||
const shadowed = new Set(shadowedSkills.map(skill => skill.key));
|
||||
return { config: writePaperclipSkillSyncPreference(config, [
|
||||
...entries.filter(entry => !shadowed.has(entry.key)),
|
||||
...skillKeys.filter(key => !selectedKeys.has(key))]), skillKeys,
|
||||
...(shadowedSkills.length ? { shadowedSkills } : {}),
|
||||
};
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { enableRunFailureReporting } from "../__tests__/helpers/run-failure-repo
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, completionContracts, createDb, heartbeatRuns, issues, issueRecoveryActions, issueThreadInteractions, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db";
|
||||
import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, issueRecoveryActions, issueThreadInteractions, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
@@ -153,6 +153,65 @@ describeEmbeddedPostgres("reconcileAbandonedExecutionControl reports a genuine f
|
||||
expect(result.surfaced).toBe(1);
|
||||
expect(mockCaptureRunFailure.mock.calls.slice(captureCallsBefore)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(["prepared", "finalized", "mismatched", "reassigned", "accepted"] as const)("preserves only an exact unfinished native workspace after its deadline (%s)", async state => {
|
||||
const f = await seedAbandonedRunFixture();
|
||||
const [environment] = await db.insert(environments).values({ name: `Deadline ${f.runId}`, driver: "sandbox" }).returning();
|
||||
const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId,
|
||||
heartbeatRunId: f.runId, environmentId: environment.id, provider: "daytona", providerLeaseId: randomUUID(),
|
||||
status: "active", leasePolicy: "reuse_by_environment", metadata: { remoteCwd: "/work" },
|
||||
}).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, runnerProfileJson: {
|
||||
nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: state === "finalized" ? state : "prepared",
|
||||
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null,
|
||||
workspaceId: f.runId, leaseId: lease.id, providerLeaseId: state === "mismatched" ? randomUUID() : lease.providerLeaseId,
|
||||
remoteCwd: "/work", resourceDisposition: "keep_running" },
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: f.runId, companyId: f.companyId, issueId: f.issueId,
|
||||
phase: "observed", leaseOwner: "abandoned-controller", controlDeadlineAt: new Date(Date.now() - 60_000) });
|
||||
if (state === "reassigned") {
|
||||
const nextAgentId = randomUUID();
|
||||
await db.insert(agents).values({ id: nextAgentId, companyId: f.companyId, name: "New owner", adapterType: "codex_local" });
|
||||
await db.update(issues).set({ assigneeAgentId: nextAgentId, executionRunId: null, checkoutRunId: null })
|
||||
.where(eq(issues.id, f.issueId));
|
||||
}
|
||||
if (state === "accepted") {
|
||||
const contractId = randomUUID(), resultId = randomUUID();
|
||||
await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId,
|
||||
revision: 1, schemaVersion: "1", policyVersion: "test", risk: "low", completionAuthority: "agent_claim_policy",
|
||||
incompleteCriteriaPolicy: "block", contractJson: {}, canonicalSha256: randomUUID(), createdByActorType: "agent", createdByActorId: f.agentId });
|
||||
await db.update(heartbeatRuns).set({ completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, runId: f.runId,
|
||||
completionContractId: contractId, serverFingerprint: randomUUID(), schemaStatus: "valid", resultJson: {}, canonicalSha256: randomUUID() });
|
||||
await db.update(nativeRunFinalizations).set({ resultId }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
}
|
||||
|
||||
await reconcileAbandonedExecutionControl(db);
|
||||
await reconcileAbandonedExecutionControl(db);
|
||||
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const [retained] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
|
||||
expect(run).toMatchObject(state === "accepted" ? { status: "running", errorCode: null }
|
||||
: { status: "failed", errorCode: "execution_finalization_deadline_exceeded" });
|
||||
if (state === "prepared" || state === "reassigned") {
|
||||
expect(run.resultJson).toMatchObject({ workspaceRestoreRecovery: { leaseIds: [lease.id] } });
|
||||
expect(retained).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure",
|
||||
metadata: { sandboxStopAndRetain: { runId: f.runId, providerLeaseId: lease.providerLeaseId } } });
|
||||
const recovery = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(recovery.filter(action => action.evidence?.workspaceRestoreRecovery)).toMatchObject([
|
||||
{ evidence: { runId: f.runId, workspaceRestoreRecovery: { leaseIds: [lease.id] } } },
|
||||
]);
|
||||
} else {
|
||||
expect(run.resultJson ?? {}).not.toHaveProperty("workspaceRestoreRecovery");
|
||||
expect(retained).toMatchObject({ status: "active", leasePolicy: "reuse_by_environment" });
|
||||
}
|
||||
const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
if (state === "reassigned") {
|
||||
expect(task.assigneeAgentId).not.toBe(f.agentId);
|
||||
expect(task).toMatchObject({ status: "in_progress", executionRunId: null, checkoutRunId: null });
|
||||
}
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describeEmbeddedPostgres("native review execution reconciliation admission", () => {
|
||||
|
||||
@@ -14,6 +14,7 @@ import { issueRecoveryActionService } from "./issue-recovery-actions.js";
|
||||
import { hasLiveLegacyController } from "./legacy-controller-lease.js";
|
||||
import { reportRunFailure } from "./run-failure-report.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
|
||||
import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js";
|
||||
|
||||
/** Only newly recorded control deadlines are eligible. Upgrades never replay ambiguous historical runs. */
|
||||
export async function reconcileAbandonedExecutionControl(
|
||||
@@ -233,8 +234,10 @@ export async function reconcileAbandonedExecutionControl(
|
||||
["done", "cancelled"].includes(task.status) ||
|
||||
(task.executionRunId && task.executionRunId !== run.id) ||
|
||||
(task.checkoutRunId && task.checkoutRunId !== run.id)
|
||||
)
|
||||
) {
|
||||
if (updatedRun) await preserveNativeFailedWorkspace(tx as unknown as Db, updatedRun);
|
||||
return true;
|
||||
}
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ executionRunId: null, checkoutRunId: null, updatedAt: now })
|
||||
@@ -261,6 +264,7 @@ export async function reconcileAbandonedExecutionControl(
|
||||
maxAttempts: 3,
|
||||
supersedeOnIdentityChange: true,
|
||||
});
|
||||
if (updatedRun) await preserveNativeFailedWorkspace(tx as unknown as Db, updatedRun);
|
||||
return true;
|
||||
});
|
||||
if (repaired) surfaced += 1;
|
||||
|
||||
@@ -26,6 +26,7 @@ import { resolveAssignedConnectionInstructionsForRun } from "../native-runtime/a
|
||||
import { recordFailedSkillPreparation, type NativeRunHistoricalSpan } from "../native-runtime/native-run-trace.js";
|
||||
import { collectRunFailureSecretValues } from "../run-failure-diagnostics.js";
|
||||
import { skillVersionSelectionMap } from "../runtime-skill-selections.js";
|
||||
import { environmentSkillConfig } from "../environment-skills.js";
|
||||
import type { selectHeartbeatEnvironment } from "./environment-selection.js";
|
||||
import type { createHeartbeatRetries } from "./retries.js";
|
||||
import type { createHeartbeatLifecycle } from "./run-lifecycle.js";
|
||||
@@ -231,18 +232,30 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
} else {
|
||||
delete context.paperclipSecrets;
|
||||
}
|
||||
const effectiveResolvedConfig = applyRunScopedMentionedSkillKeys(
|
||||
resolvedConfig,
|
||||
runScopedSkillKeys,
|
||||
);
|
||||
const storedVersions = parseObject(run.contextSnapshot?.skillVersions);
|
||||
const hasStoredVersions = Object.prototype.hasOwnProperty.call(run.contextSnapshot ?? {}, "skillVersions");
|
||||
const savedShadowedSkills = Array.isArray(run.contextSnapshot?.shadowedSkills)
|
||||
? run.contextSnapshot.shadowedSkills.flatMap(value => {
|
||||
const saved = parseObject(value);
|
||||
return typeof saved.key === "string" && typeof saved.shadowedBy === "string" && Object.hasOwn(storedVersions, saved.shadowedBy)
|
||||
? [{ key: saved.key, shadowedBy: saved.shadowedBy }] : [];
|
||||
}) : [];
|
||||
const selectedConfig = applyRunScopedMentionedSkillKeys(resolvedConfig, runScopedSkillKeys);
|
||||
const { config: effectiveResolvedConfig, skillKeys: environmentSkills, shadowedSkills } = hasStoredVersions
|
||||
? { config: selectedConfig, skillKeys: [], shadowedSkills: savedShadowedSkills }
|
||||
: await environmentSkillConfig(db, agent.companyId, selectedEnvironmentForConfig, selectedConfig);
|
||||
if (shadowedSkills?.length) {
|
||||
context.shadowedSkills = shadowedSkills;
|
||||
if (!hasStoredVersions) await appendRunEvent(run, { eventType: "skills.shadowed", stream: "system", level: "warn",
|
||||
message: shadowedSkills.map(({ key, shadowedBy }) => `Skill "${key}" was skipped because required plugin skill "${shadowedBy}" uses the same name.`).join("\n"),
|
||||
payload: { shadowedSkills } });
|
||||
}
|
||||
const runtimeSkillPreference = readPaperclipSkillSyncPreference(
|
||||
effectiveResolvedConfig,
|
||||
);
|
||||
const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
|
||||
const skillsPrepareStartedAtMs = Date.now();
|
||||
let skillDeliveryConfig = effectiveResolvedConfig;
|
||||
const storedVersions = parseObject(run.contextSnapshot?.skillVersions);
|
||||
const hasStoredVersions = Object.prototype.hasOwnProperty.call(run.contextSnapshot ?? {}, "skillVersions");
|
||||
const prepareSkills = async () => {
|
||||
|
||||
try {
|
||||
@@ -270,7 +283,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
// assignments from delivery; explicit task mentions remain errors below.
|
||||
skillDeliveryConfig = writePaperclipSkillSyncPreference(effectiveResolvedConfig,
|
||||
resolved.map(entry => ({ key: entry.key, versionId: entry.versionId ?? null })));
|
||||
for (const key of runScopedMentionedSkillKeys) {
|
||||
for (const key of [...runScopedMentionedSkillKeys.filter(key => !shadowedSkills?.some(skill => skill.key === key)), ...(hasStoredVersions ? [] : environmentSkills)]) {
|
||||
const entry = resolved.find(candidate => candidate.key === key);
|
||||
if (!entry || entry.sourceStatus === "missing") {
|
||||
throw new ConfigurationIncompleteFailure("A referenced skill is unavailable. Restore its installed version or update the task's skill reference.", {
|
||||
@@ -282,7 +295,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
const skillVersions = Object.fromEntries(resolved.map(entry => [entry.key, entry.versionId ?? entry.currentVersionId ?? null]));
|
||||
context.skillVersions = skillVersions;
|
||||
context.skillSelectionSnapshot = true;
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ skillVersions, skillSelectionSnapshot: true })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ skillVersions, skillSelectionSnapshot: true, ...(shadowedSkills?.length ? { shadowedSkills } : {}) })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));
|
||||
}
|
||||
return resolved;
|
||||
}));
|
||||
|
||||
@@ -1876,6 +1876,10 @@ export interface IssueFilters {
|
||||
*/
|
||||
assigneeAgentId?: string | null;
|
||||
participantAgentId?: string;
|
||||
issueIds?: string[];
|
||||
listProjection?: "index" | "rows";
|
||||
parentIds?: string[];
|
||||
oneChildPerParent?: boolean;
|
||||
assigneeUserId?: string;
|
||||
touchedByUserId?: string;
|
||||
inboxArchivedByUserId?: string;
|
||||
@@ -8038,6 +8042,32 @@ export function issueService(db: Db) {
|
||||
clearCheckoutRunIfTerminal,
|
||||
addStopRelayCommentIfNeeded,
|
||||
|
||||
unreadInboxIssueIds: async (companyId: string, userId: string, readCondition: SQL<boolean>) => {
|
||||
// Rank the visible Mine set before counting unread, just like the inbox.
|
||||
// This projection never reads descriptions, run payloads, or recovery details.
|
||||
const rows = await db.select({
|
||||
id: issues.id,
|
||||
}).from(issues).where(and(
|
||||
eq(issues.companyId, companyId),
|
||||
visibleIssueCondition(),
|
||||
isNull(issues.conversationAgentId),
|
||||
nonPluginOperationIssueCondition(),
|
||||
ne(issues.status, "cancelled"),
|
||||
touchedByUserCondition(companyId, userId),
|
||||
inboxVisibleForUserCondition(companyId, userId),
|
||||
readCondition,
|
||||
)).orderBy(desc(issueCanonicalLastActivityAtExpr(companyId)), desc(issues.updatedAt), desc(issues.id)).limit(100);
|
||||
if (rows.length === 0) return [];
|
||||
// Keep correlated predicates in WHERE (as in the list's user stats).
|
||||
// A single-table Drizzle SELECT projection unqualifies nested columns.
|
||||
return db.select({ id: issues.id }).from(issues).where(and(
|
||||
eq(issues.companyId, companyId),
|
||||
inArray(issues.id, rows.map((row) => row.id)),
|
||||
unreadForUserCondition(companyId, userId),
|
||||
readCondition,
|
||||
)).then((unread) => unread.map((row) => row.id));
|
||||
},
|
||||
|
||||
list: async (companyId: string, filters?: IssueFilters) => {
|
||||
if (filters?.sortField === "id" && filters.attention) {
|
||||
throw unprocessable("ID ordering is not supported for blocked attention lists");
|
||||
@@ -8060,7 +8090,15 @@ export function issueService(db: Db) {
|
||||
eq(issues.companyId, companyId),
|
||||
visibleIssueCondition(),
|
||||
];
|
||||
if (!filters?.q?.trim()) {
|
||||
if (filters?.issueIds) {
|
||||
if (filters.issueIds.length === 0) return [];
|
||||
conditions.push(inArray(issues.id, filters.issueIds));
|
||||
}
|
||||
if (filters?.parentIds) {
|
||||
if (filters.parentIds.length === 0) return [];
|
||||
conditions.push(inArray(issues.parentId, filters.parentIds));
|
||||
}
|
||||
if (!filters?.q?.trim() && !filters?.issueIds) {
|
||||
conditions.push(isNull(issues.conversationAgentId));
|
||||
if (!filters?.touchedByUserId && !filters?.unreadForUserId && !filters?.inboxArchivedByUserId) {
|
||||
conditions.push(nonIdleSlackIssueCondition());
|
||||
@@ -8218,7 +8256,14 @@ export function issueService(db: Db) {
|
||||
}
|
||||
const priorityOrder = sql`CASE ${issues.priority} WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END`;
|
||||
const searchOrder = sql<number>`-task_search.score`;
|
||||
const issueSource = db.select(issueListSelect).from(issues);
|
||||
const selectedFields = filters?.listProjection ? {
|
||||
...issueListSelect,
|
||||
description: sql<string | null>`NULL`,
|
||||
descriptionTruncated: sql<boolean>`false`,
|
||||
} : issueListSelect;
|
||||
const issueSource = (filters?.oneChildPerParent
|
||||
? db.selectDistinctOn([issues.parentId], selectedFields)
|
||||
: db.select(selectedFields)).from(issues);
|
||||
const searchedSource = hasSearch
|
||||
? issueSource.innerJoin(sql`(
|
||||
${taskSearchCtes(companyId, taskSearch, true, and(...conditions), filters?.readCondition)}
|
||||
@@ -8228,6 +8273,7 @@ export function issueService(db: Db) {
|
||||
const baseQuery = searchedSource
|
||||
.where(and(...conditions))
|
||||
.orderBy(
|
||||
...(filters?.oneChildPerParent ? [asc(issues.parentId)] : []),
|
||||
...issueListOrderBy(companyId, {
|
||||
hasSearch,
|
||||
priorityOrder,
|
||||
@@ -8252,7 +8298,7 @@ export function issueService(db: Db) {
|
||||
),
|
||||
}));
|
||||
const withLabels = await withIssueLabels(db, rows);
|
||||
const runMap = await activeRunMapForIssues(db, withLabels);
|
||||
const runMap = filters?.listProjection === "index" ? new Map() : await activeRunMapForIssues(db, withLabels);
|
||||
const withRuns = withActiveRuns(withLabels, runMap);
|
||||
if (withRuns.length === 0) {
|
||||
return withRuns;
|
||||
@@ -8298,8 +8344,8 @@ export function issueService(db: Db) {
|
||||
reviewAttentionByIssueId,
|
||||
blockedInboxAttentionByIssueId,
|
||||
] = await Promise.all([
|
||||
listIssueBlockerAttentionMap(db, companyId, withRuns),
|
||||
listIssueReviewAttentionMap(db, companyId, withRuns),
|
||||
filters?.listProjection === "index" ? Promise.resolve(new Map<string, IssueBlockerAttention>()) : listIssueBlockerAttentionMap(db, companyId, withRuns),
|
||||
filters?.listProjection === "index" ? Promise.resolve(new Map<string, IssueReviewAttention>()) : listIssueReviewAttentionMap(db, companyId, withRuns),
|
||||
includeBlockedInboxAttention
|
||||
? listIssueBlockedInboxAttentionMap(db, companyId, withRuns)
|
||||
: Promise.resolve(new Map<string, IssueBlockedInboxAttention>()),
|
||||
|
||||
@@ -100,6 +100,30 @@ describe("remote native artifact preparation", () => {
|
||||
expect(f.requests.some(request => request.command === "npm")).toBe(false);
|
||||
});
|
||||
|
||||
it("reads installed pack metadata in the existing final runner probe", async () => {
|
||||
const f = await fixture();
|
||||
const installation = { root: "/opt/paperclip-runner/provider-pack", manifest: { digest: "selected" }, immutable: true };
|
||||
await writeFile(f.runnerSource, `#!/bin/sh\n[ "$1" = --build-metadata ] || exit 17\nprintf '%s\\n' '${JSON.stringify({ ...metadata, providerPacks: [installation, { ...installation, root: "/unrequested" }] })}'\n`, { mode: 0o700 });
|
||||
const artifacts = createRemoteNativeArtifactPreparation({ ...f.options, providerPackRoots: [installation.root] });
|
||||
await artifacts.prepare("listen_ws");
|
||||
expect(artifacts.providerPackInventory()).toEqual([installation]);
|
||||
const probes = f.requests.filter(request => request.command === f.runnerTarget && request.args?.[0] === "--build-metadata");
|
||||
expect(probes).toHaveLength(1);
|
||||
expect(probes[0].args).toEqual(["--build-metadata", "--provider-pack-root", installation.root]);
|
||||
expect(f.requests.some(request => request.args?.includes("--version") && request.command.includes("provider-pack"))).toBe(false);
|
||||
});
|
||||
|
||||
it("reattaches an older runner without the optional pack metadata argument", async () => {
|
||||
const f = await fixture();
|
||||
await writeFile(f.runnerSource, `#!/bin/sh\n[ "$#" = 1 ] && [ "$1" = --build-metadata ] || exit 17\nprintf '%s\\n' '${JSON.stringify(metadata)}'\n`, { mode: 0o700 });
|
||||
const artifacts = createRemoteNativeArtifactPreparation({ ...f.options, providerPackRoots: ["/opt/paperclip-runner/provider-pack"] });
|
||||
await artifacts.verifyRemoteRunner("listen_ws", f.runnerSource, true);
|
||||
expect(artifacts.providerPackInventory()).toEqual([]);
|
||||
expect(f.requests.map(request => request.args)).toEqual([
|
||||
["--build-metadata", "--provider-pack-root", "/opt/paperclip-runner/provider-pack"], ["--build-metadata"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps providers without concurrent sync support serial", async () => {
|
||||
const f = await fixture({ concurrent: false });
|
||||
await createRemoteNativeArtifactPreparation(f.options).prepare("listen_ws");
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { RunnerIngressEndpoint } from "@paperclipai/adapter-utils/runner-connectivity";
|
||||
import { createNativeRunnerExecutionDriver } from "./native-runner-execution-driver.js";
|
||||
import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./native-runner-execution-driver.js";
|
||||
|
||||
const transport = vi.hoisted(() => ({
|
||||
register: vi.fn(),
|
||||
@@ -181,3 +181,12 @@ describe("shared Runner execution driver", () => {
|
||||
await expect(connection.ready!()).rejects.toThrow("process_identity_invalid");
|
||||
});
|
||||
});
|
||||
|
||||
it("validates connectivity before resolving provider artifacts without acquiring an ingress endpoint", async () => {
|
||||
const prepare = vi.fn(async () => {});
|
||||
await expect(prepareNativeRemoteRunner({ target: ssh, runId: "run", prepareRemote: prepare })).rejects.toThrow("runner_transport_ineligible");
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
await prepareNativeRemoteRunner({ target: sandbox(), runnerIngressAuthorized: true, runId: "run", prepareRemote: prepare });
|
||||
expect(prepare).toHaveBeenCalledExactlyOnceWith("listen_ws");
|
||||
expect(transport.connect).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -59,6 +59,30 @@ export function resolveRemoteRunnerTransportMode(input: {
|
||||
return requiredMode;
|
||||
}
|
||||
|
||||
type RemotePreparation = Pick<NativeRunnerExecutionDriverInput,
|
||||
"target" | "runnerIngressAuthorized" | "runnerPublicUrl" | "runnerCaBundlePath" | "prepareRemote"> & { runId: string };
|
||||
|
||||
async function resolveOutboundTransport(input: RemotePreparation): Promise<PaperclipRunnerTransport> {
|
||||
const transport = await resolvePaperclipRunnerTransport({ target: input.target, runId: input.runId,
|
||||
localConnectUrl: "ws://127.0.0.1/unused", runnerPublicUrl: input.runnerPublicUrl,
|
||||
runnerCaBundlePath: input.runnerCaBundlePath, runnerIngressAuthorized: input.runnerIngressAuthorized === true });
|
||||
if (transport.mode === "direct_outbound" && transport.caBundlePath && !existsSync(transport.caBundlePath)) {
|
||||
throw new Error("runner_direct_wss_failed: configured runner CA bundle is unavailable");
|
||||
}
|
||||
return transport;
|
||||
}
|
||||
|
||||
/** Resolve artifacts before provider-profile binding. Endpoint acquisition stays
|
||||
* in connection registration; outbound eligibility is checked before staging.
|
||||
*/
|
||||
export async function prepareNativeRemoteRunner(input: RemotePreparation) {
|
||||
const mode = resolveRemoteRunnerTransportMode({ target: input.target,
|
||||
runnerIngressAuthorized: input.runnerIngressAuthorized === true });
|
||||
if (mode === "dial_wss") await resolveOutboundTransport(input);
|
||||
if (!input.prepareRemote) throw new Error("runner_remote_preparation_unavailable");
|
||||
await input.prepareRemote(mode);
|
||||
}
|
||||
|
||||
/** Built-in environment lifecycle. PRP commands and authentication are shared;
|
||||
* only environment preparation, process ownership, and connection routing vary.
|
||||
*/
|
||||
@@ -116,28 +140,8 @@ export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDr
|
||||
transport = await measureNativeRunnerSpan(
|
||||
input.trace,
|
||||
"runner.transport.resolve",
|
||||
() =>
|
||||
resolvePaperclipRunnerTransport({
|
||||
target: input.target,
|
||||
runId:
|
||||
attachmentIdentity?.runId ??
|
||||
input.binding.runId,
|
||||
localConnectUrl: "ws://127.0.0.1/unused",
|
||||
runnerPublicUrl: input.runnerPublicUrl,
|
||||
runnerCaBundlePath: input.runnerCaBundlePath,
|
||||
runnerIngressAuthorized:
|
||||
input.runnerIngressAuthorized === true,
|
||||
}),
|
||||
() => resolveOutboundTransport({ ...input, runId: attachmentIdentity?.runId ?? input.binding.runId }),
|
||||
);
|
||||
if (
|
||||
transport.mode === "direct_outbound" &&
|
||||
transport.caBundlePath &&
|
||||
!existsSync(transport.caBundlePath)
|
||||
) {
|
||||
throw new Error(
|
||||
"runner_direct_wss_failed: configured runner CA bundle is unavailable",
|
||||
);
|
||||
}
|
||||
await measureNativeRunnerSpan(
|
||||
input.trace,
|
||||
"runner.artifact.prepare",
|
||||
|
||||
@@ -1440,6 +1440,7 @@ describe("remote provider pack manifest", () => {
|
||||
opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs",
|
||||
}).map(([key, path]) => [key, { path, sha256: digest }])) as Record<"nodeCommand" | "opencodeCommand", { path: string; sha256: string }>;
|
||||
const payload = {
|
||||
compatibility: { opencode: 1, acpx: 1 },
|
||||
target: { platform: "linux", architecture }, runnerSourceRevision: "a".repeat(40),
|
||||
pins: { nodeMinimum: "24.11.0", codex: QUALIFIED_ACPX_PROFILES.codex.agentRuntimeVersion,
|
||||
opencode: "1.18.34", acpx: "0.13.1", grok: QUALIFIED_ACPX_PROFILES.grok.agentRuntimeVersion,
|
||||
@@ -1449,6 +1450,11 @@ describe("remote provider pack manifest", () => {
|
||||
distDigest: digest, artifacts,
|
||||
bridgeDigest: "sha256:" + createHash("sha256").update(digest + "\n" + digest + "\n" + digest).digest("hex"),
|
||||
};
|
||||
if (provider === "cursor") Object.assign(payload, { providers: { cursor: {
|
||||
version: QUALIFIED_ACPX_PROFILES.cursor.agentServerVersion, profileDigest: QUALIFIED_ACPX_PROFILES.cursor.commandDigest,
|
||||
closureDigest: `sha256:${CURSOR_DISTRIBUTION_PINS["linux-x64"].closureSha256}`, qualification: "qualified",
|
||||
path: "provider-assets/cursor/linux-x64", sha256: digest,
|
||||
} } });
|
||||
const manifest = { schema: "paperclip-runner/remote-provider-pack/v1", payload,
|
||||
digest: "sha256:" + createHash("sha256").update(canonical(payload)).digest("hex") };
|
||||
state.bundledPackManifestPath = join(root, "provider-pack.json");
|
||||
@@ -1463,11 +1469,22 @@ describe("remote provider pack manifest", () => {
|
||||
workspace: { ...execution.workspace, cwd: root },
|
||||
session: { ...execution.session, normalizedSessionId: `bundled-${provider}-${randomUUID()}`, driverKind: provider === "opencode" ? "opencode_server" : "acpx_runtime" },
|
||||
} as unknown as NativeExecutionInputV1;
|
||||
const syncIn = vi.fn(async () => undefined);
|
||||
const execute = vi.fn(async (command: { command: string; args?: string[] }) => {
|
||||
if (command.args?.[0] === "--build-metadata") throw new Error("reached-staged-Linux-daemon-verification");
|
||||
return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };
|
||||
});
|
||||
const syncIn = vi.fn(async (_operations: { files: { kind: string }[] }[]) => undefined);
|
||||
const installed = structuredClone(manifest);
|
||||
if (provider === "opencode") {
|
||||
installed.payload.runnerSourceRevision = "c".repeat(40);
|
||||
installed.payload.artifacts.nodeCommand.sha256 = "sha256:" + "d".repeat(64);
|
||||
installed.digest = "sha256:" + createHash("sha256").update(canonical(installed.payload)).digest("hex");
|
||||
}
|
||||
const execute = vi.fn(async (command: { command: string; args?: string[] }) => ({
|
||||
exitCode: 0, signal: null, timedOut: false, stderr: "",
|
||||
stdout: command.args?.[0] === "--build-metadata" ? JSON.stringify({
|
||||
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
|
||||
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"],
|
||||
providerPacks: [{ root: "/opt/paperclip-runner/provider-pack", manifest: installed, immutable: true }],
|
||||
}) : "",
|
||||
}));
|
||||
try {
|
||||
await createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution,
|
||||
runnerInstanceId: "bundled-image-runner", runnerIngressAuthorized: true,
|
||||
@@ -1491,12 +1508,17 @@ describe("remote provider pack manifest", () => {
|
||||
const { bundledRemoteProviderPackManifestPath } = await import("../../vendor/paperclip-runner/index.js");
|
||||
expect(bundledRemoteProviderPackManifestPath).toHaveBeenCalledWith(undefined, `linux-${architecture}`);
|
||||
}
|
||||
await expect(transport.controlPlaneRegistration({})).rejects.toThrow("reached-staged-Linux-daemon-verification");
|
||||
expect(syncIn).toHaveBeenCalledWith([expect.objectContaining({ files: [expect.objectContaining({
|
||||
sourcePath: state.bundledRunnerBinary, targetPath: "/workspace/.paperclip-runtime/paperclip-runner/bin/paperclip-runnerd",
|
||||
})] })]);
|
||||
expect(execute.mock.calls.some(([call]) => call.args?.[1] === "uname -s; uname -m")).toBe(false);
|
||||
expect(existsSync(join(transport.stateDirectory!, "provider-pack-authority.json"))).toBe(false);
|
||||
expect(transport).toMatchObject({ providerPackAuthorityDigest: installed.digest,
|
||||
providerNodeCommand: "/opt/paperclip-runner/provider-pack/node_modules/node/bin/node",
|
||||
providerNodeCommandSha256: installed.payload.artifacts.nodeCommand.sha256 });
|
||||
expect(execute.mock.calls.filter(([call]) => call.args?.[0] === "--build-metadata")).toHaveLength(1);
|
||||
expect(execute.mock.calls.some(([call]) => call.args?.[0] === "-e" || call.args?.[0] === "--version")).toBe(false);
|
||||
expect(syncIn.mock.calls.every(([operations]) => operations.every(operation => operation.files.every(file => file.kind === "file")))).toBe(true);
|
||||
if (provider === "claude-x64") {
|
||||
const identity = transport.prpIdentity!;
|
||||
const artifact = { version: "0.0.0", digest: "sha256:" + createHash("sha256").update(daemon).digest("hex") };
|
||||
@@ -1514,24 +1536,30 @@ describe("remote provider pack manifest", () => {
|
||||
await writeFile(state.bundledPackManifestPath!, JSON.stringify(next));
|
||||
await writeFile(state.bundledRunnerBinary!, Buffer.concat([daemon, Buffer.from("new release")]));
|
||||
const recoveryExecute = vi.fn(async (request: { args: string[] }) => {
|
||||
if (request.args[0] === "--build-metadata") return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({
|
||||
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
|
||||
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"],
|
||||
providerPacks: [{ root: "/opt/paperclip-runner/provider-pack", manifest, immutable: true }],
|
||||
}) };
|
||||
if (request.args[0] === "-e" || request.args[0] === "--version") return { exitCode: 0, timedOut: false, stderr: "", stdout: request.args[0] === "--version" ? "1.18.34" : "" };
|
||||
const script = request.args[1];
|
||||
const stdout = script.includes("base64") ? Buffer.from(JSON.stringify({ ...identity, lifecycle: "running" })).toString("base64")
|
||||
: script.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "";
|
||||
return { stdout, stderr: "", exitCode: 0, timedOut: false };
|
||||
});
|
||||
await createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution,
|
||||
runnerInstanceId: identity.runnerInstanceId,
|
||||
runnerInstanceId: identity.runnerInstanceId, runnerIngressAuthorized: true,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace",
|
||||
environmentId: "environment", leaseId: "lease", providerKey: "daytona", runner: { execute: recoveryExecute },
|
||||
environmentId: "environment", leaseId: "lease", providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute: recoveryExecute },
|
||||
sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never,
|
||||
restartRecovery: { kind: "reattach_remote_runner", runId: remoteExecution.binding.runId,
|
||||
leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "graceful", recoveryRequestId: null,
|
||||
remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" } },
|
||||
});
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({
|
||||
providerPackAuthorityDigest: manifest.digest, runnerArtifact: artifact, adoptExistingRunner: { pid: 123 },
|
||||
});
|
||||
const recovered = state.createTransport.mock.calls.at(-1)![0];
|
||||
expect(recovered).toMatchObject({ providerPackAuthorityDigest: manifest.digest, runnerArtifact: artifact, adoptExistingRunner: { pid: 123 } });
|
||||
}
|
||||
} finally {
|
||||
state.bundledPackManifestPath = null; state.bundledRunnerBinary = null;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { providerPackWorkspaceId, recordAdmittedRemoteProviderPack, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js";
|
||||
import { verifyRemoteProviderPack as verifyRemoteProviderPackIdentity } from "./remote-provider-pack-verification.js";
|
||||
import { type ProviderPackInstallation, verifyRemoteProviderPack as verifyRemoteProviderPackIdentity } from "./remote-provider-pack-verification.js";
|
||||
import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import { writeExecutionControl } from "../execution-control-notifications.js";
|
||||
@@ -23,7 +23,7 @@ import { isCloudManagedInstance } from "../cloud-instance.js";
|
||||
import { dotRunnerBroker } from "../dot-runner-broker.js";
|
||||
import { CURSOR_DISTRIBUTION_PINS, QUALIFIED_ACPX_PROFILES, QUALIFIED_ACPX_VERSION } from "../../vendor/paperclip-runner/index.js";
|
||||
import { isProviderMode } from "../../vendor/paperclip-runner/index.js";
|
||||
import { bundledRemoteProviderPackManifestPath, bundledRemoteRunnerBinary, runnerBinaryTarget } from "../../vendor/paperclip-runner/index.js";
|
||||
import { PROVIDER_PACK_COMPATIBILITY, bundledRemoteProviderPackManifestPath, bundledRemoteRunnerBinary, runnerBinaryTarget } from "../../vendor/paperclip-runner/index.js";
|
||||
import { nativeRetryCancellationEligible, rethrowNativeCancellationLockConflict, assertCancellationRequest, cancellationIntentId as callerCancellationIntentId, cancellationRequestId } from "./native-cancellation-request.js";
|
||||
import { readNativePlanWait } from "./native-plan-wait.js";
|
||||
import { nativeProviderLifecycle } from "./provider-lifecycle.js";
|
||||
@@ -133,7 +133,7 @@ import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js";
|
||||
import { discoverRemoteExecutable, parseRemoteExecutableCandidate } from "./codex-model-fallback.js";
|
||||
export { parseRemoteExecutableCandidate } from "./codex-model-fallback.js";
|
||||
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
import { createNativeRunnerExecutionDriver, resolveRemoteRunnerTransportMode } from "./native-runner-execution-driver.js";
|
||||
import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver, resolveRemoteRunnerTransportMode } from "./native-runner-execution-driver.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
and,
|
||||
@@ -10316,6 +10316,7 @@ type RemoteProviderPackManifest = {
|
||||
schema: typeof REMOTE_PROVIDER_PACK_SCHEMA;
|
||||
digest: string;
|
||||
payload: {
|
||||
compatibility?: { opencode: number; acpx: number };
|
||||
pins: typeof REMOTE_PROVIDER_PACK_PINS;
|
||||
target: { platform: string; architecture: string };
|
||||
runnerSourceRevision: string;
|
||||
@@ -10427,6 +10428,7 @@ function readRemoteProviderPackIdentity(packRoot: string, verifyControllerFiles:
|
||||
if (
|
||||
manifest?.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||
|
||||
!payload ||
|
||||
(payload.compatibility !== undefined && canonicalJson(payload.compatibility) !== canonicalJson(PROVIDER_PACK_COMPATIBILITY)) ||
|
||||
canonicalJson(payload.pins) !== canonicalJson(REMOTE_PROVIDER_PACK_PINS) ||
|
||||
canonicalJson(payload.acpxProfileDigests) !==
|
||||
canonicalJson(REMOTE_PROVIDER_PACK_PROFILE_DIGESTS) ||
|
||||
@@ -12141,7 +12143,9 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
|| !lstatSync(configuredProviderPackRoot).isDirectory()) {
|
||||
throw new Error("runner_remote_provider_artifact_incompatible: configure PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH with the build-owned provider pack");
|
||||
}
|
||||
return readRemoteProviderPackManifest(configuredProviderPackRoot);
|
||||
// Selection reads identity only. Content verification belongs to the
|
||||
// install path; a compatible image pack needs no controller tree scan.
|
||||
return readRemoteProviderPackIdentity(configuredProviderPackRoot, false);
|
||||
})();
|
||||
return currentProviderPackManifest;
|
||||
};
|
||||
@@ -12261,6 +12265,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
: Promise.resolve();
|
||||
|
||||
const verifyRemoteProviderPack = async (packRoot: string) => {
|
||||
if (providerPackVerified && packRoot === activeRemoteProviderPackRoot) return;
|
||||
if (!remoteTarget || !remoteCommandRunner || !expectedProviderPackManifest)
|
||||
return;
|
||||
await verifyRemoteProviderPackIdentity({ runner: remoteCommandRunner,
|
||||
@@ -12293,6 +12298,8 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
model: input.execution.provider.kind === "codex" ? input.execution.provider.model : null,
|
||||
reuseRetainedRunner: sandboxLeaseAcquisition?.outcome === "resumed", targetPlatformArtifactVerified: remotePiCompanion !== null || useBundledRemoteImageAssets || targetMatchedCodexArtifact || remoteClaude,
|
||||
trace: input.trace, onLog: input.onLog,
|
||||
providerPackRoots: requiresRemoteProviderPack && stagedRemoteProviderPackRoot
|
||||
? [stagedRemoteProviderPackRoot, "/opt/paperclip-runner/provider-pack"] : undefined,
|
||||
})
|
||||
: null;
|
||||
const verifyRemoteRunner = async (mode: "dial_wss" | "listen_ws", executable?: string | null) => {
|
||||
@@ -12300,6 +12307,33 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
};
|
||||
const verifyRemoteCodex = async () => { await remoteNativeArtifacts?.verifyRemoteCodex(); };
|
||||
|
||||
// A custom operator-selected pack is an explicit runtime selection. The
|
||||
// standard app-image path is the default transport source, not an upgrade request.
|
||||
const explicitProviderPackSelection = Boolean(input.runnerRemoteProviderPackPath?.trim()
|
||||
&& input.runnerRemoteProviderPackPath.trim() !== "/opt/paperclip-runner/provider-pack");
|
||||
const useInstalledProviderPack = async () => {
|
||||
const expected = expectedProviderPackManifest;
|
||||
if (!expected || !remoteTarget || !remoteCommandRunner) return false;
|
||||
for (const installation of remoteNativeArtifacts?.providerPackInventory() ?? []) {
|
||||
try {
|
||||
const manifest = await measureNativeRunnerSpan(input.trace, "provider_pack.compatibility", () => verifyRemoteProviderPackIdentity({
|
||||
runner: remoteCommandRunner, cwd: remoteTarget.remoteCwd, packRoot: installation.root,
|
||||
manifest: expected, installation,
|
||||
provider: input.execution.provider as { kind: "opencode" } | { kind: "acpx"; agent: string },
|
||||
requireExactIdentity: reattachProviderPack || explicitProviderPackSelection,
|
||||
}));
|
||||
if (!reattachProviderPack) providerPackAuthority!.select(manifest);
|
||||
expectedProviderPackManifest = manifest;
|
||||
activeRemoteProviderPackRoot = installation.root;
|
||||
providerPackVerified = true;
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const prepareRemoteRunner = async (
|
||||
requiredMode: "dial_wss" | "listen_ws",
|
||||
) => {
|
||||
@@ -12312,7 +12346,11 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
return;
|
||||
selectedRemoteMode = requiredMode;
|
||||
if (input.restartRecovery?.kind === "reattach_remote_runner" || input.remoteWarmRecovery) {
|
||||
await verifyRemoteRunner(requiredMode);
|
||||
await remoteNativeArtifacts?.verifyRemoteRunner(requiredMode, undefined, true);
|
||||
if (requiresRemoteProviderPack && await useInstalledProviderPack()) {
|
||||
remotePrepared = true;
|
||||
return;
|
||||
}
|
||||
if (requiresRemoteProviderPack && stagedRemoteProviderPackRoot) {
|
||||
await verifyRemoteProviderPack(stagedRemoteProviderPackRoot);
|
||||
activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot;
|
||||
@@ -12321,6 +12359,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
return;
|
||||
}
|
||||
await remoteNativeArtifacts?.prepare(requiredMode);
|
||||
if (requiresRemoteProviderPack && await useInstalledProviderPack()) {
|
||||
remotePrepared = true;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
requiresRemoteProviderPack &&
|
||||
expectedProviderPackManifest &&
|
||||
@@ -12383,8 +12425,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
"runner_remote_provider_artifact_incompatible: this remote transport cannot stage a provider pack; preinstall the exact manifest-matched pack",
|
||||
);
|
||||
}
|
||||
const archive = await measureNativeRunnerSpan(input.trace, "provider_pack.archive.verify", () =>
|
||||
readProviderPackArchive(configuredProviderPackRoot, expectedProviderPackManifest!.digest),
|
||||
const archive = await measureNativeRunnerSpan(input.trace, "provider_pack.archive.verify", () => {
|
||||
readRemoteProviderPackManifest(configuredProviderPackRoot);
|
||||
return readProviderPackArchive(configuredProviderPackRoot, expectedProviderPackManifest!.digest);
|
||||
},
|
||||
);
|
||||
const escapedPackRoot = stagedRemoteProviderPackRoot.replaceAll(
|
||||
"'",
|
||||
@@ -13365,6 +13409,11 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
if (!current) throw new Error("native_session_tool_authority_unavailable");
|
||||
return current.execute(call);
|
||||
};
|
||||
if (requiresRemoteProviderPack) {
|
||||
await prepareNativeRemoteRunner({ target, runnerIngressAuthorized: input.runnerIngressAuthorized === true,
|
||||
runnerPublicUrl: input.runnerPublicUrl, runnerCaBundlePath: input.runnerCaBundlePath,
|
||||
runId: input.execution.binding.runId, prepareRemote: prepareRemoteRunner });
|
||||
}
|
||||
const backend = createNativeSessionBackend(runnerExecution, {
|
||||
runnerInstanceId:
|
||||
retainedTransition?.runnerInstanceId ?? input.runnerInstanceId,
|
||||
@@ -13476,10 +13525,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(expectedProviderPackManifest && stagedRemoteProviderPackRoot
|
||||
...(expectedProviderPackManifest && activeRemoteProviderPackRoot
|
||||
? {
|
||||
providerNodeCommand: posix.join(
|
||||
stagedRemoteProviderPackRoot,
|
||||
activeRemoteProviderPackRoot,
|
||||
expectedProviderPackManifest.payload.artifacts.nodeCommand.path,
|
||||
),
|
||||
providerNodeCommandSha256:
|
||||
@@ -13487,7 +13536,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
.sha256,
|
||||
providerPackAuthorityDigest: expectedProviderPackManifest.digest,
|
||||
opencodeCommand: posix.join(
|
||||
stagedRemoteProviderPackRoot,
|
||||
activeRemoteProviderPackRoot,
|
||||
expectedProviderPackManifest.payload.artifacts
|
||||
.opencodeExecutable.path,
|
||||
),
|
||||
@@ -13495,7 +13544,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
expectedProviderPackManifest.payload.artifacts
|
||||
.opencodeExecutable.sha256,
|
||||
opencodeProxyPath: posix.join(
|
||||
stagedRemoteProviderPackRoot,
|
||||
activeRemoteProviderPackRoot,
|
||||
expectedProviderPackManifest.payload.artifacts.opencodeProxy
|
||||
.path,
|
||||
),
|
||||
@@ -13503,7 +13552,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
expectedProviderPackManifest.payload.artifacts.opencodeProxy
|
||||
.sha256,
|
||||
acpxSidecarPath: posix.join(
|
||||
stagedRemoteProviderPackRoot,
|
||||
activeRemoteProviderPackRoot,
|
||||
expectedProviderPackManifest.payload.artifacts.acpxSidecar.path,
|
||||
),
|
||||
acpxSidecarSha256:
|
||||
@@ -13768,6 +13817,7 @@ export function createRemoteNativeArtifactPreparation(input: {
|
||||
model?: string | null;
|
||||
reuseRetainedRunner?: boolean;
|
||||
targetPlatformArtifactVerified?: boolean;
|
||||
providerPackRoots?: readonly string[];
|
||||
trace?: NativeRunTrace;
|
||||
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
||||
}) {
|
||||
@@ -13778,18 +13828,25 @@ export function createRemoteNativeArtifactPreparation(input: {
|
||||
const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null;
|
||||
const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() || null;
|
||||
if (explicitRemoteCodex && remoteCodexNpmSpec) throw new Error("runner_remote_codex_source_conflict");
|
||||
let providerPackInventory: ProviderPackInstallation[] = [];
|
||||
const verifyRemoteRunner = async (
|
||||
requiredMode: "dial_wss" | "listen_ws",
|
||||
executable = remoteBinary,
|
||||
inspectProviderPacks = false,
|
||||
) => {
|
||||
if (!remoteTarget || !remoteCommandRunner || !executable) return;
|
||||
const metadataResult = await remoteCommandRunner.execute({
|
||||
let metadataResult = await remoteCommandRunner.execute({
|
||||
command: executable,
|
||||
args: ["--build-metadata"],
|
||||
args: ["--build-metadata", ...(inspectProviderPacks ? (input.providerPackRoots ?? []).flatMap(root => ["--provider-pack-root", root]) : [])],
|
||||
cwd: remoteTarget.remoteCwd,
|
||||
bypassSession: true,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
if (inspectProviderPacks && input.providerPackRoots?.length && metadataResult.exitCode !== null
|
||||
&& metadataResult.exitCode !== 0 && !metadataResult.timedOut && !metadataResult.signal) {
|
||||
metadataResult = await remoteCommandRunner.execute({ command: executable, args: ["--build-metadata"],
|
||||
cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 30_000 });
|
||||
}
|
||||
if (metadataResult.exitCode !== 0 || metadataResult.timedOut) {
|
||||
throw new Error("runner_remote_artifact_verification_failed");
|
||||
}
|
||||
@@ -13802,6 +13859,12 @@ export function createRemoteNativeArtifactPreparation(input: {
|
||||
});
|
||||
}
|
||||
assertRemoteRunnerBuildMetadata(metadata, requiredMode);
|
||||
if (inspectProviderPacks) {
|
||||
providerPackInventory = Array.isArray(metadata.providerPacks)
|
||||
? metadata.providerPacks.filter((pack): pack is ProviderPackInstallation =>
|
||||
typeof pack?.root === "string" && input.providerPackRoots?.includes(pack.root) === true
|
||||
&& typeof pack.immutable === "boolean") : [];
|
||||
}
|
||||
};
|
||||
|
||||
const reportedCodexVersions = new Set<string | null>();
|
||||
@@ -13982,7 +14045,7 @@ export function createRemoteNativeArtifactPreparation(input: {
|
||||
});
|
||||
}
|
||||
await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () =>
|
||||
verifyRemoteRunner(requiredMode),
|
||||
verifyRemoteRunner(requiredMode, undefined, true),
|
||||
);
|
||||
};
|
||||
const prepareHarness = async () => {
|
||||
@@ -14097,5 +14160,5 @@ export function createRemoteNativeArtifactPreparation(input: {
|
||||
await prepareHarness();
|
||||
}
|
||||
};
|
||||
return { prepare, verifyRemoteRunner, verifyRemoteCodex };
|
||||
return { prepare, verifyRemoteRunner, verifyRemoteCodex, providerPackInventory: () => providerPackInventory };
|
||||
}
|
||||
@@ -119,3 +119,14 @@ it("retains the original projectless lease across warm run placeholders but reje
|
||||
const changed = { ...binding, workspaceId: providerPackWorkspaceId("project-workspace", "run-2", "original-lease") };
|
||||
expect(() => resolveRemoteProviderPackAuthority({ ...input, binding: changed, reattach: true })).toThrow("does not match this execution");
|
||||
});
|
||||
|
||||
it("persists the compatible installation selected at launch, then pins it on reattachment", () => {
|
||||
const { input } = fixture();
|
||||
const authority = resolveRemoteProviderPackAuthority(input);
|
||||
const installed = manifest("b");
|
||||
authority.select(installed);
|
||||
authority.verified();
|
||||
const attached = resolveRemoteProviderPackAuthority({ ...input, reattach: true });
|
||||
expect(attached.manifest).toEqual(installed);
|
||||
expect(() => attached.select(manifest("c"))).toThrow("cannot replace an attached pack authority");
|
||||
});
|
||||
@@ -33,7 +33,7 @@ export function resolveRemoteProviderPackAuthority<T extends ProviderPackIdentit
|
||||
reattach: boolean;
|
||||
current: () => T;
|
||||
runnerArtifact: () => RunnerArtifactIdentity;
|
||||
}): { manifest: T; runnerArtifact: RunnerArtifactIdentity; verified: () => void } {
|
||||
}): { manifest: T; runnerArtifact: RunnerArtifactIdentity; select: (manifest: T) => void; verified: () => void } {
|
||||
const root = lstatSync(input.stateDirectory);
|
||||
if (!root.isDirectory() || root.isSymbolicLink()) {
|
||||
throw new Error("runner_remote_provider_artifact_incompatible: provider pack authority directory is unsafe");
|
||||
@@ -68,11 +68,22 @@ export function resolveRemoteProviderPackAuthority<T extends ProviderPackIdentit
|
||||
assertManifest(manifest);
|
||||
runnerArtifact ??= input.runnerArtifact();
|
||||
const selectedRunner = structuredClone(runnerArtifact);
|
||||
const selected = structuredClone(manifest);
|
||||
const receipt = JSON.stringify({ schema, binding: input.binding, manifest: selected, runnerArtifact: selectedRunner });
|
||||
if (Buffer.byteLength(receipt) > maximumBytes) throw new Error("runner_remote_provider_artifact_incompatible: provider pack authority exceeds its size limit");
|
||||
let selected = structuredClone(manifest);
|
||||
const serialize = (manifest: T) => {
|
||||
const receipt = JSON.stringify({ schema, binding: input.binding, manifest, runnerArtifact: selectedRunner });
|
||||
if (Buffer.byteLength(receipt) > maximumBytes) throw new Error("runner_remote_provider_artifact_incompatible: provider pack authority exceeds its size limit");
|
||||
return receipt;
|
||||
};
|
||||
let receipt = serialize(selected);
|
||||
return {
|
||||
manifest: selected,
|
||||
get manifest() { return selected; },
|
||||
select: (manifest: T) => {
|
||||
if (input.reattach) throw new Error("runner_remote_provider_artifact_incompatible: cannot replace an attached pack authority");
|
||||
assertManifest(manifest);
|
||||
const next = structuredClone(manifest);
|
||||
receipt = serialize(next);
|
||||
selected = next;
|
||||
},
|
||||
runnerArtifact: selectedRunner,
|
||||
verified: () => {
|
||||
const temporary = `${path}.${randomUUID()}.tmp`;
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { canonicalProviderPackJson, verifyRemoteProviderPack, type ProviderPackIdentity } from "./remote-provider-pack-verification.js";
|
||||
|
||||
const sha = (text: string) => `sha256:${createHash("sha256").update(text).digest("hex")}`;
|
||||
const manifest = (revision: string): ProviderPackIdentity => {
|
||||
const payload = {
|
||||
compatibility: { opencode: 1, acpx: 1 },
|
||||
pins: { opencode: "1.18.34", nodeMinimum: "24.11.0" },
|
||||
target: { platform: "linux", architecture: "x64" },
|
||||
runnerSourceRevision: revision.repeat(40),
|
||||
artifacts: Object.fromEntries(Object.entries({
|
||||
nodeCommand: "node_modules/node/bin/node", opencodeCommand: "node_modules/.bin/opencode",
|
||||
productionLock: "pnpm-lock.yaml", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe",
|
||||
opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs",
|
||||
grokLauncher: "dist/providers/grok/launcher.cjs",
|
||||
}).map(([name, path]) => [name, { path, sha256: sha(`${revision}-${name}`) }])) as ProviderPackIdentity["payload"]["artifacts"],
|
||||
};
|
||||
return { schema: "paperclip-runner/remote-provider-pack/v1", payload, digest: sha(canonicalProviderPackJson(payload)) };
|
||||
};
|
||||
|
||||
describe("installed provider pack compatibility", () => {
|
||||
it("reuses a protected pack across app revisions without executing verification commands", async () => {
|
||||
const installed = manifest("a");
|
||||
const current = structuredClone(installed);
|
||||
current.payload.runnerSourceRevision = "b".repeat(40);
|
||||
current.digest = sha(canonicalProviderPackJson(current.payload));
|
||||
const root = await mkdtemp(join(tmpdir(), "installed-pack-compatibility-"));
|
||||
await mkdir(dirname(join(root, installed.payload.artifacts.nodeCommand.path)), { recursive: true });
|
||||
await symlink(process.execPath, join(root, installed.payload.artifacts.nodeCommand.path));
|
||||
await writeFile(join(root, "provider-pack.json"), JSON.stringify(installed));
|
||||
const execute = vi.fn(async (request: Parameters<CommandManagedRuntimeRunner["execute"]>[0]) => {
|
||||
try {
|
||||
const result = await promisify(execFile)(request.command, request.args ?? [], { cwd: root, timeout: 2_000 });
|
||||
return { ...result, exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString() };
|
||||
} catch (error) {
|
||||
const failed = error as Error & { code: number; stdout: string; stderr: string };
|
||||
if (typeof failed.code !== "number") throw error;
|
||||
return { stdout: failed.stdout, stderr: failed.stderr, exitCode: failed.code, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString() };
|
||||
}
|
||||
});
|
||||
const input = { runner: { execute }, cwd: root, packRoot: root, manifest: current };
|
||||
try {
|
||||
// Exercise the old complete-manifest comparison in a real Node process.
|
||||
// Only provenance changed; provider bytes and interface are identical.
|
||||
await expect(verifyRemoteProviderPack(input)).rejects.toThrow("manifest mismatch");
|
||||
expect(execute).toHaveBeenCalledOnce();
|
||||
execute.mockClear();
|
||||
const selected = await verifyRemoteProviderPack({ ...input,
|
||||
installation: { root, manifest: installed, immutable: true }, provider: { kind: "opencode" } });
|
||||
expect(selected).toEqual(installed);
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
const reseal = (pack: ProviderPackIdentity) => ({ ...pack, digest: sha(canonicalProviderPackJson(pack.payload)) });
|
||||
const check = (installed: ProviderPackIdentity, options: { immutable?: boolean; requireExactIdentity?: boolean } = {}) => {
|
||||
const execute = vi.fn();
|
||||
return verifyRemoteProviderPack({ runner: { execute }, cwd: "/workspace", packRoot: "/opt/paperclip-runner/provider-pack",
|
||||
manifest: manifest("b"), installation: { root: "/opt/paperclip-runner/provider-pack", manifest: installed, immutable: options.immutable ?? true },
|
||||
provider: { kind: "opencode" }, requireExactIdentity: options.requireExactIdentity });
|
||||
};
|
||||
it.each(["interface", "platform", "schema", "artifact_path", "digest", "legacy"])("rejects an incompatible %s", async field => {
|
||||
let pack = manifest("a");
|
||||
if (field === "interface") pack.payload.compatibility!.opencode += 1;
|
||||
if (field === "legacy") delete pack.payload.compatibility;
|
||||
if (field === "platform") pack.payload.target.architecture = "arm64";
|
||||
if (field === "schema") pack.schema = "unknown";
|
||||
if (field === "artifact_path") pack.payload.artifacts.nodeCommand.path = "../node";
|
||||
pack = reseal(pack);
|
||||
if (field === "digest") pack.digest = sha("corrupt");
|
||||
await expect(check(pack)).rejects.toThrow("runner_remote_provider_artifact_incompatible");
|
||||
});
|
||||
it("keeps writable installations at the content-verification boundary", async () => {
|
||||
await expect(check(manifest("b"), { immutable: false })).rejects.toThrow("installation is incompatible");
|
||||
});
|
||||
it("does not replace an admitted pack during reattachment", async () => {
|
||||
await expect(check(manifest("a"), { requireExactIdentity: true })).rejects.toThrow("incompatible");
|
||||
await expect(check(manifest("b"), { requireExactIdentity: true })).resolves.toEqual(manifest("b"));
|
||||
});
|
||||
it("ignores unrelated provider and dependency changes for OpenCode", async () => {
|
||||
const pack = manifest("a");
|
||||
pack.payload.compatibility!.acpx += 1;
|
||||
pack.payload.pins.acpx = "different-version";
|
||||
pack.payload.acpxProfileDigests = { codex: sha("different-codex") };
|
||||
await expect(check(reseal(pack))).resolves.toEqual(reseal(pack));
|
||||
});
|
||||
it("checks only the selected ACPX profile", async () => {
|
||||
const current = manifest("b"), installed = manifest("a");
|
||||
current.payload.acpxProfileDigests = { claude: sha("claude"), codex: sha("codex") };
|
||||
installed.payload.acpxProfileDigests = { claude: sha("claude"), codex: sha("other-codex") };
|
||||
const input = { runner: { execute: vi.fn() }, cwd: "/workspace", packRoot: "/opt/paperclip-runner/provider-pack",
|
||||
manifest: reseal(current), installation: { root: "/opt/paperclip-runner/provider-pack", manifest: reseal(installed), immutable: true },
|
||||
provider: { kind: "acpx" as const, agent: "claude" } };
|
||||
await expect(verifyRemoteProviderPack(input)).resolves.toEqual(reseal(installed));
|
||||
await expect(verifyRemoteProviderPack({ ...input, provider: { kind: "acpx", agent: "codex" } })).rejects.toThrow("profile is incompatible");
|
||||
});
|
||||
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { posix } from "node:path";
|
||||
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
import { runRemoteProviderVerification } from "./remote-provider-pack.js";
|
||||
@@ -13,6 +14,10 @@ export type ProviderPackIdentity = {
|
||||
schema: string;
|
||||
digest: string;
|
||||
payload: {
|
||||
compatibility?: { opencode: number; acpx: number };
|
||||
acpxProfileDigests?: Record<string, string>;
|
||||
providers?: Record<string, unknown>;
|
||||
candidateProviders?: Record<string, unknown>;
|
||||
pins: { opencode: string; [key: string]: string | null };
|
||||
target: { platform: string; architecture: string };
|
||||
runnerSourceRevision: string;
|
||||
@@ -20,12 +25,66 @@ export type ProviderPackIdentity = {
|
||||
};
|
||||
};
|
||||
|
||||
export async function verifyRemoteProviderPack({ runner, cwd, packRoot, manifest }: {
|
||||
export type ProviderPackInstallation = { root: string; manifest: unknown; immutable: boolean };
|
||||
export type PackProvider = { kind: "opencode" } | { kind: "acpx"; agent: string };
|
||||
|
||||
/** Admission of a build-qualified, OS-protected installation. Source revisions,
|
||||
* dependency pins and unrelated provider changes are provenance, not an interface.
|
||||
* The metadata comes from the trusted runner, never from a provider tool result.
|
||||
*/
|
||||
function compatibleInstallation<T extends ProviderPackIdentity>(
|
||||
installation: ProviderPackInstallation, expected: T, packRoot: string,
|
||||
provider: PackProvider, requireExactIdentity: boolean,
|
||||
): T {
|
||||
const actual = installation.manifest as T | null;
|
||||
const payload = actual?.payload;
|
||||
const contract = expected.payload.compatibility?.[provider.kind];
|
||||
if (!installation.immutable || installation.root !== packRoot || !payload
|
||||
|| actual.schema !== expected.schema
|
||||
|| actual.digest !== `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}`
|
||||
|| canonicalProviderPackJson(payload.target) !== canonicalProviderPackJson(expected.payload.target)
|
||||
|| (requireExactIdentity ? actual.digest !== expected.digest
|
||||
: !Number.isSafeInteger(contract) || contract! < 1 || payload.compatibility?.[provider.kind] !== contract)
|
||||
|| typeof payload.pins?.opencode !== "string") {
|
||||
throw new Error("runner_remote_provider_artifact_incompatible: installed provider pack interface or installation is incompatible");
|
||||
}
|
||||
// Launch profiles and recovery receipts bind the selected installation's own
|
||||
// bytes, never the current controller's artifact hashes.
|
||||
for (const [name, expectedArtifact] of Object.entries(expected.payload.artifacts)) {
|
||||
const artifact = payload.artifacts?.[name];
|
||||
if (artifact?.path !== expectedArtifact.path || !/^sha256:[a-f0-9]{64}$/.test(artifact.sha256)) {
|
||||
throw new Error("runner_remote_provider_artifact_incompatible: installed provider pack artifact identity is invalid");
|
||||
}
|
||||
}
|
||||
if (provider.kind === "acpx") {
|
||||
const agent = provider.agent;
|
||||
const currentProfile = expected.payload.acpxProfileDigests?.[agent];
|
||||
const selected = expected.payload.providers?.[agent] ?? expected.payload.candidateProviders?.[agent];
|
||||
const installed = payload.providers?.[agent] ?? payload.candidateProviders?.[agent];
|
||||
// ACPX's existing qualified launch profiles remain authoritative for the
|
||||
// selected agent. Other providers in the same pack do not invalidate it.
|
||||
if (currentProfile ? payload.acpxProfileDigests?.[agent] !== currentProfile
|
||||
: !selected || !installed || ["version", "profileDigest", "closureDigest", "qualification", "path"].some(key =>
|
||||
(installed as Record<string, unknown>)[key] !== (selected as Record<string, unknown>)[key])) {
|
||||
throw new Error("runner_remote_provider_artifact_incompatible: selected ACPX provider profile is incompatible");
|
||||
}
|
||||
}
|
||||
return structuredClone(actual);
|
||||
}
|
||||
|
||||
export async function verifyRemoteProviderPack<T extends ProviderPackIdentity>({ runner, cwd, packRoot, manifest, installation, provider, requireExactIdentity = false }: {
|
||||
runner: CommandManagedRuntimeRunner;
|
||||
cwd: string;
|
||||
packRoot: string;
|
||||
manifest: ProviderPackIdentity;
|
||||
}): Promise<void> {
|
||||
manifest: T;
|
||||
installation?: ProviderPackInstallation;
|
||||
provider?: PackProvider;
|
||||
requireExactIdentity?: boolean;
|
||||
}): Promise<T> {
|
||||
if (installation) {
|
||||
if (!provider) throw new Error("runner_remote_provider_artifact_incompatible: provider interface is required");
|
||||
return compatibleInstallation(installation, manifest, packRoot, provider, requireExactIdentity);
|
||||
}
|
||||
const expected = Buffer.from(
|
||||
canonicalProviderPackJson(manifest),
|
||||
"utf8",
|
||||
@@ -82,4 +141,5 @@ export async function verifyRemoteProviderPack({ runner, cwd, packRoot, manifest
|
||||
"runner_remote_provider_artifact_incompatible: OpenCode version mismatch",
|
||||
);
|
||||
}
|
||||
return structuredClone(manifest);
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
import { and, desc, eq } from "drizzle-orm";
|
||||
import { agents, plugins, pluginCompanySettings, resourceLifecycleEvents, type Db } from "@paperclipai/db";
|
||||
import { agents, environments, plugins, pluginCompanySettings, resourceLifecycleEvents, type Db } from "@paperclipai/db";
|
||||
import type { PluginAgentRuntimeState, PluginAgentRuntimeIdentity } from "@paperclipai/plugin-sdk";
|
||||
import { agentRecordQueries } from "../lib/agent-records.js";
|
||||
import { conflict, forbidden } from "../errors.js";
|
||||
import { conflict, forbidden, unprocessable } from "../errors.js";
|
||||
import { agentInstructionsService } from "./agent-instructions.js";
|
||||
import { pluginAgentConnectionAssignments } from "./plugin-agent-connections.js";
|
||||
import { companySkillService } from "./company-skills.js";
|
||||
@@ -11,6 +11,7 @@ import { readPaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/ser
|
||||
import { skillVersionSelectionMap } from "./runtime-skill-selections.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { agentInstanceSecretService } from "./agent-instance-secrets.js";
|
||||
import { environmentSkillConfig } from "./environment-skills.js";
|
||||
|
||||
async function readState(db: Db, pluginId: string, companyId: string, agentId: string, identityOnly: true): Promise<PluginAgentRuntimeIdentity | null>;
|
||||
async function readState(db: Db, pluginId: string, companyId: string, agentId: string, identityOnly?: false): Promise<Omit<PluginAgentRuntimeState, "skills" | "connectionAssignments"> | null>;
|
||||
@@ -50,24 +51,33 @@ export async function pluginAgentRuntimeIdentity(db: Db, pluginId: string, compa
|
||||
return readState(db, pluginId, companyId, agentId, true);
|
||||
}
|
||||
|
||||
export async function pluginAgentRuntimeState(db: Db, pluginId: string, companyId: string, agentId: string): Promise<PluginAgentRuntimeState | null> {
|
||||
export async function pluginAgentRuntimeState(db: Db, pluginId: string, companyId: string, agentId: string, environmentId?: string, installEnvironmentSkills = false): Promise<PluginAgentRuntimeState | null> {
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const state = await readState(db, pluginId, companyId, agentId);
|
||||
if (!state) return null;
|
||||
const preference = readPaperclipSkillSyncPreference(state.agent.adapterConfig);
|
||||
const settings = await instanceSettingsService(db).getExperimental();
|
||||
const instance = await instanceSettingsService(db).get();
|
||||
const effectiveEnvironmentId = environmentId ?? state.agent.defaultEnvironmentId ?? instance.defaultEnvironmentId;
|
||||
const [environment] = effectiveEnvironmentId
|
||||
? await db.select().from(environments).where(eq(environments.id, effectiveEnvironmentId)) : [];
|
||||
if (environmentId && !environment) throw unprocessable("The selected environment is unavailable");
|
||||
const { config: selectionConfig, skillKeys: environmentSkills, shadowedSkills } = await environmentSkillConfig(db, companyId, environment, state.agent.adapterConfig, installEnvironmentSkills);
|
||||
const preference = readPaperclipSkillSyncPreference(selectionConfig);
|
||||
const settings = instance.experimental;
|
||||
const skills = await withSkillActor({ type: "agent", agentId, companyId, source: "agent_key" }, () =>
|
||||
companySkillService(db).listRuntimeSkillEntries(companyId, {
|
||||
selectionConfig: state.agent.adapterConfig, includeLegacy: state.agent.adapterType !== "paperclip_runner", snapshotLatest: true,
|
||||
selectionConfig, includeLegacy: state.agent.adapterType !== "paperclip_runner", snapshotLatest: true,
|
||||
versionSelections: skillVersionSelectionMap(preference.desiredSkillEntries, { versionPinsEnabled: settings.enableBetaSkills }),
|
||||
}));
|
||||
if (environmentSkills.some(key => !skills.some(skill => skill.key === key && skill.sourceStatus !== "missing"))) {
|
||||
throw unprocessable("An environment skill is unavailable to this agent");
|
||||
}
|
||||
const connectionAssignments = await pluginAgentConnectionAssignments(db, companyId, agentId, { publicMcp: settings.enablePublicMcp, chatConnectors: settings.enableChatConnectors });
|
||||
const fallback = !state.instructionsRootPath ? await agentInstructionsService().exportFiles(state.agent, { rejectSymlinks: true }) : null;
|
||||
const entryFile = fallback?.entryFile ?? state.instructionsEntryFile;
|
||||
const entryContent = fallback?.files[entryFile];
|
||||
const current = await readState(db, pluginId, companyId, agentId);
|
||||
if (!current) return null;
|
||||
if (current.revision === state.revision) return { ...current, instructionsEntryFile: entryFile, ...(entryContent === undefined ? {} : { instructionsEntryContent: entryContent }), skills, connectionAssignments };
|
||||
if (current.revision === state.revision) return { ...current, instructionsEntryFile: entryFile, ...(entryContent === undefined ? {} : { instructionsEntryContent: entryContent }), skills, connectionAssignments, ...(shadowedSkills?.length ? { shadowedSkills } : {}) };
|
||||
}
|
||||
throw conflict("Agent runtime changed while resolving synchronization inputs; retry");
|
||||
}
|
||||
@@ -3010,7 +3010,7 @@ export function buildHostServices(
|
||||
async getRuntime(params) {
|
||||
const companyId = ensureCompanyId(params.companyId);
|
||||
await ensurePluginAvailableForCompany(companyId);
|
||||
return pluginAgentRuntimeState(db, pluginId, companyId, params.agentId);
|
||||
return pluginAgentRuntimeState(db, pluginId, companyId, params.agentId, params.environmentId, params.installEnvironmentSkills);
|
||||
},
|
||||
async list(params) {
|
||||
const companyId = ensureCompanyId(params.companyId);
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
pluginManagedResources,
|
||||
} from "@paperclipai/db";
|
||||
import { normalizeAgentUrlKey } from "@paperclipai/shared";
|
||||
import { joinFrontmatterBlock, normalizeAgentUrlKey, splitFrontmatterBlock } from "@paperclipai/shared";
|
||||
import type {
|
||||
CompanySkill,
|
||||
PaperclipPluginManifestV1,
|
||||
@@ -30,6 +30,11 @@ function canonicalSkillKey(pluginKey: string, skillKey: string) {
|
||||
return `plugin/${pluginKeySlug(pluginKey)}/${skillKey}`;
|
||||
}
|
||||
|
||||
function managedSkillName(pluginKey: string, declaration: PluginManagedSkillDeclaration) {
|
||||
const shortPlugin = pluginKeySlug(pluginKey.split(".").at(-1)!).replace(/^plugin-/, "");
|
||||
return `plugin-${shortPlugin}-${normalizeAgentUrlKey(declaration.slug ?? declaration.skillKey) ?? declaration.skillKey}`;
|
||||
}
|
||||
|
||||
function yamlString(value: string) {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
@@ -41,7 +46,8 @@ function buildDefaultMarkdown(
|
||||
const description = declaration.description?.trim() || `${declaration.displayName} plugin skill.`;
|
||||
return [
|
||||
"---",
|
||||
`name: ${yamlString(declaration.displayName)}`,
|
||||
`name: ${yamlString(managedSkillName(pluginKey, declaration))}`,
|
||||
`slug: ${yamlString(managedSkillName(pluginKey, declaration))}`,
|
||||
`description: ${yamlString(description)}`,
|
||||
`key: ${yamlString(canonicalSkillKey(pluginKey, declaration.skillKey))}`,
|
||||
"---",
|
||||
@@ -53,25 +59,24 @@ function buildDefaultMarkdown(
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function withManagedSkillKey(markdown: string, canonicalKey: string) {
|
||||
const keyLine = `key: ${yamlString(canonicalKey)}`;
|
||||
const normalized = markdown.replace(/\r\n/g, "\n");
|
||||
const frontmatter = /^---\n([\s\S]*?)\n---(\n?)/.exec(normalized);
|
||||
if (!frontmatter) {
|
||||
return [
|
||||
"---",
|
||||
keyLine,
|
||||
"---",
|
||||
"",
|
||||
normalized,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
const currentBody = frontmatter[1] ?? "";
|
||||
const nextBody = /^key\s*:/m.test(currentBody)
|
||||
? currentBody.replace(/^key\s*:.*$/m, keyLine)
|
||||
: [currentBody, keyLine].filter(Boolean).join("\n");
|
||||
return `---\n${nextBody}\n---${frontmatter[2] ?? ""}${normalized.slice(frontmatter[0].length)}`;
|
||||
function withManagedSkillKey(markdown: string, canonicalKey: string, name: string) {
|
||||
const block = splitFrontmatterBlock(markdown.replace(/\r\n/g, "\n"));
|
||||
const fields: Record<string, string> = { key: canonicalKey, name, slug: name };
|
||||
const seen = new Set<string>();
|
||||
let continuation = false;
|
||||
const lines = block.frontmatterText.split("\n").flatMap(line => {
|
||||
const field = /^(key|name|slug|"(?:key|name|slug)"|'(?:key|name|slug)')\s*:/.exec(line);
|
||||
if (field) {
|
||||
const key = field[1].replace(/["']/g, "");
|
||||
seen.add(key); continuation = true;
|
||||
return [`${key}: ${yamlString(fields[key])}`];
|
||||
}
|
||||
if (continuation && /^(?:\s|$)/.test(line)) return [];
|
||||
continuation = false;
|
||||
return [line];
|
||||
});
|
||||
for (const [key, value] of Object.entries(fields)) if (!seen.has(key)) lines.push(`${key}: ${yamlString(value)}`);
|
||||
return joinFrontmatterBlock({ ...block, hasFrontmatter: true, frontmatterText: lines.join("\n") });
|
||||
}
|
||||
|
||||
function buildPackageFiles(
|
||||
@@ -82,7 +87,7 @@ function buildPackageFiles(
|
||||
const canonicalKey = canonicalSkillKey(pluginKey, declaration.skillKey);
|
||||
const files: Record<string, string> = {
|
||||
[`${root}/SKILL.md`]: declaration.markdown?.trim()
|
||||
? withManagedSkillKey(declaration.markdown, canonicalKey)
|
||||
? withManagedSkillKey(declaration.markdown, canonicalKey, managedSkillName(pluginKey, declaration))
|
||||
: buildDefaultMarkdown(pluginKey, declaration),
|
||||
};
|
||||
for (const file of declaration.files ?? []) {
|
||||
@@ -112,7 +117,7 @@ function buildSkillDefaults(
|
||||
return {
|
||||
skillKey: declaration.skillKey,
|
||||
displayName: declaration.displayName,
|
||||
slug: declaration.slug ?? declaration.skillKey,
|
||||
slug: managedSkillName(pluginKey, declaration),
|
||||
description: declaration.description ?? null,
|
||||
canonicalKey: canonicalSkillKey(pluginKey, declaration.skillKey),
|
||||
files: [
|
||||
@@ -286,7 +291,7 @@ export function pluginManagedSkillService(
|
||||
const results = await skills.importPackageFiles(
|
||||
companyId,
|
||||
buildPackageFiles(options.pluginKey, declaration),
|
||||
{ onConflict: "replace" },
|
||||
{ onConflict: "replace", allowSlugCollision: true },
|
||||
);
|
||||
const imported = results.find((result) =>
|
||||
result.skill.key === canonicalSkillKey(options.pluginKey, declaration.skillKey)
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { and, desc, eq, inArray, not } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray, not, type SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agents, approvals, heartbeatRuns } from "@paperclipai/db";
|
||||
import { isHeartbeatRunVisibleInMine, type SidebarBadges } from "@paperclipai/shared";
|
||||
import { isHeartbeatRunVisibleInMine, latestFailedRunsByAgent, type SidebarBadges } from "@paperclipai/shared";
|
||||
import { authorizationService, type AuthorizationActor } from "./authorization.js";
|
||||
import { getHeartbeatRunReadDecisions } from "./heartbeat-run-privacy.js";
|
||||
|
||||
const ACTIONABLE_APPROVAL_STATUSES = ["pending", "revision_requested"];
|
||||
const FAILED_HEARTBEAT_STATUSES = ["failed", "timed_out"];
|
||||
|
||||
function normalizeTimestamp(value: Date | string | null | undefined): number {
|
||||
if (!value) return 0;
|
||||
@@ -23,6 +24,7 @@ function isDismissed(
|
||||
}
|
||||
|
||||
export function sidebarBadgeService(db: Db) {
|
||||
const authorization = authorizationService(db);
|
||||
return {
|
||||
get: async (
|
||||
companyId: string,
|
||||
@@ -31,6 +33,8 @@ export function sidebarBadgeService(db: Db) {
|
||||
dismissals?: ReadonlyMap<string, number>;
|
||||
joinRequests?: Array<{ id: string; updatedAt: Date | string | null; createdAt: Date | string }>;
|
||||
unreadTouchedIssues?: number;
|
||||
approvalReadCondition?: SQL<boolean>;
|
||||
actor?: AuthorizationActor;
|
||||
},
|
||||
): Promise<SidebarBadges> => {
|
||||
const actionableApprovals = await db
|
||||
@@ -40,16 +44,21 @@ export function sidebarBadgeService(db: Db) {
|
||||
and(
|
||||
eq(approvals.companyId, companyId),
|
||||
inArray(approvals.status, ACTIONABLE_APPROVAL_STATUSES),
|
||||
extra?.approvalReadCondition,
|
||||
),
|
||||
)
|
||||
.then((rows) =>
|
||||
rows.filter((row) => !isDismissed(extra?.dismissals ?? new Map(), `approval:${row.id}`, row.updatedAt)).length
|
||||
);
|
||||
|
||||
const latestRunByAgent = await db
|
||||
.selectDistinctOn([heartbeatRuns.agentId], {
|
||||
const recentRuns = await db
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
runStatus: heartbeatRuns.status,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
scopeKind: heartbeatRuns.scopeKind,
|
||||
issueId: heartbeatRuns.issueId,
|
||||
status: heartbeatRuns.status,
|
||||
responsibleUserId: heartbeatRuns.responsibleUserId,
|
||||
createdAt: heartbeatRuns.createdAt,
|
||||
})
|
||||
@@ -62,13 +71,16 @@ export function sidebarBadgeService(db: Db) {
|
||||
not(eq(agents.status, "terminated")),
|
||||
),
|
||||
)
|
||||
.orderBy(heartbeatRuns.agentId, desc(heartbeatRuns.createdAt));
|
||||
.orderBy(desc(heartbeatRuns.createdAt))
|
||||
.limit(200);
|
||||
|
||||
const failedRuns = latestRunByAgent.filter((row) =>
|
||||
FAILED_HEARTBEAT_STATUSES.includes(row.runStatus)
|
||||
&& (extra?.currentUserId === undefined || isHeartbeatRunVisibleInMine(row, extra.currentUserId))
|
||||
const failures = latestFailedRunsByAgent(recentRuns).filter((row) =>
|
||||
(extra?.currentUserId === undefined || isHeartbeatRunVisibleInMine(row, extra.currentUserId))
|
||||
&& !isDismissed(extra?.dismissals ?? new Map(), `run:${row.id}`, row.createdAt),
|
||||
).length;
|
||||
);
|
||||
const failedRuns = extra?.actor
|
||||
? (await getHeartbeatRunReadDecisions(db, authorization, extra.actor, failures)).filter(Boolean).length
|
||||
: failures.length;
|
||||
|
||||
const joinRequests = (extra?.joinRequests ?? []).filter((row) =>
|
||||
!isDismissed(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issues, issueRecoveryActions, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db";
|
||||
import { agents, agentWakeupRequests, companies, createDb, environmentLeases, environments, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
import { priorRecoveryApiReceipt, unresolvedRecoveryToolInvocation } from "./recovery-effect-receipts.js";
|
||||
import { requestTaskRetry } from "./task-retry.js";
|
||||
@@ -10,6 +10,7 @@ import { settleUnrecoverableExecutions } from "./execution-recovery-resolution.j
|
||||
import { createSystemRecovery, failSystemRecovery, readSystemRecovery, SystemRecoveryFailure } from "./system-recovery-state.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { heartbeatService } from "./heartbeat.js";
|
||||
import { taskChatRetryCanAwaitRecovery, taskChatRetryExecutionStopped } from "./chat-channels/task-retry-authority.js";
|
||||
|
||||
describe("durable task Retry admission", () => {
|
||||
let postgres: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
@@ -56,6 +57,53 @@ describe("durable task Retry admission", () => {
|
||||
expect(await requestTaskRetry(input)).toMatchObject({ accepted: false, stale: true });
|
||||
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, input.companyId))).toHaveLength(0);
|
||||
});
|
||||
it.each(["prepared", "mismatched", "owned"] as const)("reconstructs missing native preservation on deliberate Retry only for an exact unowned source: %s", async state => {
|
||||
const input = await seed();
|
||||
const [issue] = await db.select().from(issues).where(eq(issues.id, input.issueId));
|
||||
const [environment] = await db.insert(environments).values({ name: input.observedRunId, driver: "sandbox" }).returning();
|
||||
const [lease] = await db.insert(environmentLeases).values({ companyId: input.companyId, issueId: input.issueId,
|
||||
heartbeatRunId: input.observedRunId, environmentId: environment.id, provider: "daytona", providerLeaseId: randomUUID(),
|
||||
status: "active", leasePolicy: "reuse_by_environment", metadata: { remoteCwd: "/work" },
|
||||
}).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: input.issueId,
|
||||
nativePhase: "terminal_failure", errorCode: "execution_finalization_deadline_exceeded", runnerProfileJson: {
|
||||
nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared",
|
||||
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null,
|
||||
workspaceId: input.observedRunId, leaseId: lease.id,
|
||||
providerLeaseId: state === "mismatched" ? randomUUID() : lease.providerLeaseId,
|
||||
remoteCwd: "/work", resourceDisposition: "keep_running" },
|
||||
} }).where(eq(heartbeatRuns.id, input.observedRunId));
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: input.companyId, issueId: input.issueId,
|
||||
runId: input.observedRunId, phase: "terminal_failure", leaseOwner: state === "owned" ? "live-controller" : null });
|
||||
await db.insert(issueRecoveryActions).values({ companyId: input.companyId, sourceIssueId: input.issueId,
|
||||
kind: "active_run_watchdog", status: "resolved", ownerType: "board",
|
||||
cause: "execution_finalization_deadline_exceeded", fingerprint: randomUUID(), outcome: "blocked",
|
||||
nextAction: "Inspect the failed run.", evidence: { runId: input.observedRunId,
|
||||
providerOwnership: "unverified", automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } } });
|
||||
|
||||
const receipt = await requestTaskRetry(input);
|
||||
expect((await requestTaskRetry(input)).requestId).toBe(receipt.requestId);
|
||||
const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.observedRunId));
|
||||
const [retained] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
|
||||
const repairs = (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, input.issueId)))
|
||||
.filter(action => action.evidence.workspaceRestoreRecovery);
|
||||
if (state === "prepared") {
|
||||
expect(source.resultJson).toMatchObject({ workspaceRestoreRecovery: { leaseIds: [lease.id] } });
|
||||
expect(retained).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" });
|
||||
expect(repairs).toHaveLength(1);
|
||||
expect(readSystemRecovery(repairs[0].evidence)).toMatchObject({ state: "scheduled", intentId: receipt.requestId });
|
||||
expect(await taskChatRetryCanAwaitRecovery(db, source, input.issueId, receipt.requestId!)).toBe(true);
|
||||
} else {
|
||||
expect(source.resultJson).toBeNull();
|
||||
expect(retained).toMatchObject({ status: "active", leasePolicy: "reuse_by_environment" });
|
||||
expect(repairs).toHaveLength(0);
|
||||
}
|
||||
expect(await taskChatRetryExecutionStopped(db, source)).toBe(false);
|
||||
expect(retained.metadata).not.toHaveProperty("remoteExecutionTermination");
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, input.companyId))).toHaveLength(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, input.issueId)))[0])
|
||||
.toMatchObject({ status: "blocked", assigneeAgentId: issue.assigneeAgentId, executionRunId: null, checkoutRunId: null });
|
||||
});
|
||||
it("requires current Agent Chat ownership and enablement even when replaying a retry receipt", async () => {
|
||||
const input = await seed();
|
||||
const settings = instanceSettingsService(db);
|
||||
|
||||
@@ -7,10 +7,13 @@ import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, hea
|
||||
import { conflict, forbidden, notFound } from "../errors.js";
|
||||
import { persistActivity } from "./activity-log.js";
|
||||
import { createSystemRecovery, readSystemRecovery } from "./system-recovery-state.js";
|
||||
import { updateSystemRecovery } from "./system-recovery.js";
|
||||
import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { notifyChatActionWork } from "./chat-work-notifications.js";
|
||||
import { authorizationService } from "./authorization.js";
|
||||
import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js";
|
||||
import { NativeWorkspaceFinalizationBusyError } from "./native-runtime/native-workspace-finalization-ownership.js";
|
||||
import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js";
|
||||
|
||||
export async function taskRetryMutationAllowed(db: Db, input: {
|
||||
companyId: string; issueId: string; actorId: string; actorSource: TaskRetryIntent["actorSource"];
|
||||
@@ -116,9 +119,22 @@ export async function requestTaskRetry(input: {
|
||||
active = undefined;
|
||||
}
|
||||
const busy = ["queued", "running", "scheduled_retry"].includes(latest.status);
|
||||
if (!busy) {
|
||||
// Historical deadline failures can predate their preservation record.
|
||||
// Register the exact prepared source; this proves neither a stopped
|
||||
// executor nor permission to replay its work.
|
||||
try { await preserveNativeFailedWorkspace(db, latest); }
|
||||
catch (error) { if (!(error instanceof NativeWorkspaceFinalizationBusyError)) throw error; }
|
||||
}
|
||||
const records = await tx.select().from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.sourceIssueId, issue.id),
|
||||
sql`${issueRecoveryActions.evidence}->>'runId' = ${latest.id}`)).for("update");
|
||||
for (let index = 0; !busy && index < records.length; index++) {
|
||||
const action = records[index];
|
||||
if (!hasRequiredWorkspaceRecovery(action.evidence) || readSystemRecovery(action.evidence)) continue;
|
||||
await adoptSystemRecovery(db, { companyId: issue.companyId, issueId: issue.id, runId: latest.id, actionId: action.id });
|
||||
[records[index]] = await tx.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id));
|
||||
}
|
||||
// An unadmitted request can outlive its exhausted repair. A new deliberate
|
||||
// click replaces that generation; a replay of the same key returned above.
|
||||
if (active && !busy && !active.runId && records.some(action =>
|
||||
|
||||
@@ -72,7 +72,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
if (priorHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = priorHome;
|
||||
if (priorInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = priorInstance;
|
||||
});
|
||||
async function seed(mode = "native", nativeDescriptor = true) {
|
||||
async function seed(mode = "native", nativeDescriptor = true, resourceDisposition: "destroy" | "keep_running" = "destroy") {
|
||||
const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), actionId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Workspace recovery", defaultResponsibleUserId: "board", issuePrefix: `R${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "idle", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } });
|
||||
@@ -101,7 +101,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
targets.set(lease.id, target);
|
||||
if (nativeDescriptor) {
|
||||
const prepared = await prepareNativeWorkspaceSync({ db, companyId, workspaceId: issueId, workspaceLocalDir: host,
|
||||
runId, lease: lease as EnvironmentLease, target, resourceDisposition: "destroy" });
|
||||
runId, lease: lease as EnvironmentLease, target, resourceDisposition });
|
||||
expect(prepared).not.toBeNull();
|
||||
} else {
|
||||
// Provider boundary only: real temporary files, no native baseline. The
|
||||
@@ -137,6 +137,64 @@ describe("automatic retained workspace recovery", () => {
|
||||
const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
|
||||
const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
|
||||
|
||||
it("repairs an unregistered historical deadline failure through HTTP Retry without admitting work before verified stop", async () => {
|
||||
const f = await seed("native", true, "keep_running");
|
||||
const source = await run(f.runId);
|
||||
await db.update(heartbeatRuns).set({ resultJson: null, errorCode: "execution_finalization_deadline_exceeded" })
|
||||
.where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.update(environmentLeases).set({ status: "active", leasePolicy: "reuse_by_environment",
|
||||
releasedAt: null, cleanupStatus: null, metadata: f.lease.metadata }).where(eq(environmentLeases.id, f.lease.id));
|
||||
await db.update(issueRecoveryActions).set({ status: "resolved", outcome: "blocked",
|
||||
cause: "execution_finalization_deadline_exceeded", evidence: { runId: f.runId, providerOwnership: "unverified",
|
||||
automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } } }).where(eq(issueRecoveryActions.id, f.actionId));
|
||||
// Leave admission observable without substituting a fake model turn.
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" });
|
||||
let releaseStop!: () => void;
|
||||
const stopped = new Promise<void>(resolve => { releaseStop = resolve; });
|
||||
f.runtime.retryPendingSandboxTeardown.mockImplementation(async () => {
|
||||
await stopped;
|
||||
return { providerLeaseId: f.providerLeaseId, state: "stopped" };
|
||||
});
|
||||
const errors = vi.fn(), heartbeat = heartbeatService(db);
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: errors });
|
||||
const worker = registerWorkspaceRecoveryWork(db, coordinator, f.environmentRuntime, heartbeat.resumeRemoteStopComments);
|
||||
const successors = () => db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")));
|
||||
try {
|
||||
await worker.ready;
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => { req.actor = { type: "board", source: "local_implicit", userId: "board" }; next(); });
|
||||
app.use("/api", issueRoutes(db, {} as Parameters<typeof issueRoutes>[1]));
|
||||
app.use(errorHandler);
|
||||
const body = { idempotencyKey: randomUUID(), observedRunId: f.runId };
|
||||
const retry = await request(app).post(`/api/issues/${f.issueId}/retry`).send(body).expect(202);
|
||||
expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(body).expect(202)).body.requestId).toBe(retry.body.requestId);
|
||||
await expect.poll(() => f.runtime.retryPendingSandboxTeardown.mock.calls.length, { timeout: 20_000 }).toBe(1);
|
||||
await heartbeat.resumeTaskRetry(f.companyId, retry.body.requestId);
|
||||
expect(await successors()).toHaveLength(0);
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
|
||||
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].metadata)
|
||||
.not.toHaveProperty("remoteExecutionTermination");
|
||||
await expect(readFile(path.join(f.host, "saved.txt"), "utf8")).rejects.toMatchObject({ code: "ENOENT" });
|
||||
releaseStop();
|
||||
await expect.poll(async () => (await successors()).length, { timeout: 20_000 }).toBe(1);
|
||||
const [successor] = await successors();
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
|
||||
expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n");
|
||||
expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n");
|
||||
expect(await readFreshWorkspaceReadiness(db, successor)).toMatchObject({ mode: "resume_preserved_workspace" });
|
||||
await Promise.all([heartbeat.resumeTaskRetry(f.companyId, retry.body.requestId), heartbeat.resumeRemoteStopComments(await run(f.runId))]);
|
||||
expect(await successors()).toHaveLength(1);
|
||||
expect(f.runtime.resumeRunLease).toHaveBeenCalledOnce();
|
||||
expect((await run(f.runId)).status).toBe("failed");
|
||||
const reloaded = await request(app).get(`/api/issues/${f.issueId}/recovery-actions`).expect(200);
|
||||
expect(reloaded.body.active).toBeNull();
|
||||
const records = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(records.some(record => readSystemRecovery(record.evidence)?.state === "complete")).toBe(true);
|
||||
expect(errors).not.toHaveBeenCalled();
|
||||
} finally { releaseStop(); await coordinator.stop(); await worker.stop(); }
|
||||
}, 40_000);
|
||||
|
||||
it("wakes physical preservation from HTTP Retry before passive history without replaying the provider", async () => {
|
||||
for (let index = 0; index < 5; index++) {
|
||||
const old = await seed();
|
||||
|
||||
Reference in new issue
Block a user