mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-02 16:58:52 +01:00
Merge current master for native recovery qualification
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
5ea45f8d17
322 files changed
+21745
-1361
No files matched your search
@@ -333,6 +333,20 @@ describe("activity routes", () => {
|
||||
expect(res.body).toEqual([{ runId: "run-1", adapterType: "codex_local" }]);
|
||||
});
|
||||
|
||||
it("validates bounded run IDs before using the authorized issue projection", async () => {
|
||||
const runId = "10000000-0000-4000-8000-000000000001";
|
||||
mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-uuid-1", companyId: "company-1" });
|
||||
mockActivityService.runsForIssue.mockResolvedValue([{ runId }]);
|
||||
const app = await createApp();
|
||||
const response = await requestApp(app, base => request(base).get(`/api/issues/PAP-1/runs?runIds=${runId}`));
|
||||
expect(response.status).toBe(200);
|
||||
expect(mockActivityService.runsForIssue).toHaveBeenCalledWith("company-1", "issue-uuid-1", "user-1", undefined, [runId]);
|
||||
for (const ids of ["", "not-a-uuid", Array.from({ length: 101 }, () => runId).join(",")]) {
|
||||
const denied = await requestApp(app, base => request(base).get("/api/issues/PAP-1/runs").query({ runIds: ids }));
|
||||
expect(denied.status).toBe(400);
|
||||
}
|
||||
});
|
||||
|
||||
it("requires company access before creating activity events", async () => {
|
||||
const app = await createApp();
|
||||
const res = await requestApp(app, (baseUrl) => request(baseUrl)
|
||||
|
||||
@@ -300,7 +300,7 @@ describe("adapter management route authorization", () => {
|
||||
mocks.reloadExternalAdapter.mockImplementation(async (type: string) =>
|
||||
createAdapter(type),
|
||||
);
|
||||
}, 20_000);
|
||||
}, 60_000);
|
||||
|
||||
afterEach(() => {
|
||||
unregisterServerAdapter(EXTERNAL_ADAPTER_TYPE);
|
||||
@@ -373,35 +373,31 @@ describe("adapter management route authorization", () => {
|
||||
},
|
||||
);
|
||||
|
||||
describe("cloud-managed adapter code install floor", () => {
|
||||
beforeEach(() => {
|
||||
process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "test-server-token";
|
||||
describe("cloud-managed adapter code floor", () => {
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it.each(
|
||||
["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"].flatMap((signal) =>
|
||||
["install", "reinstall", "reload", "override", "delete"].map((routeName) => [signal, routeName]),
|
||||
),
|
||||
)("blocks %s adapter %s before package or registration access", async (signal, routeName) => {
|
||||
seedInstalledExternalAdapter();
|
||||
vi.clearAllMocks();
|
||||
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-server-token");
|
||||
const app = createApp(instanceAdmin);
|
||||
|
||||
const res = await sendMutatingRequest(app, routeName);
|
||||
|
||||
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(403);
|
||||
expect(res.body.details).toMatchObject({ code: "adapter_install_platform_managed" });
|
||||
expect(mocks.execFile).not.toHaveBeenCalled();
|
||||
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
|
||||
expect(mocks.reloadExternalAdapter).not.toHaveBeenCalled();
|
||||
expect(mocks.getAdapterPluginByType).not.toHaveBeenCalled();
|
||||
expect(mocks.addAdapterPlugin).not.toHaveBeenCalled();
|
||||
expect(mocks.removeAdapterPlugin).not.toHaveBeenCalled();
|
||||
expect(mocks.externalRecords.get(EXTERNAL_ADAPTER_TYPE)).toEqual(installedRecord());
|
||||
});
|
||||
afterEach(() => {
|
||||
delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
|
||||
});
|
||||
|
||||
it.each(["install", "reinstall"] as const)(
|
||||
"floors adapter %s off for instance admins on cloud-managed instances",
|
||||
async (routeName) => {
|
||||
resetInstalledExternalAdapterState();
|
||||
if (routeName !== "install") {
|
||||
seedInstalledExternalAdapter();
|
||||
}
|
||||
const app = createApp(instanceAdmin);
|
||||
|
||||
const res = await sendMutatingRequest(app, routeName);
|
||||
|
||||
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(
|
||||
403,
|
||||
);
|
||||
expect(res.body.details).toMatchObject({
|
||||
code: "adapter_install_platform_managed",
|
||||
});
|
||||
expect(mocks.execFile).not.toHaveBeenCalled();
|
||||
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("operator-hidden adapter management floor", () => {
|
||||
|
||||
@@ -10,6 +10,7 @@ import { eq } from "drizzle-orm";
|
||||
import { agentIdentityKeys, agents, companies, createDb, closeRegisteredClients, ensurePostgresDatabase, runDatabaseBackup, runDatabaseRestore, type Db } from "@paperclipai/db";
|
||||
import { resolveWorktreeSeedPlan } from "../../../cli/src/commands/worktree-lib.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { agentIdentityService, supportsManagedAgentIdentity } from "../services/agent-identity.js";
|
||||
import { agentService } from "../services/agents.js";
|
||||
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
|
||||
@@ -21,9 +22,10 @@ import { actorMiddleware } from "../middleware/auth.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";
|
||||
|
||||
const externalIdentityDatabase = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_IDENTITY_TEST_DATABASE_URL");
|
||||
describe("agent cryptographic identity", () => {
|
||||
let db: Db;
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let database: Pick<Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, "connectionString" | "cleanup">;
|
||||
let directory: string;
|
||||
let companyId: string;
|
||||
|
||||
@@ -32,7 +34,9 @@ describe("agent cryptographic identity", () => {
|
||||
vi.stubEnv("PAPERCLIP_HOME", directory);
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(directory, "master.key"));
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-identity-");
|
||||
database = externalIdentityDatabase ? { connectionString: externalIdentityDatabase,
|
||||
cleanup: () => closeRegisteredClients(externalIdentityDatabase) }
|
||||
: await startEmbeddedPostgresTestDatabase("paperclip-identity-");
|
||||
db = createDb(database.connectionString);
|
||||
companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Identity test", issuePrefix: "KEY", defaultResponsibleUserId: "identity-test-owner" });
|
||||
@@ -51,6 +55,7 @@ describe("agent cryptographic identity", () => {
|
||||
it("migration and public reads leave existing agents unprovisioned", async () => {
|
||||
const agent = await oldAgent();
|
||||
expect(await agentIdentityService(db).getPublicIdentity(companyId, agent.id)).toBeNull();
|
||||
expect((await agentIdentityService(db).createExistingLogRedactor(companyId, agent.id)).values).toEqual([]);
|
||||
expect(await db.select().from(agentIdentityKeys).where(eq(agentIdentityKeys.agentId, agent.id))).toHaveLength(0);
|
||||
for (const type of ["http", "openclaw_gateway", "hermes_gateway"]) expect(supportsManagedAgentIdentity(type)).toBe(false);
|
||||
for (const provider of ["claude_managed", "aws_agentcore"]) expect(supportsManagedAgentIdentity("paperclip_runner", { provider })).toBe(false);
|
||||
|
||||
@@ -103,6 +103,11 @@ vi.mock("../services/instance-settings.js", () => ({
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
}));
|
||||
|
||||
vi.mock("../services/adapter-settings.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../services/adapter-settings.js")>()),
|
||||
adapterSettingsService: () => ({ getDisabledTypes: async () => [] }),
|
||||
}));
|
||||
|
||||
vi.mock("../services/dot-runner-broker.js", () => ({
|
||||
dotRunnerBroker: () => ({ enabled: async () => true, bindingForAgent: async () => null }),
|
||||
}));
|
||||
|
||||
@@ -9,7 +9,8 @@ vi.mock("../services/subscription-refresh.js", () => ({ refreshSubscriptionConne
|
||||
import * as localCredentials from "../services/local-ai-credentials.js";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { createRequire } from "node:module";
|
||||
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:fs/promises";
|
||||
@@ -38,6 +39,8 @@ import { secretService } from "../services/secrets.js";
|
||||
import { resolveExecutionRunAdapterConfig } from "../services/heartbeat.js";
|
||||
import { aiConnectionBindingSchema, connectionPurposeTransportSchema, isAiConnectionCompatible } from "@paperclipai/shared";
|
||||
import { openRouterClaudeHeaders } from "@paperclipai/shared/openrouter-attribution";
|
||||
import { createServer } from "node:http";
|
||||
import { once } from "node:events";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { aiConnectionRoutes, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest } from "../routes/ai-connections.js";
|
||||
@@ -385,8 +388,8 @@ describe("managed AI connections", () => {
|
||||
await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_unavailable");
|
||||
});
|
||||
|
||||
it("persists first-time recovery directories before consuming a single-use token", async () => {
|
||||
const owner = "quota-durable-path";
|
||||
it("commits one exchanged token without writable credential recovery storage", async () => {
|
||||
const owner = "quota-database-first";
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const original = JSON.stringify({ tokens: { account_id: owner, access_token: "expired", refresh_token: "single-use" } });
|
||||
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
|
||||
@@ -394,32 +397,32 @@ describe("managed AI connections", () => {
|
||||
const file = quotaCredentialRecoveryPath(companyId, row.grant.id);
|
||||
const recoveryRoot = path.dirname(path.dirname(file));
|
||||
await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const synced: string[] = [];
|
||||
let rejectRoot = true;
|
||||
const realOpen = fs.open.bind(fs);
|
||||
const writes: string[] = [];
|
||||
const rejectRecoveryWrite = (target: unknown) => {
|
||||
if (String(target).startsWith(recoveryRoot)) {
|
||||
writes.push(String(target));
|
||||
throw Object.assign(new Error("Fixture recovery storage is read-only"), { code: "EROFS" });
|
||||
}
|
||||
};
|
||||
const realOpen = fs.open.bind(fs), realMkdir = fs.mkdir.bind(fs);
|
||||
const open = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
|
||||
const handle = await realOpen(...args);
|
||||
const sync = handle.sync.bind(handle);
|
||||
vi.spyOn(handle, "sync").mockImplementation(async () => {
|
||||
if (rejectRoot && args[0] === recoveryRoot) throw new Error("Injected recovery directory flush failure");
|
||||
await sync(); synced.push(String(args[0]));
|
||||
});
|
||||
return handle;
|
||||
rejectRecoveryWrite(args[0]);
|
||||
return realOpen(...args);
|
||||
});
|
||||
const request = vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
|
||||
expect(synced).toContain(recoveryRoot);
|
||||
expect(synced).toContain(path.dirname(recoveryRoot));
|
||||
return Response.json({ access_token: "fresh-durable", refresh_token: "replacement-durable" });
|
||||
const mkdir = vi.spyOn(fs, "mkdir").mockImplementation(async (...args: Parameters<typeof fs.mkdir>) => {
|
||||
rejectRecoveryWrite(args[0]);
|
||||
return realMkdir(...args);
|
||||
});
|
||||
const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "fresh-direct", refresh_token: "replacement-direct" }));
|
||||
try {
|
||||
await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow("Injected recovery directory flush failure");
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
expect(await service.credential(row)).toBe(original);
|
||||
rejectRoot = false;
|
||||
await service.refreshQuotaCredential(row, original, new AbortController().signal);
|
||||
const refreshed = await service.refreshQuotaCredential(row, original, new AbortController().signal);
|
||||
expect(JSON.parse(refreshed)).toMatchObject({ tokens: { access_token: "fresh-direct", refresh_token: "replacement-direct" } });
|
||||
expect(await service.credential(row)).toBe(refreshed);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
expect(JSON.parse(await service.credential(row))).toMatchObject({ tokens: { access_token: "fresh-durable", refresh_token: "replacement-durable" } });
|
||||
} finally { open.mockRestore(); request.mockRestore(); }
|
||||
expect(writes).toEqual([]);
|
||||
await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally { open.mockRestore(); mkdir.mockRestore(); request.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["timeout", "unexpected"] as const)("classifies a manual probe's %s lock failure before starting the provider", async (kind) => {
|
||||
@@ -703,7 +706,7 @@ describe("managed AI connections", () => {
|
||||
} finally { request.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("recovers exchanged OAuth tokens after %s persistence failure without another exchange", async (failure) => {
|
||||
it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("fails truthfully after %s persistence failure and permits sign-in recovery without a journal", async (failure) => {
|
||||
const owner = `quota-rollback-${failure}`;
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const original = JSON.stringify({ tokens: { access_token: `expired-${failure}`, refresh_token: "single-use", account_id: owner } });
|
||||
@@ -723,17 +726,25 @@ describe("managed AI connections", () => {
|
||||
await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
expect(await service.credential(row)).toBe(original);
|
||||
const encrypted = await readFile(file, "utf8");
|
||||
expect(encrypted).not.toContain(`fresh-${failure}`);
|
||||
expect(encrypted).not.toContain(`replacement-${failure}`);
|
||||
await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await db.execute(sql.raw(`drop trigger quota_save_failure on ${table}; drop function quota_save_failure();`));
|
||||
triggerDropped = true;
|
||||
// A new service instance recovers from disk, without an in-memory cache.
|
||||
const recovered = aiConnectionService(db);
|
||||
await recovered.refreshQuotaCredential(row, original, new AbortController().signal);
|
||||
expect(JSON.parse(await recovered.credential(row))).toMatchObject({ tokens: { access_token: `fresh-${failure}`, refresh_token: `replacement-${failure}` } });
|
||||
// A new receiver has no replacement to recover. A failed use routes to the
|
||||
// existing sign-in flow, rather than claiming the failed save succeeded.
|
||||
const receiver = aiConnectionService(db);
|
||||
const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
|
||||
await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
|
||||
provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
|
||||
mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
|
||||
} });
|
||||
expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
|
||||
const signedIn = JSON.stringify({ tokens: { access_token: "signed-in", refresh_token: "new-login" } });
|
||||
await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription",
|
||||
ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
|
||||
const [current] = await receiver.quotaAccounts(companyId, owner);
|
||||
expect(current.grant.status).toBe("active");
|
||||
expect(await receiver.runtimeCredential(current)).toBe(signedIn);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
await expect(access(file)).rejects.toThrow();
|
||||
expect((await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, ref.secretId))).map(version => version.version).sort()).toEqual([1, 2]);
|
||||
} finally {
|
||||
request.mockRestore();
|
||||
@@ -819,6 +830,112 @@ describe("managed AI connections", () => {
|
||||
} finally { request.mockRestore(); await oldRun.cleanup(); await nextRun?.cleanup(); await journal.clear(); }
|
||||
});
|
||||
|
||||
it("requires sign-in after the receiver process dies with an exchanged but unsaved token", async () => {
|
||||
const owner = "quota-receiver-process-loss";
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const original = JSON.stringify({ tokens: { access_token: "expired-process", refresh_token: "single-use-process" } });
|
||||
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
|
||||
const [row] = await service.quotaAccounts(companyId, owner);
|
||||
const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
|
||||
const held = deferredSignal(), release = deferredSignal();
|
||||
const lockKey = 7192031;
|
||||
const holder = db.transaction(async tx => {
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(${lockKey})`);
|
||||
held.resolve(); await release.promise;
|
||||
});
|
||||
await held.promise;
|
||||
await db.execute(sql.raw(`create function hold_receiver_save() returns trigger language plpgsql as $$ begin
|
||||
if new.secret_id = '${ref.secretId}'::uuid then perform pg_advisory_xact_lock(${lockKey}); end if; return new; end $$;
|
||||
create trigger hold_receiver_save before insert on company_secret_versions for each row execute function hold_receiver_save();`));
|
||||
let exchanges = 0;
|
||||
const provider = createServer((req, res) => {
|
||||
exchanges++; req.resume();
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.end(JSON.stringify({ access_token: "unsaved-process", refresh_token: "replacement-process" }));
|
||||
});
|
||||
provider.listen(0, "127.0.0.1"); await once(provider, "listening");
|
||||
const address = provider.address() as { port: number };
|
||||
const child = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), new URL("./helpers/quota-credential-process.ts", import.meta.url).pathname], {
|
||||
env: { ...process.env, CREDENTIAL_TEST_DATABASE_URL: database.connectionString,
|
||||
CREDENTIAL_TEST_PROVIDER_URL: `http://127.0.0.1:${address.port}`, CREDENTIAL_TEST_COMPANY_ID: companyId, CREDENTIAL_TEST_OWNER: owner },
|
||||
stdio: "ignore",
|
||||
});
|
||||
const exited = once(child, "exit");
|
||||
try {
|
||||
let blocked = false;
|
||||
const deadline = Date.now() + 25_000;
|
||||
while (!blocked && Date.now() < deadline && child.exitCode === null) {
|
||||
const rows = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
|
||||
and wait_event_type = 'Lock' and wait_event = 'advisory'`);
|
||||
blocked = exchanges === 1 && rows.length > 0;
|
||||
if (!blocked) await new Promise(resolve => setTimeout(resolve, 25));
|
||||
}
|
||||
expect(blocked).toBe(true);
|
||||
child.kill("SIGKILL"); await exited;
|
||||
} finally {
|
||||
if (child.exitCode === null && child.signalCode === null) { child.kill("SIGKILL"); await exited; }
|
||||
release.resolve(); await holder;
|
||||
await db.execute(sql`drop trigger hold_receiver_save on company_secret_versions; drop function hold_receiver_save()`);
|
||||
await new Promise<void>(resolve => provider.close(() => resolve()));
|
||||
}
|
||||
expect(exchanges).toBe(1);
|
||||
expect(await service.credential(row)).toBe(original);
|
||||
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const receiver = aiConnectionService(db);
|
||||
const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
|
||||
await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
|
||||
provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
|
||||
mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
|
||||
} });
|
||||
expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
|
||||
const signedIn = JSON.stringify({ tokens: { access_token: "process-sign-in", refresh_token: "process-new-login" } });
|
||||
await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
|
||||
const [current] = await receiver.quotaAccounts(companyId, owner);
|
||||
expect(await receiver.runtimeCredential(current)).toBe(signedIn);
|
||||
}, 40_000);
|
||||
|
||||
it("does not exchange a token when a required database transaction fails", async () => {
|
||||
const owner = "quota-db-prerequisite";
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
|
||||
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
|
||||
const [row] = await service.quotaAccounts(companyId, owner);
|
||||
const request = vi.spyOn(globalThis, "fetch");
|
||||
const unavailableUrl = new URL(database.connectionString);
|
||||
unavailableUrl.port = "1";
|
||||
const unavailable = createDb(unavailableUrl.toString());
|
||||
try {
|
||||
await expect(aiConnectionService(unavailable).refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally { await unavailable.$client.end(); request.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["reconnect", "same_value_new_epoch", "revoked"] as const)("lets %s win after a returned token's save rolls back", async change => {
|
||||
const owner = `quota-delayed-save-${change}`;
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
|
||||
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
|
||||
const [row] = await service.quotaAccounts(companyId, owner);
|
||||
const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
|
||||
const transaction = db.transaction.bind(db);
|
||||
const interrupted = vi.spyOn(db, "transaction").mockImplementationOnce(async (work, config) => {
|
||||
await transaction(async tx => { await work(tx); throw new Error("fixture rollback"); }, config).catch(() => {});
|
||||
if (change === "revoked") await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, row.grant.id));
|
||||
else await secretService(db).rotate(ref.secretId, { value: change === "reconnect" ? "new-login" : original });
|
||||
throw new Error("fixture lost response");
|
||||
});
|
||||
const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "late", refresh_token: "late-refresh" }));
|
||||
try {
|
||||
const refresh = service.refreshQuotaCredential(row, original, new AbortController().signal);
|
||||
if (change === "revoked") await expect(refresh).rejects.toThrow("credentials_unavailable");
|
||||
else expect(await refresh).toBe(change === "reconnect" ? "new-login" : original);
|
||||
expect(await service.credential(row)).toBe(change === "reconnect" ? "new-login" : original);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally { interrupted.mockRestore(); request.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["rollback", "lost_commit_reply"] as const)("automatically saves one exchanged token across %s", async (failure) => {
|
||||
const owner = `quota-once-${failure}`;
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
@@ -863,7 +980,10 @@ describe("managed AI connections", () => {
|
||||
triggerInstalled = true;
|
||||
await expect(service.refreshQuotaCredential(row, current, new AbortController().signal)).rejects.toThrow();
|
||||
expect(await service.credential(row)).toBe(current);
|
||||
expect(JSON.parse((await journal.read())!.value).tokens.access_token).toBe("C");
|
||||
expect(await journal.read()).toBeNull();
|
||||
// Only observed records from the earlier release use the legacy reader.
|
||||
await journal.write({ companyId, grantId: row.grant.id, connectionId: row.connection.id, secretId: ref.secretId,
|
||||
baseHash: quotaCredentialHash(current), value: JSON.stringify({ tokens: { access_token: "C", refresh_token: "refresh-C" } }) });
|
||||
await db.execute(sql`drop trigger delayed_quota_commit_failure on company_secret_versions; drop function delayed_quota_commit_failure()`);
|
||||
triggerInstalled = false;
|
||||
const recovered = await service.refreshQuotaCredential(row, original, new AbortController().signal);
|
||||
|
||||
@@ -1406,6 +1406,22 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
|
||||
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
|
||||
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
|
||||
const summaryEvents: Array<Record<string, unknown>> = [], captureNow = Date.now();
|
||||
const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
|
||||
installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
|
||||
try {
|
||||
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
|
||||
expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
|
||||
expect(summary.totalCount).toBe(feed.totalCount);
|
||||
const index = await attentionService(db).taskIndex(companyId, { userId: "board-user" });
|
||||
expect(index.taskIds).toEqual([...new Set(feed.items.flatMap(item => item.relatedIssue ? [item.relatedIssue.id] : item.subject.kind === "issue" ? [item.subject.id] : []))]);
|
||||
expect(index.totalCount).toBe(feed.totalCount);
|
||||
|
||||
} finally { summaryCapture.flush(); }
|
||||
const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
&& ["documents", "assets", "issue_attachments"].includes(String(event.table)));
|
||||
expect(cardReads).toEqual([]);
|
||||
|
||||
|
||||
const planItem = detailsByKind.get("plan_approval");
|
||||
expect(planItem?.subject.title).toBe("Plan approval - Approve launch plan");
|
||||
@@ -1527,6 +1543,9 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
|
||||
const interactions = feed.items.filter(item => item.sourceKind === "issue_thread_interaction");
|
||||
expect(interactions.map(item => item.subject.id).sort()).toEqual([ids[0], ids[1], ids[3]].sort());
|
||||
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
|
||||
expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
|
||||
|
||||
// Resolving one output removes only that decision, leaving the other output
|
||||
// and the ordinary confirmation independently actionable.
|
||||
await db.update(issueThreadInteractions).set({ status: "accepted", resolvedAt: new Date() }).where(eq(issueThreadInteractions.id, ids[0]));
|
||||
@@ -1858,6 +1877,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
|
||||
await expect(attentionService(db).list(companyId, { userId: "board-user" }))
|
||||
.resolves.toMatchObject({ totalCount: 1 }); // agent_error_alert from seed
|
||||
await expect(attentionService(db).summary(companyId, { userId: "board-user" })).resolves.toMatchObject({ totalCount: 1 });
|
||||
const hiddenFeed = await attentionService(db).list(companyId, { userId: "board-user", includeDismissed: true });
|
||||
expect(hiddenFeed.items.find((item) => item.dedupKey === `approval:${approvalId}`)?.dismissal)
|
||||
.toMatchObject({ kind: "snooze", isActive: true, snoozedUntil: "2099-01-02T00:00:00.000Z" });
|
||||
@@ -1871,6 +1891,9 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
const visibleApproval = visibleFeed.items.find((item) => item.dedupKey === `approval:${approvalId}`);
|
||||
expect(visibleApproval?.dismissal).toMatchObject({ kind: "snooze", isActive: false });
|
||||
expect(visibleApproval).toBeTruthy();
|
||||
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
|
||||
expect(summary.totalCount).toBe(visibleFeed.totalCount);
|
||||
|
||||
});
|
||||
|
||||
it("enriches, filters, paginates, snoozes, and ranks the decide-now feed", async () => {
|
||||
@@ -2113,6 +2136,13 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
expect(approvalRows.every((approval) => internalSnapshotIds.has(approval.id))).toBe(true);
|
||||
expect(internalSnapshot.items.length).toBeGreaterThan(100);
|
||||
expect(internalSnapshot.nextCursor).toBeNull();
|
||||
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
|
||||
expect(summary).toEqual({
|
||||
companyId, generatedAt: expect.any(String), totalCount: internalSnapshot.totalCount,
|
||||
deskBadgeCount: internalSnapshot.deskBadgeCount, countsBySourceKind: internalSnapshot.countsBySourceKind,
|
||||
});
|
||||
expect(JSON.stringify(summary).length).toBeLessThan(JSON.stringify(internalSnapshot).length / 10);
|
||||
|
||||
|
||||
await expect(attentionService(db).list(companyId, { userId: "board-user", all: true }))
|
||||
.rejects.toThrow("all requires a queue filter");
|
||||
@@ -2166,6 +2196,10 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
allowUnscopedAll: true,
|
||||
});
|
||||
expect(complete.items.filter((item) => item.sourceKind === "decision")).toHaveLength(2);
|
||||
const summary = await svc.summary(companyId, { userId: "board-user" });
|
||||
expect(summary.totalCount).toBe(complete.totalCount);
|
||||
expect(summary.countsBySourceKind).toEqual(complete.countsBySourceKind);
|
||||
|
||||
expect(complete.nextCursor).toBeNull();
|
||||
});
|
||||
|
||||
@@ -2281,6 +2315,18 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
.get(`/api/companies/${companyId}/attention?includeDismissed=true&all=true`)
|
||||
.expect(200);
|
||||
expect(completeFeed.body.nextCursor).toBeNull();
|
||||
const summary = await request(app(board)).get(`/api/companies/${companyId}/attention/summary`).expect(200);
|
||||
expect(summary.body).not.toHaveProperty("items");
|
||||
expect(summary.body.totalCount).toBe(completeFeed.body.totalCount);
|
||||
const taskIndex = await request(app(board)).get(`/api/companies/${companyId}/attention/tasks`).expect(200);
|
||||
expect(taskIndex.body).not.toHaveProperty("items");
|
||||
expect(taskIndex.body.taskIds).toEqual([]);
|
||||
await request(app(agent)).get(`/api/companies/${companyId}/attention/tasks`).expect(403);
|
||||
await request(app({ ...board, source: "session", companyIds: [] })).get(`/api/companies/${companyId}/attention/tasks`).expect(403);
|
||||
|
||||
await request(app(agent)).get(`/api/companies/${companyId}/attention/summary`).expect(403);
|
||||
await request(app({ ...board, source: "session", companyIds: [] })).get(`/api/companies/${companyId}/attention/summary`).expect(403);
|
||||
|
||||
await request(app(board))
|
||||
.get(`/api/companies/${companyId}/attention?activitySince=yesterday`)
|
||||
.expect(400, { error: "activitySince must be an ISO timestamp" });
|
||||
@@ -2317,6 +2363,10 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
|
||||
expect(feed.items.map((item) => item.subject.title)).not.toContain("Confidential review");
|
||||
expect(JSON.stringify(feed)).not.toContain(privateIssueId);
|
||||
const summary = await attentionService(db).summary(companyId, { userId: actor.userId, actor });
|
||||
expect(summary.totalCount).toBe(feed.totalCount);
|
||||
expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
|
||||
|
||||
});
|
||||
|
||||
it("computes the aging shelf uniformly across approval, interaction, and review sources", async () => {
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { rmSync, writeFileSync, symlinkSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
|
||||
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
|
||||
import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
|
||||
import { readDistributionPluginCatalog } from "../services/distribution-plugin-catalog.js";
|
||||
|
||||
import boxd from "../../../packages/plugins/sandbox-providers/boxd/src/manifest.js";
|
||||
import createos from "../../../packages/plugins/sandbox-providers/createos/src/manifest.js";
|
||||
import cloudflare from "../../../packages/plugins/sandbox-providers/cloudflare/src/manifest.js";
|
||||
import daytona from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
|
||||
import e2b from "../../../packages/plugins/sandbox-providers/e2b/src/manifest.js";
|
||||
import exe_dev from "../../../packages/plugins/sandbox-providers/exe-dev/src/manifest.js";
|
||||
import kubernetes from "../../../packages/plugins/sandbox-providers/kubernetes/src/manifest.js";
|
||||
import modal from "../../../packages/plugins/sandbox-providers/modal/src/manifest.js";
|
||||
import novita from "../../../packages/plugins/sandbox-providers/novita/src/manifest.js";
|
||||
|
||||
const manifests = { boxd, createos, cloudflare, daytona, e2b, "exe-dev": exe_dev, kubernetes, modal, novita };
|
||||
const roots: string[] = [];
|
||||
function fixture() { const result = bundledImageFixture(); roots.push(result.root); return result; }
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
describe("bundled image package identity", () => {
|
||||
it("resolves independent image roots from the same registration without any previous path", () => {
|
||||
const first = fixture();
|
||||
const second = fixture();
|
||||
const registration = { ...first.registration, packagePath: "/lost/old/home/plugins/fixture" };
|
||||
expect(first.resolver.resolve(registration)).toBe(first.packageRoot);
|
||||
expect(second.resolver.resolve(registration)).toBe(second.packageRoot);
|
||||
expect(first.resolver.inspect(registration, first.manifest)).toEqual(second.resolver.inspect(registration, second.manifest));
|
||||
expect(registration.packagePath).toBe("/lost/old/home/plugins/fixture");
|
||||
});
|
||||
|
||||
it("requires verified manifest evidence and blocks same-version manifest drift", () => {
|
||||
const { resolver, registration, manifest } = fixture();
|
||||
expect(resolver.inspect(registration)).toMatchObject({ compatible: false, code: "missing_manifest_evidence" });
|
||||
expect(resolver.inspect(registration, { ...manifest, description: "changed" })).toMatchObject({ compatible: false, code: "incompatible_bundled_package", reason: expect.stringContaining("Approved plugin manifest") });
|
||||
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true, identity: { kind: "bundled_image", catalogKey: "fixture", pluginKey: manifest.id, version: "1.0.0" } });
|
||||
});
|
||||
|
||||
it("rejects mismatched package names, versions, identity and escaping entrypoints", () => {
|
||||
const { resolver, registration } = fixture();
|
||||
for (const update of [
|
||||
{ packageName: "@other/plugin" }, { version: "2.0.0" }, { pluginKey: "other.plugin" },
|
||||
{ manifestJson: { ...registration.manifestJson, version: "2.0.0" } },
|
||||
{ manifestJson: { ...registration.manifestJson, entrypoints: { worker: "../worker.js" } } },
|
||||
]) expect(() => resolver.resolve({ ...registration, ...update })).toThrow();
|
||||
});
|
||||
|
||||
it("fails explicitly for missing or deselected bundles even when a stale npm package exists", () => {
|
||||
const { root, packageRoot, resolver, registration } = fixture();
|
||||
const fallback = path.join(root, "home", "node_modules", registration.packageName);
|
||||
mkdirSync(fallback, { recursive: true });
|
||||
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('unsafe fallback');");
|
||||
rmSync(packageRoot, { recursive: true });
|
||||
expect(() => resolver.resolve(registration)).toThrow(/ENOENT/);
|
||||
expect(() => createBundledPluginPackageResolver([]).resolve(registration)).toThrow(/absent or not selected/);
|
||||
});
|
||||
|
||||
it("rejects tampering before manifest import and symlink escapes before worker resolution", () => {
|
||||
const { root, packageRoot, registration, resolver } = fixture();
|
||||
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), "throw new Error('must never import');");
|
||||
expect(() => readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG)).toThrow(/digest mismatch/);
|
||||
rmSync(path.join(packageRoot, "dist", "worker.js"));
|
||||
symlinkSync(path.join(root, "distribution", "catalog.json"), path.join(packageRoot, "dist", "worker.js"));
|
||||
expect(() => resolver.resolve(registration)).toThrow(/escapes/);
|
||||
});
|
||||
|
||||
it.each(BUNDLED_PLUGIN_CATALOG)("accepts the real shipped $key package and manifest contract", (entry) => {
|
||||
const manifest = manifests[entry.key as keyof typeof manifests];
|
||||
const pkg = JSON.parse(readFileSync(new URL(`../../../packages/plugins/${entry.relativePath}/package.json`, import.meta.url), "utf8"));
|
||||
const { root } = fixture();
|
||||
const packageRoot = path.join(root, entry.relativePath);
|
||||
mkdirSync(path.join(packageRoot, "dist"), { recursive: true });
|
||||
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(pkg));
|
||||
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
|
||||
writeFileSync(path.join(packageRoot, "dist", "worker.js"), "export default {};");
|
||||
const installs = resolveBundledPluginInstalls([entry.key], { catalogRoot: root, enforceCatalogRoot: true, distributionPlugins: [], env: {} });
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const registration = { pluginKey: entry.pluginKey, packageName: pkg.name, version: manifest.version, manifestJson: manifest };
|
||||
expect(pkg.version).toBe(manifest.version);
|
||||
expect(resolver.resolve(registration)).toBe(packageRoot);
|
||||
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync, existsSync, mkdtempSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { applyPendingMigrations, closeRegisteredClients, companies, createDb, plugins } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import type { PluginRecord } from "@paperclipai/shared";
|
||||
import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
|
||||
import { linkSdkInto } from "../../../scripts/link-plugin-dev-sdk.mjs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
|
||||
import { inspectCloudPackages } from "../services/cloud-package-qualification.js";
|
||||
import { resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
|
||||
import { pluginRegistryService } from "../services/plugin-registry.js";
|
||||
import { pluginUiStaticRoutes } from "../routes/plugin-ui-static.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { derivePluginDatabaseNamespace } from "../services/plugin-database.js";
|
||||
import { ensureBundledPlugins } from "../services/bundled-plugins.js";
|
||||
import daytonaManifest from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
|
||||
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_BUNDLED_PLUGIN_TEST_DATABASE_URL?.trim();
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describePostgres = support.supported ? describe : describe.skip;
|
||||
|
||||
describePostgres("bundled image registration in PostgreSQL", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let peer: ReturnType<typeof createDb>;
|
||||
const roots: string[] = [];
|
||||
const fixture = () => { const result = bundledImageFixture(); roots.push(result.root); return result; };
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) {
|
||||
await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
|
||||
} else {
|
||||
database = await startEmbeddedPostgresTestDatabase("bundled-image-registration-");
|
||||
}
|
||||
db = createDb(database.connectionString);
|
||||
peer = createDb(database.connectionString);
|
||||
}, 90_000);
|
||||
afterEach(async () => {
|
||||
await db.delete(plugins);
|
||||
await db.delete(companies);
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
function runtime() {
|
||||
const output: string[] = [];
|
||||
const services = {
|
||||
workerManager: {
|
||||
startWorker: vi.fn(async (_id: string, options: { entrypointPath: string }) => { output.push(execFileSync(process.execPath, [options.entrypointPath, "--source"], { encoding: "utf8" })); }),
|
||||
call: vi.fn(async () => ({})),
|
||||
},
|
||||
lifecycleManager: { markError: vi.fn() },
|
||||
jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
|
||||
toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
|
||||
buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" },
|
||||
};
|
||||
return { services, output };
|
||||
}
|
||||
function loaderFor(image: ReturnType<typeof fixture>, databaseClient = db, services?: PluginRuntimeServices) {
|
||||
return pluginLoader(databaseClient, {
|
||||
localPluginDir: path.join(image.root, "lost-home"), prebuiltOnly: true,
|
||||
resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, services);
|
||||
}
|
||||
|
||||
it("starts current-image workers after losing the old home and retains each company's config", async () => {
|
||||
const old = fixture();
|
||||
const first = fixture();
|
||||
const second = fixture();
|
||||
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
let row = (await registry.getByKey(old.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
const [companyA, companyB, unconfigured] = await db.insert(companies).values([
|
||||
{ name: "Company A", issuePrefix: "CFA" }, { name: "Company B", issuePrefix: "CFB" }, { name: "Unconfigured", issuePrefix: "CFC" },
|
||||
]).returning();
|
||||
await registry.upsertConfig(row.id, companyA!.id, { configJson: { token: "only-A" } });
|
||||
await registry.upsertConfig(row.id, companyB!.id, { configJson: { token: "only-B" } });
|
||||
row = (await registry.getById(row.id))!;
|
||||
rmSync(old.root, { recursive: true });
|
||||
const original = { ...row };
|
||||
for (const image of [first, second]) {
|
||||
const current = runtime();
|
||||
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
|
||||
const result = await loader.loadSingle(row.id);
|
||||
expect(result.success).toBe(true);
|
||||
expect(current.output).toEqual([expect.stringContaining(image.packageRoot)]);
|
||||
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({
|
||||
entrypointPath: path.join(image.packageRoot, "dist", "worker.js"), config: {}, proactiveCompanyScopes: [companyA!.id, companyB!.id].sort(),
|
||||
}));
|
||||
expect(current.services.workerManager.call.mock.calls).toEqual(expect.arrayContaining([
|
||||
[row.id, "configChanged", { companyId: companyA!.id, config: { token: "only-A" } }],
|
||||
[row.id, "configChanged", { companyId: companyB!.id, config: { token: "only-B" } }],
|
||||
]));
|
||||
expect(current.services.workerManager.call).toHaveBeenCalledTimes(2);
|
||||
expect(await registry.getConfig(row.id, unconfigured!.id)).toBeNull();
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", userId: "fixture-user", source: "session", isInstanceAdmin: false, companyIds: [companyA!.id] };
|
||||
next();
|
||||
});
|
||||
app.use(pluginUiStaticRoutes(db, { localPluginDir: path.join(image.root, "lost-home"), resolvePackageRoot: image.resolver.resolve }));
|
||||
app.use(errorHandler);
|
||||
const served = await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyA!.id });
|
||||
expect(served.status).toBe(200);
|
||||
expect(served.text).toContain("current-image");
|
||||
expect((await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyB!.id })).status).toBe(403);
|
||||
|
||||
}
|
||||
expect(await registry.getById(row.id)).toEqual(original);
|
||||
});
|
||||
|
||||
it("completes the real worker initialize and health protocol from a relocated image", async () => {
|
||||
const old = fixture(); const image = fixture();
|
||||
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(old.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
rmSync(old.root, { recursive: true });
|
||||
const manager = createPluginWorkerManager();
|
||||
try {
|
||||
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
|
||||
const result = await loaderFor(image, db, services).loadSingle(row.id);
|
||||
expect(result.success).toBe(true);
|
||||
expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(image.packageRoot), configs: {} } });
|
||||
} finally { await manager.stopAll(); }
|
||||
});
|
||||
|
||||
it("qualifies and initializes the current release SDK/shared exports through the baked loader", async () => {
|
||||
const image = fixture();
|
||||
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
|
||||
const provider = mkdtempSync(path.join(catalogRoot, "sandbox-providers", "image-sdk-proof-"));
|
||||
roots.push(provider);
|
||||
cpSync(image.packageRoot, provider, { recursive: true });
|
||||
linkSdkInto(provider);
|
||||
writeFileSync(path.join(provider, "dist", "worker.js"), 'import { definePlugin, runWorker } from "@paperclipai/plugin-sdk"; runWorker(definePlugin({ async setup() {}, async onHealth() { return { status: "ok", message: "Current image SDK/shared loaded" }; } }), import.meta.url);');
|
||||
const installs = [{ key: "sdk-proof", pluginKey: image.manifest.id, localPath: provider }];
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const manager = createPluginWorkerManager();
|
||||
const loader = pluginLoader(db, {
|
||||
prebuiltOnly: true, resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices);
|
||||
const registry = pluginRegistryService(db);
|
||||
try {
|
||||
await loader.installPlugin({ localPath: provider });
|
||||
const installed = (await registry.getByKey(image.manifest.id))!;
|
||||
await registry.updateStatus(installed.id, { status: "ready" });
|
||||
const registration = (await registry.getById(installed.id))!;
|
||||
expect((await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) })).blockers).toEqual([]);
|
||||
expect((await loader.loadSingle(installed.id)).success).toBe(true);
|
||||
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Current image SDK/shared loaded" });
|
||||
} finally { await manager.stopAll(); }
|
||||
}, 90_000);
|
||||
|
||||
it.skipIf(!existsSync(fileURLToPath(new URL("../../../packages/plugins/sandbox-providers/daytona/dist/worker.js", import.meta.url))))("qualifies and initializes the actual prebuilt Daytona release package", async () => {
|
||||
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
|
||||
const installs = resolveBundledPluginInstalls(["daytona"], { catalogRoot, env: {}, enforceCatalogRoot: true, distributionPlugins: [] });
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const manager = createPluginWorkerManager();
|
||||
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
|
||||
const loader = pluginLoader(db, {
|
||||
localPluginDir: "/lost/release/home/plugins", prebuiltOnly: true,
|
||||
resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, services);
|
||||
const registry = pluginRegistryService(db);
|
||||
vi.stubEnv("DAYTONA_API_KEY", "");
|
||||
try {
|
||||
await loader.installPlugin({ localPath: installs[0]!.localPath });
|
||||
const installed = (await registry.getByKey(daytonaManifest.id))!;
|
||||
await registry.update(installed.id, { packagePath: "/lost/release/home/daytona" });
|
||||
await registry.updateStatus(installed.id, { status: "ready" });
|
||||
const registration = (await registry.getById(installed.id))!;
|
||||
const inspected = await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) });
|
||||
expect(inspected.blockers).toEqual([]);
|
||||
expect(inspected.packages[0]?.packageDigest).toMatch(/^sha256:/);
|
||||
expect((await loader.loadSingle(installed.id)).success).toBe(true);
|
||||
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Daytona sandbox provider plugin healthy" });
|
||||
} finally { await manager.stopAll(); vi.unstubAllEnvs(); }
|
||||
}, 90_000);
|
||||
|
||||
it("registers the same package concurrently from two roots without replacing the winner", async () => {
|
||||
const first = fixture(); const second = fixture();
|
||||
await Promise.all([
|
||||
loaderFor(first).installPlugin({ localPath: first.packageRoot }),
|
||||
loaderFor(second, peer).installPlugin({ localPath: second.packageRoot }),
|
||||
]);
|
||||
const rows = await pluginRegistryService(db).listInstalled();
|
||||
expect(rows).toHaveLength(1);
|
||||
const original = rows[0]!;
|
||||
expect([first.packageRoot, second.packageRoot]).toContain(original.packagePath);
|
||||
await loaderFor(second, peer).installPlugin({ localPath: second.packageRoot });
|
||||
expect(await pluginRegistryService(db).getById(original.id)).toEqual(original);
|
||||
await expect(pluginRegistryService(peer).install({ packageName: original.packageName }, { ...first.manifest, description: "different package metadata" }, { idempotent: true })).rejects.toThrow(/already installed/);
|
||||
});
|
||||
|
||||
it("preserves disabled registrations and avoids path writes during managed boot", async () => {
|
||||
const old = fixture(); const image = fixture();
|
||||
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(old.manifest.id))!;
|
||||
await db.update(plugins).set({ status: "disabled" }).where(eq(plugins.id, row.id));
|
||||
const original = await registry.getById(row.id);
|
||||
await ensureBundledPlugins(image.installs, { registry, loader: loaderFor(image), lifecycle: { load: vi.fn() }, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }, { reinstallUninstalled: true, preservePackagePath: true });
|
||||
expect(await registry.getById(row.id)).toEqual(original);
|
||||
});
|
||||
|
||||
it("fails before worker startup for missing image bytes without using a home fallback or building", async () => {
|
||||
const image = fixture();
|
||||
await loaderFor(image).installPlugin({ localPath: image.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(image.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
const fallback = path.join(image.root, "lost-home", "node_modules", row.packageName);
|
||||
mkdirSync(fallback, { recursive: true });
|
||||
writeFileSync(path.join(fallback, "package.json"), JSON.stringify({ name: row.packageName }));
|
||||
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('must not start');");
|
||||
rmSync(path.join(image.packageRoot, "dist", "worker.js"));
|
||||
const current = runtime();
|
||||
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
|
||||
const result = await loader.loadSingle(row.id);
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain("ENOENT");
|
||||
expect(current.services.workerManager.startWorker).not.toHaveBeenCalled();
|
||||
await expect(loader.installPlugin({ packageName: row.packageName })).rejects.toThrow(/current image catalog/);
|
||||
await loader.cleanupInstallArtifacts(row as PluginRecord);
|
||||
expect(image.resolver.inspect(row, image.manifest).compatible).toBe(false);
|
||||
});
|
||||
|
||||
it("applies plugin migrations from the current image after the legacy package disappears", async () => {
|
||||
const image = bundledImageFixture({
|
||||
capabilities: ["database.namespace.migrate", "database.namespace.read", "database.namespace.write"],
|
||||
database: { migrationsDir: "migrations" },
|
||||
});
|
||||
roots.push(image.root);
|
||||
const [row] = await db.insert(plugins).values({
|
||||
...image.registration, packagePath: "/lost/legacy/home/plugin", apiVersion: 1, categories: image.manifest.categories, status: "ready",
|
||||
}).returning();
|
||||
const current = runtime();
|
||||
const result = await loaderFor(image, db, current.services as unknown as PluginRuntimeServices).loadSingle(row!.id);
|
||||
expect(result.success).toBe(true);
|
||||
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row!.id, expect.objectContaining({ databaseNamespace: derivePluginDatabaseNamespace(image.manifest.id) }));
|
||||
await db.execute((await import("drizzle-orm")).sql.raw(`DROP SCHEMA "${derivePluginDatabaseNamespace(image.manifest.id)}" CASCADE`));
|
||||
});
|
||||
|
||||
it("retains registry-install provider credentials for an ordinary npm package", async () => {
|
||||
const npmManifest = { ...daytonaManifest, id: "paperclip.image-npm-fixture" };
|
||||
const image = bundledImageFixture(npmManifest);
|
||||
roots.push(image.root);
|
||||
const localPluginDir = path.join(image.root, "npm-home");
|
||||
const packageRoot = path.join(localPluginDir, "node_modules", "@paperclipai", "plugin-daytona");
|
||||
mkdirSync(path.dirname(packageRoot), { recursive: true });
|
||||
cpSync(image.packageRoot, packageRoot, { recursive: true });
|
||||
const metadata = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8"));
|
||||
metadata.name = "@paperclipai/plugin-daytona";
|
||||
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(metadata));
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.install({ packageName: metadata.name }, npmManifest))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
vi.stubEnv("DAYTONA_API_KEY", "synthetic-provider-key");
|
||||
try {
|
||||
const current = runtime();
|
||||
const loader = pluginLoader(db, { localPluginDir, enableNpmDiscovery: false, enableLocalFilesystem: false }, current.services as unknown as PluginRuntimeServices);
|
||||
expect((await loader.loadSingle(row.id)).success).toBe(true);
|
||||
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({ env: expect.objectContaining({ DAYTONA_API_KEY: "synthetic-provider-key" }) }));
|
||||
} finally { vi.unstubAllEnvs(); }
|
||||
});
|
||||
|
||||
it("keeps self-hosted local-path loading available without a managed resolver", async () => {
|
||||
const image = fixture();
|
||||
const loader = pluginLoader(db, { localPluginDir: path.join(image.root, "selfhost"), enableNpmDiscovery: false, enableLocalFilesystem: false });
|
||||
await loader.installPlugin({ localPath: image.packageRoot });
|
||||
const row = (await pluginRegistryService(db).getByKey(image.manifest.id))!;
|
||||
expect(row.packagePath).toBe(image.packageRoot);
|
||||
await expect(loader.installPlugin({ localPath: image.packageRoot })).rejects.toThrow(/already installed/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { build, transform } from "esbuild";
|
||||
import { bundleRuntime } from "../../scripts/bundle-runtime.mjs";
|
||||
import { resolveBundledPluginRuntimeLoader } from "../services/bundled-plugin-runtime.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
function fixture() {
|
||||
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-plugin-runtime-")));
|
||||
roots.push(root);
|
||||
fs.mkdirSync(path.join(root, "server", "dist-runtime"), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, "server", "package.json"), JSON.stringify({ name: "@paperclipai/server", type: "module" }));
|
||||
const loader = path.join(root, "cli", "node_modules", "tsx", "dist", "loader.mjs");
|
||||
fs.mkdirSync(path.dirname(loader), { recursive: true });
|
||||
fs.writeFileSync(loader, "export {};\n");
|
||||
return { root, loader };
|
||||
}
|
||||
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
describe("current image plugin runtime loader", () => {
|
||||
it.each(["src/services", "dist/services", "dist-runtime"])("resolves the owning release from server/%s", (location) => {
|
||||
const { root, loader } = fixture();
|
||||
const moduleDirectory = path.join(root, "server", location);
|
||||
fs.mkdirSync(moduleDirectory, { recursive: true });
|
||||
expect(resolveBundledPluginRuntimeLoader(moduleDirectory)).toBe(loader);
|
||||
});
|
||||
|
||||
it("resolves a bundled entrypoint without preserved source module locations", async () => {
|
||||
const { root, loader } = fixture();
|
||||
const runtime = path.join(root, "server", "dist-runtime", "index.mjs");
|
||||
await build({
|
||||
stdin: { contents: `import { resolveBundledPluginRuntimeLoader } from ${JSON.stringify(fileURLToPath(new URL("../services/bundled-plugin-runtime.ts", import.meta.url)))}; process.stdout.write(resolveBundledPluginRuntimeLoader());`, resolveDir: process.cwd() },
|
||||
outfile: runtime, bundle: true, platform: "node", format: "esm", target: "node24", logLevel: "silent",
|
||||
});
|
||||
expect(execFileSync(process.execPath, [runtime], { encoding: "utf8" })).toBe(loader);
|
||||
});
|
||||
|
||||
it("uses the production bundler's preserved module location at the dist-runtime entrypoint", async () => {
|
||||
const { root, loader } = fixture();
|
||||
const service = path.join(root, "server", "dist", "services", "bundled-plugin-runtime.js");
|
||||
fs.mkdirSync(path.dirname(service), { recursive: true });
|
||||
const source = fs.readFileSync(new URL("../services/bundled-plugin-runtime.ts", import.meta.url), "utf8");
|
||||
fs.writeFileSync(service, (await transform(source, { loader: "ts", format: "esm", target: "node24" })).code);
|
||||
const entry = path.join(root, "server", "dist", "probe.js");
|
||||
fs.writeFileSync(entry, 'import { resolveBundledPluginRuntimeLoader } from "./services/bundled-plugin-runtime.js"; process.stdout.write(resolveBundledPluginRuntimeLoader());');
|
||||
const bundled = await bundleRuntime({ entry, outdir: path.join(root, "server", "dist-runtime") });
|
||||
expect(execFileSync(process.execPath, [bundled.outfile], { encoding: "utf8" })).toBe(loader);
|
||||
});
|
||||
|
||||
it("accepts the baked pnpm target inside the current image node_modules", () => {
|
||||
const { root, loader } = fixture();
|
||||
const physical = path.join(root, "node_modules", ".pnpm", "tsx@1", "node_modules", "tsx", "dist", "loader.mjs");
|
||||
fs.mkdirSync(path.dirname(physical), { recursive: true });
|
||||
fs.writeFileSync(physical, "export {};\n");
|
||||
fs.rmSync(loader); fs.symlinkSync(path.relative(path.dirname(loader), physical), loader);
|
||||
expect(resolveBundledPluginRuntimeLoader(path.join(root, "server", "dist-runtime"))).toBe(physical);
|
||||
});
|
||||
|
||||
it("fails explicitly for missing or legacy-home loaders", () => {
|
||||
const { root, loader } = fixture();
|
||||
const moduleDirectory = path.join(root, "server", "dist-runtime");
|
||||
fs.rmSync(loader);
|
||||
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("missing from the current image");
|
||||
const outside = fixture().loader;
|
||||
fs.symlinkSync(outside, loader);
|
||||
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("escapes the current image");
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
ensureBundledPlugins,
|
||||
resolveBundledCatalogRoot,
|
||||
resolveBundledPluginInstalls,
|
||||
resolveManagedBundledPluginKeys,
|
||||
type BundledPluginProvisionerDeps,
|
||||
type ResolvedBundledPlugin,
|
||||
} from "../services/bundled-plugins.js";
|
||||
@@ -30,6 +31,21 @@ afterAll(() => {
|
||||
// Catalog-root resolution (fail-to-start allowlist)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("managed bundled plugin selection", () => {
|
||||
it.each([
|
||||
{ env: {}, selected: undefined, expected: null },
|
||||
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: undefined, expected: [] },
|
||||
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: " " }, selected: undefined, expected: null },
|
||||
{ env: { PAPERCLIP_MANAGED_CONFIG: "" }, selected: undefined, expected: [] },
|
||||
{ env: { PAPERCLIP_MANAGED_CONFIG: "{}" }, selected: null, expected: [] },
|
||||
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: ["daytona"], expected: ["daytona"] },
|
||||
{ env: {}, selected: ["modal"], expected: ["modal"] },
|
||||
{ env: {}, selected: [], expected: [] },
|
||||
])("uses the canonical Cloud floor and preserves explicit selection: $env / $selected", ({ env, selected, expected }) => {
|
||||
expect(resolveManagedBundledPluginKeys(selected, env)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveBundledPluginInstalls", () => {
|
||||
it("resolves the optional CreateOS sandbox provider without making it a self-hosted default", () => {
|
||||
expect(resolveBundledPluginInstalls(["createos"], {
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
function file(name: string, content = "export const value = 1;") {
|
||||
fs.mkdirSync(path.dirname(name), { recursive: true }); fs.writeFileSync(name, content);
|
||||
}
|
||||
function link(target: string, name: string) {
|
||||
fs.mkdirSync(path.dirname(name), { recursive: true }); fs.symlinkSync(path.relative(path.dirname(name), target), name);
|
||||
}
|
||||
function fixture() {
|
||||
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-package-graph-"))); roots.push(root);
|
||||
const catalogRoot = path.join(root, "packages", "plugins");
|
||||
const provider = path.join(catalogRoot, "sandbox-providers", "daytona");
|
||||
const sdk = path.join(catalogRoot, "sdk");
|
||||
const shared = path.join(root, "packages", "shared");
|
||||
const dependency = path.join(root, "node_modules", ".pnpm", "dependency@1", "node_modules", "dependency");
|
||||
const transitive = path.join(root, "node_modules", ".pnpm", "transitive@1", "node_modules", "transitive");
|
||||
for (const directory of [provider, sdk, shared, dependency, transitive]) file(path.join(directory, "index.js"));
|
||||
link(sdk, path.join(provider, "node_modules", "@paperclipai", "plugin-sdk"));
|
||||
link(shared, path.join(sdk, "node_modules", "@paperclipai", "shared"));
|
||||
link(dependency, path.join(shared, "node_modules", "dependency"));
|
||||
link(transitive, path.join(path.dirname(dependency), "transitive"));
|
||||
return { root, catalogRoot, provider, sdk, shared, dependency, transitive };
|
||||
}
|
||||
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
describe("bounded current-image package dependency inventory", () => {
|
||||
it("hashes external SDK, shared, and pnpm sibling dependency bytes", () => {
|
||||
const image = fixture();
|
||||
const digest = () => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
|
||||
let previous = digest();
|
||||
for (const directory of [image.sdk, image.shared, image.dependency, image.transitive]) {
|
||||
file(path.join(directory, "index.js"), "export const value = 2;");
|
||||
const next = digest(); expect(next).not.toBe(previous); previous = next;
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the digest stable when an identical release moves to a new root", () => {
|
||||
const original = fixture(); const replacement = fixture();
|
||||
expect(cloudBundledPackageDigest(original.provider, { catalogRoot: original.catalogRoot }))
|
||||
.toBe(cloudBundledPackageDigest(replacement.provider, { catalogRoot: replacement.catalogRoot }));
|
||||
});
|
||||
|
||||
it("rejects legacy-home links and preserves package-contained defaults", () => {
|
||||
const image = fixture();
|
||||
expect(() => cloudBundledPackageDigest(image.provider)).toThrow("escapes its package");
|
||||
const outside = fixture();
|
||||
link(path.join(outside.sdk, "index.js"), path.join(image.shared, "legacy-home.js"));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("authorized image roots");
|
||||
});
|
||||
|
||||
it("deduplicates cycles without scanning unrelated image node_modules", () => {
|
||||
const image = fixture();
|
||||
link(image.sdk, path.join(image.shared, "node_modules", "cycle"));
|
||||
const unrelated = path.join(image.root, "node_modules", "unrelated", "special");
|
||||
file(unrelated);
|
||||
const first = cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
|
||||
file(unrelated, "changed unrelated code");
|
||||
expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toBe(first);
|
||||
});
|
||||
|
||||
it.each(["", ".pnpm", ".pnpm/node_modules"])("rejects aggregate image dependency targets: %s", (aggregate) => {
|
||||
const image = fixture();
|
||||
const target = path.join(image.root, "node_modules", aggregate);
|
||||
fs.mkdirSync(target, { recursive: true });
|
||||
link(target, path.join(image.sdk, "node_modules", "aggregate"));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("select an individual package");
|
||||
});
|
||||
|
||||
it("accepts the shipped provider's observed entry class", () => {
|
||||
const image = fixture();
|
||||
const directory = path.join(image.provider, "many"); fs.mkdirSync(directory);
|
||||
for (let index = 0; index < 16_000; index++) fs.writeFileSync(path.join(directory, `${index}.js`), "");
|
||||
expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toMatch(/^sha256:/);
|
||||
});
|
||||
|
||||
it("rejects oversized metadata and excessive depth", () => {
|
||||
const image = fixture();
|
||||
file(path.join(image.sdk, "package.json"), " ".repeat(128 * 1024 + 1));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("metadata");
|
||||
fs.rmSync(path.join(image.sdk, "package.json"));
|
||||
const large = path.join(image.shared, "large.js");
|
||||
fs.closeSync(fs.openSync(large, "w")); fs.truncateSync(large, 256 * 1024 * 1024 + 1);
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("byte limit");
|
||||
fs.rmSync(large);
|
||||
file(path.join(image.sdk, ...Array.from({ length: 33 }, () => "deep"), "file.js"));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("depth limit");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,325 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, plugins, type Db } from "@paperclipai/db";
|
||||
import { eq, inArray, sql } from "drizzle-orm";
|
||||
import { paperclipConfigSchema } from "@paperclipai/shared";
|
||||
import { adapterSettingsService, initializeAdapterSettings } from "../services/adapter-settings.js";
|
||||
import { cloudRuntimeConfigService, initializeCloudRuntimeConfig } from "../services/cloud-runtime-config.js";
|
||||
import { cloudConfigurationMigrationService } from "../services/cloud-configuration-migration.js";
|
||||
import { qualifyCloudConfigurationForCurrentImage } from "../services/cloud-configuration-qualification.js";
|
||||
import { runCloudConfigurationQualification } from "../qualify-cloud-configuration.js";
|
||||
import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
|
||||
import { pluginRegistryService } from "../services/plugin-registry.js";
|
||||
import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { ensureBundledPlugins } from "../services/bundled-plugins.js";
|
||||
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL?.trim();
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const keys = ["adapter-settings:v1", "cloud-runtime-config:v1", "cloud-configuration-migration:v1"];
|
||||
(support.supported ? describe : describe.skip)("Cloud configuration migration and current-image qualification", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let peer: Db;
|
||||
let homeA: string;
|
||||
let homeB: string;
|
||||
const roots: string[] = [];
|
||||
const fixture = (version = "1.0.0") => { const image = bundledImageFixture({ version }); roots.push(image.root); return image; };
|
||||
const env = (home = homeA) => ({ PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "test-cloud-token", PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json") });
|
||||
const migration = (client = db, home = homeA) => cloudConfigurationMigrationService(client, { runtimeEnv: env(home) });
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) {
|
||||
await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
|
||||
} else database = await startEmbeddedPostgresTestDatabase("cloud-configuration-migration-");
|
||||
db = createDb(database.connectionString);
|
||||
peer = createDb(database.connectionString);
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
homeA = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-legacy-A-"));
|
||||
homeB = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-replacement-B-"));
|
||||
roots.push(homeA, homeB);
|
||||
await db.delete(instanceSettings).where(inArray(instanceSettings.singletonKey, keys));
|
||||
await db.delete(plugins);
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
async function legacy() {
|
||||
const config = paperclipConfigSchema.parse({
|
||||
$meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
|
||||
database: { mode: "postgres" }, logging: { mode: "cloud" },
|
||||
server: { port: 3199 }, telemetry: { enabled: false },
|
||||
});
|
||||
await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config));
|
||||
await fs.writeFile(path.join(homeA, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local", "future_adapter"] }));
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
|
||||
return config;
|
||||
}
|
||||
|
||||
function loaderFor(image: ReturnType<typeof fixture>, client = db, services?: PluginRuntimeServices) {
|
||||
return pluginLoader(client, {
|
||||
localPluginDir: path.join(homeB, "plugins"), prebuiltOnly: true,
|
||||
resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, services);
|
||||
}
|
||||
async function install(image: ReturnType<typeof fixture>) {
|
||||
await loaderFor(image).installPlugin({ localPath: image.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(image.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
return row;
|
||||
}
|
||||
function qualify(image: ReturnType<typeof fixture>, client = db, home = homeA, expectedPackageEvidenceDigest?: string) {
|
||||
const loader = loaderFor(image, client);
|
||||
return migration(client, home).qualify({ installs: image.installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest });
|
||||
}
|
||||
|
||||
it("imports A atomically, retains later DB revisions, and starts B's real image worker after A disappears", async () => {
|
||||
await legacy();
|
||||
const oldImage = fixture();
|
||||
const newImage = fixture();
|
||||
const row = await install(oldImage);
|
||||
const first = await migration().migrate();
|
||||
expect(first.migration.adapter.authority).toBe("legacy_file");
|
||||
expect(first.migration.config.authority).toBe("legacy_file");
|
||||
const original = await qualify(oldImage);
|
||||
expect(original.configurationPackagesEligible).toBe(true);
|
||||
expect(original.samePackageOverlapEligible).toBe(false);
|
||||
await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
|
||||
const config = (await cloudRuntimeConfigService(peer).inspectStored())!;
|
||||
expect(config.config).not.toBeNull();
|
||||
await cloudRuntimeConfigService(peer).replace({ ...config.config!, server: { ...config.config!.server, port: 3299 } }, config.revision);
|
||||
await fs.rm(homeA, { recursive: true });
|
||||
await fs.rm(oldImage.root, { recursive: true });
|
||||
expect((await migration(peer, homeB).migrate()).migration).toEqual(first.migration);
|
||||
await initializeAdapterSettings(peer, { runtimeEnv: env(homeB) });
|
||||
const restored = await initializeCloudRuntimeConfig(peer, { runtimeEnv: env(homeB) });
|
||||
expect(restored?.current.config?.server.port).toBe(3299);
|
||||
expect(await adapterSettingsService(peer, { runtimeEnv: env(homeB) }).getDisabledTypes()).toEqual(["claude_local", "codex_local", "future_adapter"]);
|
||||
const replacement = await qualify(newImage, peer, homeB, original.packageEvidenceDigest);
|
||||
expect(replacement).toMatchObject({ adapterRevision: 1, configRevision: 1, configurationPackagesEligible: true, samePackageOverlapEligible: true });
|
||||
expect(replacement.configurationDigest).not.toBe(original.configurationDigest);
|
||||
const manager = createPluginWorkerManager();
|
||||
try {
|
||||
const services = {
|
||||
workerManager: manager, lifecycleManager: { markError: vi.fn() },
|
||||
jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
|
||||
toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
|
||||
buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "qualification-fixture", hostVersion: "1.0.0" },
|
||||
} as unknown as PluginRuntimeServices;
|
||||
expect((await loaderFor(newImage, peer, services).loadSingle(row.id)).success).toBe(true);
|
||||
expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(newImage.packageRoot) } });
|
||||
} finally { await manager.stopAll(); }
|
||||
});
|
||||
|
||||
it("serializes concurrent attempts and never imports stale inputs over later DB changes", async () => {
|
||||
await legacy();
|
||||
const [first, second] = await Promise.all([migration().migrate(), migration(peer).migrate()]);
|
||||
expect(first).toEqual(second);
|
||||
await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("claude_local", false);
|
||||
await fs.writeFile(path.join(homeA, "config.json"), "invalid stale file");
|
||||
await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid stale file");
|
||||
expect(await migration().migrate()).toEqual(first);
|
||||
expect(await adapterSettingsService(db, { runtimeEnv: env() }).getDisabledTypes()).toEqual(["future_adapter"]);
|
||||
});
|
||||
|
||||
it("rolls back both seeds and the receipt if completion fails, then retries after the crash boundary", async () => {
|
||||
await legacy();
|
||||
await db.execute(sql`CREATE FUNCTION reject_cloud_migration_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.singleton_key = 'cloud-configuration-migration:v1' THEN RAISE EXCEPTION 'test receipt failure'; END IF; RETURN NEW; END $$`);
|
||||
await db.execute(sql`CREATE TRIGGER reject_cloud_migration_receipt BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_cloud_migration_receipt()`);
|
||||
try {
|
||||
await expect(migration().migrate()).rejects.toMatchObject({ cause: { message: expect.stringContaining("test receipt failure") } });
|
||||
expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
|
||||
} finally {
|
||||
await db.execute(sql`DROP TRIGGER reject_cloud_migration_receipt ON instance_settings`);
|
||||
await db.execute(sql`DROP FUNCTION reject_cloud_migration_receipt()`);
|
||||
}
|
||||
expect((await migration(peer).migrate()).revision).toBe(0);
|
||||
expect((await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).length).toBe(3);
|
||||
});
|
||||
|
||||
it("fails without partial authority for malformed legacy config and preserves already-authoritative rows", async () => {
|
||||
await legacy();
|
||||
await fs.writeFile(path.join(homeA, "config.json"), "invalid");
|
||||
await expect(migration().migrate()).rejects.toThrow("Legacy Cloud runtime configuration");
|
||||
expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
|
||||
await adapterSettingsService(db, { runtimeEnv: env() }).seedIfAbsent({ disabledTypes: ["codex_local"] });
|
||||
await cloudRuntimeConfigService(db).seedIfAbsent(null);
|
||||
await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid");
|
||||
const receipt = await migration(peer).migrate();
|
||||
expect(receipt.migration.adapter.authority).toBe("database");
|
||||
expect(receipt.migration.config.authority).toBe("database");
|
||||
expect((await cloudRuntimeConfigService(peer).inspectStored())?.config).toBeNull();
|
||||
});
|
||||
|
||||
it("returns database errors instead of falling back to files or a completed receipt", async () => {
|
||||
await legacy();
|
||||
await migration().migrate();
|
||||
const broken = createDb(database.connectionString);
|
||||
await broken.$client.end({ timeout: 0 });
|
||||
await expect(migration(broken).migrate()).rejects.toThrow();
|
||||
const image = fixture();
|
||||
await expect(qualify(image, broken)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("keeps external registration blockers durable when B has no legacy home and requires deliberate original-source resolution", async () => {
|
||||
await legacy();
|
||||
const sentinel = path.join(homeA, "external-code-executed");
|
||||
await fs.writeFile(path.join(homeA, "external.js"), `require('node:fs').writeFileSync(${JSON.stringify(sentinel)}, 'executed');`);
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), JSON.stringify([{ type: "legacy_external", packageName: "legacy-adapter", localPath: path.join(homeA, "external.js") }]));
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "unsupported_external_adapters" }));
|
||||
await expect(fs.stat(sentinel)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await migration(peer, homeB).migrate();
|
||||
const replacement = await qualify(image, peer, homeB);
|
||||
expect(replacement.configurationPackagesEligible).toBe(false);
|
||||
await expect(migration(peer, homeB).recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("original legacy source");
|
||||
await fs.unlink(path.join(homeA, "adapter-plugins.json"));
|
||||
await expect(migration().recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("present, valid registration file");
|
||||
expect((await migration().inspectStored())?.migration.externalRegistrations.blocker).toBe("unsupported_external_adapters");
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
|
||||
await expect(migration().recheckLegacyExternalRegistrations(0)).rejects.toThrow("revision conflict");
|
||||
await migration().recheckLegacyExternalRegistrations(replacement.receiptRevision);
|
||||
expect((await qualify(image)).configurationPackagesEligible).toBe(true);
|
||||
});
|
||||
|
||||
it("retains a metadata blocker for malformed or oversized registration files", async () => {
|
||||
await legacy();
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), Buffer.alloc(65_537));
|
||||
const receipt = await migration().migrate();
|
||||
expect(receipt.migration.externalRegistrations.blocker).toBe("invalid_external_registration_metadata");
|
||||
await fs.rm(homeA, { recursive: true });
|
||||
expect((await migration(peer, homeB).migrate()).migration.externalRegistrations).toEqual(receipt.migration.externalRegistrations);
|
||||
});
|
||||
|
||||
it("records image mismatches before reconciliation, permits maintenance upgrades, and requires explicit fresh overlap evidence", async () => {
|
||||
await legacy();
|
||||
const oldImage = fixture();
|
||||
const nextImage = fixture("1.1.0");
|
||||
await install(oldImage);
|
||||
await migration().migrate();
|
||||
const old = await qualify(oldImage);
|
||||
const mismatch = await qualify(nextImage, peer, homeB, old.packageEvidenceDigest);
|
||||
expect(mismatch.configurationPackagesEligible).toBe(false);
|
||||
expect(mismatch.samePackageOverlapEligible).toBe(false);
|
||||
expect(mismatch.blockers).toContainEqual(expect.objectContaining({ code: "incompatible_bundled_package", registeredVersion: "1.0.0", imageVersion: "1.1.0" }));
|
||||
const registry = pluginRegistryService(db);
|
||||
await ensureBundledPlugins(nextImage.installs, {
|
||||
registry, loader: loaderFor(nextImage), lifecycle: { load: vi.fn() },
|
||||
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}, { reinstallUninstalled: true, preservePackagePath: true });
|
||||
const current = await qualify(nextImage);
|
||||
expect(current.configurationPackagesEligible).toBe(true);
|
||||
expect(current.samePackageOverlapEligible).toBe(false);
|
||||
expect((await migration().inspectStored())?.lastPackageMismatch?.packageEvidenceDigest).toBe(mismatch.packageEvidenceDigest);
|
||||
expect((await qualify(nextImage, peer, homeB, current.packageEvidenceDigest)).samePackageOverlapEligible).toBe(true);
|
||||
});
|
||||
|
||||
it("fails qualification for missing package or manifest bytes without loading a home fallback", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const good = await qualify(image);
|
||||
await fs.rm(path.join(image.packageRoot, "dist", "manifest.js"));
|
||||
const broken = await qualify(image, peer, homeB, good.packageEvidenceDigest);
|
||||
expect(broken.configurationPackagesEligible).toBe(false);
|
||||
expect(broken.samePackageOverlapEligible).toBe(false);
|
||||
expect(broken.blockers.length).toBeGreaterThan(0);
|
||||
await fs.rm(image.packageRoot, { recursive: true });
|
||||
expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "missing_bundled_package" }));
|
||||
});
|
||||
|
||||
it("rejects changed image bytes before manifest import and never uses stale completed qualification", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const original = await qualify(image);
|
||||
await fs.appendFile(path.join(image.packageRoot, "dist", "worker.js"), "\nprocess.exit(99);\n");
|
||||
const loadManifest = vi.fn();
|
||||
const current = await migration(peer, homeB).qualify({ installs: image.installs, loadManifest, expectedPackageEvidenceDigest: original.packageEvidenceDigest });
|
||||
expect(current.configurationPackagesEligible).toBe(false);
|
||||
expect(current.samePackageOverlapEligible).toBe(false);
|
||||
expect(current.blockers).toContainEqual(expect.objectContaining({ code: "package_integrity_mismatch" }));
|
||||
expect(loadManifest).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows an administrator update while manifest inspection waits and refuses to certify the stale snapshot", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
let started!: () => void;
|
||||
let resume!: () => void;
|
||||
const entered = new Promise<void>((resolve) => { started = resolve; });
|
||||
const release = new Promise<void>((resolve) => { resume = resolve; });
|
||||
const loader = loaderFor(image);
|
||||
const pending = migration().qualify({ installs: image.installs, loadManifest: async (root) => { started(); await release; return loader.loadManifest(root); } });
|
||||
await entered;
|
||||
try {
|
||||
const change = adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
expect(await Promise.race([change, new Promise<never>((_resolve, reject) => { timer = setTimeout(() => reject(new Error("Admin update was blocked by image inspection")), 2_000); })])).toBe(true);
|
||||
} finally { clearTimeout(timer); }
|
||||
} finally { resume(); }
|
||||
const result = await pending;
|
||||
expect(result).toMatchObject({ configurationPackagesEligible: false, adapterRevision: 1 });
|
||||
expect(result.blockers).toContainEqual(expect.objectContaining({ code: "configuration_changed_during_inspection" }));
|
||||
expect((await qualify(image)).configurationPackagesEligible).toBe(true);
|
||||
});
|
||||
|
||||
it("uses the current managed catalog through the operator command and durably records catalog failure", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const runtimeEnv = {
|
||||
...env(homeB), DATABASE_URL: database.connectionString, PAPERCLIP_BUNDLED_PLUGIN_ROOT: image.root,
|
||||
PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "fixture", features: {}, plugins: { autoInstall: ["fixture"] } }),
|
||||
};
|
||||
const initial = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv });
|
||||
expect(initial?.configurationPackagesEligible).toBe(true);
|
||||
const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
try {
|
||||
expect(await runCloudConfigurationQualification(["--expected-package-evidence", initial!.packageEvidenceDigest], runtimeEnv)).toBe(0);
|
||||
const report = stdout.mock.calls.map((call) => String(call[0])).find((text) => text.startsWith('{\n "receiptRevision"'));
|
||||
expect(report && JSON.parse(report).samePackageOverlapEligible).toBe(true);
|
||||
} finally { stdout.mockRestore(); }
|
||||
const missing = await qualifyCloudConfigurationForCurrentImage(peer, { runtimeEnv: { ...runtimeEnv, PAPERCLIP_MANAGED_CONFIG: runtimeEnv.PAPERCLIP_MANAGED_CONFIG.replace('"fixture"]', '"absent"]') } });
|
||||
expect(missing?.configurationPackagesEligible).toBe(false);
|
||||
expect(missing?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
|
||||
expect((await migration().inspectStored())?.lastPackageMismatch?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("detects changed pnpm dependency bytes in a builtin package overlap comparison", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const dependency = path.join(image.packageRoot, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
|
||||
await fs.mkdir(dependency, { recursive: true });
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
|
||||
await fs.symlink(path.relative(path.join(image.packageRoot, "node_modules"), dependency), path.join(image.packageRoot, "node_modules", "dep"));
|
||||
const installs = image.installs.map(({ distribution: _distribution, ...install }) => install);
|
||||
const loader = loaderFor(image);
|
||||
const original = await migration().qualify({ installs, loadManifest: (root) => loader.loadManifest(root) });
|
||||
expect(original.configurationPackagesEligible).toBe(true);
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
|
||||
const changed = await migration(peer, homeB).qualify({ installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: original.packageEvidenceDigest });
|
||||
expect(changed.configurationPackagesEligible).toBe(true);
|
||||
expect(changed.samePackageOverlapEligible).toBe(false);
|
||||
expect(changed.overlapBlockers).toContainEqual(expect.objectContaining({ code: "package_evidence_changed" }));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { readLegacyAdapterSettings } from "../services/adapter-settings.js";
|
||||
import { inspectLegacyExternalRegistrations } from "../services/cloud-configuration-evidence.js";
|
||||
import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
|
||||
|
||||
vi.mock("../adapters/plugin-loader.js", () => ({ buildExternalAdapters: async () => [] }));
|
||||
|
||||
describe("bounded Cloud legacy readers and image inventory", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-config-reader-")); });
|
||||
afterEach(async () => { await fs.rm(root, { recursive: true, force: true }); });
|
||||
|
||||
it("distinguishes an absent adapter file from a saved empty preference", async () => {
|
||||
expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toBeNull();
|
||||
await fs.writeFile(path.join(root, "adapter-settings.json"), '{"disabledTypes":[]}');
|
||||
expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toEqual({ disabledTypes: [] });
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("rejects symlinks and FIFOs without waiting for a writer", async () => {
|
||||
const target = path.join(root, "target.json");
|
||||
const settings = path.join(root, "adapter-settings.json");
|
||||
await fs.writeFile(target, '{"disabledTypes":[]}');
|
||||
await fs.symlink(target, settings);
|
||||
await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toMatchObject({ code: "ELOOP" });
|
||||
expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("unreadable_external_registration_metadata");
|
||||
await fs.unlink(settings);
|
||||
execFileSync("mkfifo", [settings]);
|
||||
await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toThrow("regular file");
|
||||
expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("invalid_external_registration_metadata");
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("hashes pnpm-style contained links and the physical dependency bytes", async () => {
|
||||
const dependency = path.join(root, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
|
||||
await fs.mkdir(dependency, { recursive: true });
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
|
||||
await fs.symlink(path.relative(path.join(root, "node_modules"), dependency), path.join(root, "node_modules", "dep"));
|
||||
const first = cloudBundledPackageDigest(root);
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
|
||||
expect(cloudBundledPackageDigest(root)).not.toBe(first);
|
||||
await fs.unlink(path.join(root, "node_modules", "dep"));
|
||||
await fs.symlink("/", path.join(root, "node_modules", "dep"));
|
||||
expect(() => cloudBundledPackageDigest(root)).toThrow("escapes its package");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,302 @@
|
||||
import fs from "node:fs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, type Db } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { paperclipConfigSchema } from "@paperclipai/shared";
|
||||
import { loadCloudBootstrapConfig, loadConfig } from "../config.js";
|
||||
import { readConfigFile } from "../config-file.js";
|
||||
import {
|
||||
CLOUD_RUNTIME_CONFIG_KEY,
|
||||
cloudRuntimeConfigService,
|
||||
initializeCloudRuntimeConfig,
|
||||
readLegacyCloudRuntimeConfig,
|
||||
} from "../services/cloud-runtime-config.js";
|
||||
import {
|
||||
getCloudRuntimeConfigSnapshot,
|
||||
projectCloudRuntimeConfig,
|
||||
setCloudRuntimeConfigSnapshot,
|
||||
} from "../services/cloud-runtime-config-snapshot.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL;
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describePostgres = support.supported ? describe : describe.skip;
|
||||
let home: string;
|
||||
let configPath: string;
|
||||
|
||||
function legacyConfig() {
|
||||
return {
|
||||
$meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
|
||||
llm: { provider: "openai", apiKey: "legacy-openai-secret" },
|
||||
database: {
|
||||
mode: "postgres",
|
||||
connectionString: "postgres://legacy:database-secret@example.test/paperclip",
|
||||
backup: { enabled: false, intervalMinutes: 73, retentionDays: 11, dir: "/tmp/legacy-backups" },
|
||||
},
|
||||
logging: { mode: "cloud", logDir: "/tmp/legacy-logs" },
|
||||
server: { deploymentMode: "authenticated", exposure: "private", port: 3210, serveUi: false, allowedHostnames: ["saved.example.test"] },
|
||||
auth: { baseUrlMode: "explicit", publicBaseUrl: "https://saved.example.test", disableSignUp: true },
|
||||
storage: { provider: "s3", s3: { bucket: "saved-bucket", region: "eu-west-1", prefix: "saved/", forcePathStyle: true } },
|
||||
secrets: { provider: "local_encrypted", strictMode: true, localEncrypted: { keyFilePath: "/tmp/legacy-master.key" } },
|
||||
telemetry: { enabled: false },
|
||||
updates: { checkEnabled: false },
|
||||
unsupportedSecret: "unknown-secret-value",
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-runtime-config-"));
|
||||
configPath = path.join(home, "config.json");
|
||||
vi.stubEnv("PAPERCLIP_HOME", home);
|
||||
vi.stubEnv("PAPERCLIP_CONFIG", configPath);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-token");
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", undefined);
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
|
||||
vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", undefined);
|
||||
vi.stubEnv("PAPERCLIP_AUTH_BASE_URL_MODE", undefined);
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", undefined);
|
||||
vi.stubEnv("BETTER_AUTH_URL", undefined);
|
||||
vi.stubEnv("BETTER_AUTH_BASE_URL", undefined);
|
||||
vi.stubEnv("PAPERCLIP_ALLOWED_HOSTNAMES", undefined);
|
||||
vi.stubEnv("PAPERCLIP_TAILNET_BIND_HOST", "100.100.0.1");
|
||||
vi.stubEnv("OPENAI_API_KEY", "environment-api-secret");
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("Cloud configuration file boundary", () => {
|
||||
it.each(["null", "{", JSON.stringify({ unexpected: true })])("rejects malformed present legacy data %s", async (contents) => {
|
||||
fs.writeFileSync(configPath, contents);
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/Legacy Cloud runtime configuration/);
|
||||
});
|
||||
|
||||
it("rejects oversized and linked legacy sources", async () => {
|
||||
fs.writeFileSync(configPath, " ".repeat(65_537));
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
|
||||
fs.rmSync(configPath);
|
||||
const target = path.join(home, "target.json");
|
||||
fs.writeFileSync(target, JSON.stringify(legacyConfig()));
|
||||
fs.symlinkSync(target, configPath);
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
|
||||
});
|
||||
|
||||
it("requires an environment credential for the legacy OpenAI consumer", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
vi.stubEnv("OPENAI_API_KEY", undefined);
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/OPENAI_API_KEY/);
|
||||
expect(() => readConfigFile()).toThrow(/not been initialized/);
|
||||
});
|
||||
|
||||
it.each(["https://user:secret@saved.example.test", "https://saved.example.test?key=secret", "https://saved.example.test#secret"])("rejects secret-bearing URL %s", async (url) => {
|
||||
const config = legacyConfig();
|
||||
config.auth.publicBaseUrl = url;
|
||||
fs.writeFileSync(configPath, JSON.stringify(config));
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/configuration is invalid/);
|
||||
});
|
||||
|
||||
it("loads the minimal Cloud bootstrap from environment only", () => {
|
||||
fs.writeFileSync(configPath, "malformed");
|
||||
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:bootstrap-secret@127.0.0.1:5432/runtime");
|
||||
expect(loadCloudBootstrapConfig()).toMatchObject({ databaseMode: "postgres", databaseUrl: process.env.DATABASE_URL });
|
||||
vi.stubEnv("DATABASE_URL", undefined);
|
||||
expect(() => loadCloudBootstrapConfig()).toThrow(/requires DATABASE_URL/);
|
||||
vi.stubEnv("DATABASE_URL", "sqlite:///tmp/db");
|
||||
expect(() => loadCloudBootstrapConfig()).toThrow(/must use PostgreSQL/);
|
||||
});
|
||||
|
||||
it.each(["", " "])("treats an empty migration URL as absent (%j)", (migrationUrl) => {
|
||||
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", migrationUrl);
|
||||
expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBeUndefined();
|
||||
});
|
||||
|
||||
it("normalizes and validates an explicit migration URL", () => {
|
||||
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", " postgres://migration:password@127.0.0.1/runtime ");
|
||||
expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBe("postgres://migration:password@127.0.0.1/runtime");
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", "https://wrong.example.test");
|
||||
expect(() => loadCloudBootstrapConfig()).toThrow("DATABASE_MIGRATION_URL must use PostgreSQL");
|
||||
});
|
||||
|
||||
it("preserves local config file behavior without managed signals", () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
expect(readConfigFile()?.llm?.apiKey).toBe("legacy-openai-secret");
|
||||
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
|
||||
});
|
||||
});
|
||||
|
||||
describePostgres("Cloud runtime configuration in PostgreSQL", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let first: Db;
|
||||
let second: Db;
|
||||
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) {
|
||||
await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl, { scope: "database" }) };
|
||||
} else {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-config-db-");
|
||||
}
|
||||
first = createDb(database.connectionString);
|
||||
second = createDb(database.connectionString);
|
||||
}, 90_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await first.delete(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
|
||||
vi.stubEnv("DATABASE_URL", database.connectionString);
|
||||
});
|
||||
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])("seeds a fresh database with %s", async (signal) => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
|
||||
const result = await initializeCloudRuntimeConfig(first);
|
||||
expect(result).toMatchObject({ seeded: true, current: { version: 1, revision: 0, config: null } });
|
||||
expect(await cloudRuntimeConfigService(second).inspectStored()).toEqual(result?.current);
|
||||
expect(fs.existsSync(configPath)).toBe(false);
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", "authenticated");
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://environment.example.test");
|
||||
expect(loadConfig()).toMatchObject({ databaseMode: "postgres", authBaseUrlMode: "explicit", secretsStrictMode: true, telemetryEnabled: true });
|
||||
});
|
||||
|
||||
it("loads the saved authenticated mode before validating an environment host bind", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
vi.stubEnv("HOST", "0.0.0.0");
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
|
||||
expect(loadCloudBootstrapConfig()).toEqual({
|
||||
databaseMode: "postgres",
|
||||
databaseUrl: database.connectionString,
|
||||
databaseMigrationUrl: process.env.DATABASE_MIGRATION_URL,
|
||||
});
|
||||
await initializeCloudRuntimeConfig(first);
|
||||
expect(loadConfig()).toMatchObject({ deploymentMode: "authenticated", host: "0.0.0.0", port: 3210 });
|
||||
});
|
||||
|
||||
it("imports existing choices without credentials and restores them from a second client and new home", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
const initial = await initializeCloudRuntimeConfig(first);
|
||||
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, databaseBackupEnabled: false, storageS3Bucket: "saved-bucket", authDisableSignUp: true });
|
||||
const [row] = await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
|
||||
const persisted = JSON.stringify(row.general);
|
||||
for (const secret of ["legacy-openai-secret", "database-secret", "unknown-secret-value", "environment-api-secret", database.connectionString]) {
|
||||
expect(persisted).not.toContain(secret);
|
||||
}
|
||||
const replacementHome = fs.mkdtempSync(path.join(home, "replacement-"));
|
||||
const replacementConfigDir = path.join(replacementHome, ".paperclip");
|
||||
fs.mkdirSync(replacementConfigDir);
|
||||
const envPath = path.join(replacementConfigDir, ".env");
|
||||
const envContents = "PORT=9999\nPAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL=should-not-load\n";
|
||||
fs.writeFileSync(envPath, envContents);
|
||||
vi.stubEnv("PAPERCLIP_HOME", replacementHome);
|
||||
vi.stubEnv("PAPERCLIP_CONFIG", path.join(replacementConfigDir, "config.json"));
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
const files = vi.spyOn(fs, "readFileSync");
|
||||
expect(await initializeCloudRuntimeConfig(second)).toEqual({ seeded: false, current: initial?.current });
|
||||
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
|
||||
expect(files.mock.calls.map(([file]) => String(file)).filter(file => file.startsWith(replacementHome))).toEqual([]);
|
||||
files.mockRestore();
|
||||
const root = fileURLToPath(new URL("../../../", import.meta.url));
|
||||
const childScript = path.join(home, "restore.mts");
|
||||
fs.writeFileSync(childScript, `
|
||||
import { createDb } from ${JSON.stringify(pathToFileURL(path.join(root, "packages/db/src/index.ts")).href)};
|
||||
import { initializeCloudRuntimeConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/services/cloud-runtime-config.ts")).href)};
|
||||
import { loadConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/config.ts")).href)};
|
||||
const db = createDb(process.env.DATABASE_URL);
|
||||
await initializeCloudRuntimeConfig(db);
|
||||
const config = loadConfig();
|
||||
console.log(JSON.stringify({ port: config.port, telemetryEnabled: config.telemetryEnabled, storageS3Bucket: config.storageS3Bucket,
|
||||
envSentinel: process.env.PAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL ?? null }));
|
||||
await db.$client.end({ timeout: 1 });
|
||||
`);
|
||||
const child = await promisify(execFile)(process.execPath, [path.join(root, "server/node_modules/tsx/dist/cli.mjs"), childScript], {
|
||||
cwd: replacementHome,
|
||||
env: { ...process.env },
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(JSON.parse(child.stdout.trim())).toEqual({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket", envSentinel: null });
|
||||
expect(fs.readFileSync(envPath, "utf8")).toBe(envContents);
|
||||
}, 45_000);
|
||||
|
||||
it("keeps environment overlays out of saved settings and ignores corrupt old files after seed", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
await initializeCloudRuntimeConfig(first);
|
||||
vi.stubEnv("PORT", "4567");
|
||||
vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "operator-bucket");
|
||||
vi.stubEnv("PAPERCLIP_AUTH_DISABLE_SIGN_UP", "false");
|
||||
fs.writeFileSync(configPath, "malformed legacy contents");
|
||||
await initializeCloudRuntimeConfig(second);
|
||||
expect(loadConfig()).toMatchObject({ port: 4567, storageS3Bucket: "operator-bucket", authDisableSignUp: false, telemetryEnabled: false });
|
||||
expect((await cloudRuntimeConfigService(first).inspectStored())?.config).toMatchObject({ server: { port: 3210 }, storage: { s3: { bucket: "saved-bucket" } }, auth: { disableSignUp: true } });
|
||||
});
|
||||
|
||||
it("keeps one committed seed when two clients propose different initial values", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
const results = await Promise.all([
|
||||
cloudRuntimeConfigService(first).seedIfAbsent({ ...config, telemetry: { enabled: true } }),
|
||||
cloudRuntimeConfigService(second).seedIfAbsent({ ...config, telemetry: { enabled: false } }),
|
||||
]);
|
||||
expect(results.filter(result => result.seeded)).toHaveLength(1);
|
||||
expect(results[0].current).toEqual(results[1].current);
|
||||
expect(await cloudRuntimeConfigService(first).inspectStored()).toEqual(results[0].current);
|
||||
});
|
||||
|
||||
it("rejects unknown or credential fields on the direct mutation path", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
await cloudRuntimeConfigService(first).seedIfAbsent(config);
|
||||
await expect(cloudRuntimeConfigService(first).replace({ ...config, llm: { provider: "openai", apiKey: "must-not-store" } } as never, 0)).rejects.toThrow(/configuration is invalid/);
|
||||
expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(0);
|
||||
});
|
||||
|
||||
it("serializes revisions and rejects competing stale writes from two database clients", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
await cloudRuntimeConfigService(first).seedIfAbsent(config);
|
||||
const results = await Promise.allSettled([
|
||||
cloudRuntimeConfigService(first).replace({ ...config, telemetry: { enabled: true } }, 0),
|
||||
cloudRuntimeConfigService(second).replace({ ...config, telemetry: { enabled: false } }, 0),
|
||||
]);
|
||||
expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1);
|
||||
expect(results.filter(result => result.status === "rejected")).toHaveLength(1);
|
||||
expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(1);
|
||||
});
|
||||
|
||||
it("composes a seed with the caller transaction and rolls back both on failure", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
await expect(first.transaction(async tx => {
|
||||
await cloudRuntimeConfigService(first).seedIfAbsent(config, tx);
|
||||
await tx.insert(instanceSettings).values({ singletonKey: "runtime-config-test-receipt", general: {} });
|
||||
throw new Error("injected migration failure");
|
||||
})).rejects.toThrow(/injected migration failure/);
|
||||
expect(await cloudRuntimeConfigService(second).inspectStored()).toBeNull();
|
||||
expect(await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "runtime-config-test-receipt"))).toEqual([]);
|
||||
});
|
||||
|
||||
it("fails closed on invalid database settings without consulting a valid legacy file", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 99, config: { secret: "hidden" } } });
|
||||
await expect(initializeCloudRuntimeConfig(second)).rejects.toThrow(/Persisted Cloud runtime configuration is invalid/);
|
||||
expect(() => getCloudRuntimeConfigSnapshot()).toThrow(/not been initialized/);
|
||||
});
|
||||
|
||||
it("does not fall back to a file or old snapshot when the database fails", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
await initializeCloudRuntimeConfig(first);
|
||||
const unavailable = createDb(database.connectionString);
|
||||
await unavailable.$client.end({ timeout: 1 });
|
||||
await expect(initializeCloudRuntimeConfig(unavailable)).rejects.toThrow();
|
||||
expect(() => readConfigFile()).toThrow(/not been initialized/);
|
||||
});
|
||||
});
|
||||
@@ -32,8 +32,10 @@ import { activityRoutes } from "../routes/activity.js";
|
||||
import { agentRoutes } from "../routes/agents.js";
|
||||
import { authorizationService } from "../services/authorization.js";
|
||||
import { canActorReadWorkspaceOperation, getHeartbeatRunReadDecisions } from "../services/heartbeat-run-privacy.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, () => {
|
||||
@@ -42,8 +44,8 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
const previousPrivacyMode = process.env.PAPERCLIP_ISSUE_PRIVACY_MODE;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-");
|
||||
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString);
|
||||
process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce";
|
||||
}, 120_000);
|
||||
|
||||
@@ -66,6 +68,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
if (previousPrivacyMode === undefined) delete process.env.PAPERCLIP_ISSUE_PRIVACY_MODE;
|
||||
else process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = previousPrivacyMode;
|
||||
await tempDb?.cleanup();
|
||||
if (externalDatabaseUrl) await db.$client.end({ timeout: 1 });
|
||||
});
|
||||
|
||||
async function seedFixture() {
|
||||
@@ -264,7 +267,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
runnerProfileJson: { instructions: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
|
||||
}).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
@@ -281,8 +284,9 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
} finally { capture.flush(); await observed.$client.end(); }
|
||||
});
|
||||
|
||||
it("returns 404 for direct private-run content and metadata-only list stubs to a non-member", async () => {
|
||||
it.each(["local_file", "s3_segments"])("returns 404 for private %s run content and metadata-only list stubs", async (logStore) => {
|
||||
const fixture = await seedFixture();
|
||||
await db.update(heartbeatRuns).set({ logStore }).where(eq(heartbeatRuns.id, fixture.privateRunId));
|
||||
const app = createApp(fixture.companyId, fixture.otherAgentId);
|
||||
|
||||
for (const path of [
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { mkdtempSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../../services/bundled-plugins.js";
|
||||
import { distributionBundleDigest, readDistributionPluginCatalog } from "../../services/distribution-plugin-catalog.js";
|
||||
import { derivePluginDatabaseNamespace } from "../../services/plugin-database.js";
|
||||
import { createBundledPluginPackageResolver } from "../../services/bundled-plugin-package.js";
|
||||
|
||||
export function bundledImageFixture(manifestOverrides: Partial<PaperclipPluginManifestV1> = {}) {
|
||||
const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "bundled-image-")));
|
||||
const packageRoot = path.join(root, "distribution", "fixture");
|
||||
const manifest: PaperclipPluginManifestV1 = {
|
||||
id: "paperclip.image-fixture", apiVersion: 1, version: "1.0.0", displayName: "Image Fixture",
|
||||
description: "Image package relocation fixture", author: "Paperclip", categories: ["automation"], capabilities: ["issues.read"],
|
||||
entrypoints: { worker: "dist/worker.js", ui: "dist/ui" }, ...manifestOverrides,
|
||||
};
|
||||
mkdirSync(path.join(packageRoot, "dist", "ui"), { recursive: true });
|
||||
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify({
|
||||
name: "@paperclipai/plugin-image-fixture", version: manifest.version, type: "module",
|
||||
paperclipPlugin: { manifest: "dist/manifest.js", ...manifest.entrypoints },
|
||||
}));
|
||||
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
|
||||
writeFileSync(path.join(packageRoot, "dist", "worker.js"), `import readline from "node:readline";
|
||||
if (process.argv.includes("--source")) process.stdout.write(import.meta.url);
|
||||
else {
|
||||
const configs = {};
|
||||
readline.createInterface({ input: process.stdin }).on("line", line => {
|
||||
const request = JSON.parse(line);
|
||||
let result = {};
|
||||
if (request.method === "initialize") result = { ok: true, supportedMethods: ["configChanged", "health"] };
|
||||
if (request.method === "configChanged") configs[request.params.companyId] = request.params.config;
|
||||
if (request.method === "health") result = { status: "ok", details: { source: import.meta.url, configs } };
|
||||
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result }) + "\\n");
|
||||
if (request.method === "shutdown") setImmediate(() => process.exit(0));
|
||||
});
|
||||
}`);
|
||||
writeFileSync(path.join(packageRoot, "dist", "ui", "index.js"), "export const source = 'current-image';");
|
||||
if (manifest.database) {
|
||||
mkdirSync(path.join(packageRoot, manifest.database.migrationsDir), { recursive: true });
|
||||
writeFileSync(path.join(packageRoot, manifest.database.migrationsDir, "001_fixture.sql"), `CREATE TABLE ${derivePluginDatabaseNamespace(manifest.id)}.image_rows (id uuid PRIMARY KEY);`);
|
||||
}
|
||||
const entry = { key: "fixture", pluginKey: manifest.id, version: manifest.version, directory: "fixture", digest: distributionBundleDigest(packageRoot) };
|
||||
writeFileSync(path.join(root, "distribution", "catalog.json"), JSON.stringify({ schemaVersion: 1, plugins: [entry] }));
|
||||
const catalog = readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG);
|
||||
const installs = resolveBundledPluginInstalls(["fixture"], { catalogRoot: root, enforceCatalogRoot: true, env: {}, distributionPlugins: catalog });
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const registration = { pluginKey: manifest.id, packageName: "@paperclipai/plugin-image-fixture", version: manifest.version, manifestJson: manifest };
|
||||
return { root, packageRoot, manifest, catalog, installs, resolver, registration };
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
|
||||
export async function createNativeRunLogTestTables(db: Db) {
|
||||
await db.execute(sql`create table heartbeat_runs(id uuid primary key, company_id uuid not null, agent_id uuid not null,
|
||||
status text not null, runtime_mode text not null default 'native', native_session_id text, runner_instance_id text,
|
||||
native_issue_id uuid, issue_id uuid, context_snapshot jsonb,
|
||||
log_store text, log_ref text, log_bytes bigint,
|
||||
log_sha256 text, last_output_seq bigint not null default 0, last_output_at timestamptz, last_output_stream text,
|
||||
last_output_bytes bigint, next_event_seq bigint not null default 1, controller_lease_expires_at timestamptz,
|
||||
created_at timestamptz not null default now(), updated_at timestamptz not null default now())`);
|
||||
await db.execute(sql`create table heartbeat_run_events(id bigserial primary key, company_id uuid not null,
|
||||
run_id uuid not null, agent_id uuid not null, seq bigint not null, event_type text not null, stream text,
|
||||
level text, color text, message text, payload jsonb, source_instance_id text, source_event_id text,
|
||||
source_seq bigint, source_payload_sha256 text, protocol_schema_version int, created_at timestamptz not null default now(),
|
||||
unique(run_id,seq), unique(run_id,source_event_id), unique(run_id,source_instance_id,source_seq))`);
|
||||
await db.execute(
|
||||
sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`,
|
||||
);
|
||||
await db.execute(sql`create table agent_identity_keys(agent_id uuid primary key,company_id uuid,algorithm text,key_id text,
|
||||
public_key_pem text,private_key_material jsonb,created_at timestamptz default now())`);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
// A real receiver process for the credential-loss boundary test. No provider
|
||||
// credentials leave this process through logs, IPC, or diagnostic exceptions.
|
||||
import { createDb } from "@paperclipai/db";
|
||||
import { aiConnectionService } from "../../services/ai-connections.js";
|
||||
|
||||
const db = createDb(process.env.CREDENTIAL_TEST_DATABASE_URL!);
|
||||
const provider = process.env.CREDENTIAL_TEST_PROVIDER_URL!;
|
||||
const request = globalThis.fetch;
|
||||
globalThis.fetch = (url, init) => {
|
||||
if (String(url) !== "https://auth.openai.com/oauth/token") throw new Error("Unexpected fixture network request");
|
||||
return request(provider, init);
|
||||
};
|
||||
try {
|
||||
const service = aiConnectionService(db);
|
||||
const [row] = await service.quotaAccounts(process.env.CREDENTIAL_TEST_COMPANY_ID!, process.env.CREDENTIAL_TEST_OWNER!);
|
||||
await service.refreshQuotaCredential(row, await service.credential(row), new AbortController().signal);
|
||||
process.exitCode = 0;
|
||||
} catch { process.exitCode = 1; }
|
||||
finally { await db.$client.end(); }
|
||||
@@ -1090,6 +1090,7 @@ describe("agent issue mutation checkout ownership", () => {
|
||||
]);
|
||||
expect(mockAccessService.decide).toHaveBeenCalledWith(expect.objectContaining({ action: "issue:read" }));
|
||||
expect(mockIssueService.listComments).toHaveBeenCalledWith(issueId, {
|
||||
companyId,
|
||||
afterCommentId: null,
|
||||
order: "desc",
|
||||
limit: null,
|
||||
|
||||
@@ -580,16 +580,33 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => {
|
||||
.where(inArray(heartbeatRuns.id, work.map(row => row.id)));
|
||||
clock();
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
let initial: Awaited<ReturnType<typeof states>> | undefined;
|
||||
let releaseFirstPass!: () => void;
|
||||
const firstPassHeld = new Promise<void>(resolve => { releaseFirstPass = resolve; });
|
||||
vi.spyOn(coordinator, "register").mockImplementation((queue, task) => register(queue, {
|
||||
...task,
|
||||
run: async signal => {
|
||||
const result = await task.run(signal);
|
||||
if (initial) return result;
|
||||
const cancelled = (await states()).filter(row => row.status === "cancelled");
|
||||
if (cancelled.length === 0) return result;
|
||||
initial = cancelled;
|
||||
await firstPassHeld;
|
||||
return result;
|
||||
},
|
||||
}));
|
||||
try {
|
||||
await heartbeatService(db, { runtimeEnv: {} }).registerScheduledRetryCancellationWork(coordinator, () => true).ready;
|
||||
expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(0);
|
||||
const earlierId = "00000000-0000-4000-8000-000000000000";
|
||||
await writeIntent({ companyId: a.companyId, retryRunId: earlierId });
|
||||
work.push({ id: earlierId });
|
||||
await writeIntent({ companyId: a.companyId, retryRunId: earlierId });
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await expect.poll(async () => (await states()).filter(row => row.status === "cancelled").length).toBe(20);
|
||||
const initial = (await states()).filter(row => row.status === "cancelled");
|
||||
expect(new Set(initial.map(row => row.companyId)).size).toBe(2);
|
||||
await expect.poll(() => initial).toBeDefined();
|
||||
expect(initial).toHaveLength(20);
|
||||
expect(new Set(initial!.map(row => row.companyId)).size).toBe(2);
|
||||
releaseFirstPass();
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(29);
|
||||
@@ -600,7 +617,7 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => {
|
||||
const reads = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(180_000);
|
||||
expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
|
||||
} finally { await coordinator.stop(); }
|
||||
} finally { releaseFirstPass(); await coordinator.stop(); }
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ import {
|
||||
readIssueCommentRunLogText,
|
||||
} from "../services/issues.ts";
|
||||
import { getRunLogStore } from "../services/run-log-store.js";
|
||||
import * as nativeRunLogRecovery from "../services/native-runtime/native-run-log-recovery.js";
|
||||
import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
|
||||
import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js";
|
||||
import {
|
||||
@@ -192,6 +193,35 @@ describeEmbeddedPostgres("issueService run attachment artifacts", () => {
|
||||
});
|
||||
|
||||
describe("readIssueCommentRunLogText", () => {
|
||||
it("does not read after a timed-out native repair", async () => {
|
||||
const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockImplementationOnce((_db, _store, _ref, _redaction, signal) => new Promise((_resolve, reject) => {
|
||||
signal!.addEventListener("abort", () => reject(signal!.reason), { once: true });
|
||||
}));
|
||||
const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValue({ content: "too late" });
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const pending = readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType<typeof createDb>);
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(3_000);
|
||||
await expect(pending).resolves.toBe("");
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await pending;
|
||||
vi.useRealTimers();
|
||||
repair.mockRestore();
|
||||
read.mockRestore();
|
||||
}
|
||||
});
|
||||
it("keeps published evidence when a concurrent native repair wins ownership", async () => {
|
||||
const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockRejectedValueOnce(new Error("The output journal is owned by another writer"));
|
||||
const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValueOnce({ content: "published evidence" });
|
||||
try {
|
||||
await expect(readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType<typeof createDb>)).resolves.toBe("published evidence");
|
||||
expect(read).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
repair.mockRestore();
|
||||
read.mockRestore();
|
||||
}
|
||||
});
|
||||
it("cancels timed-out storage reads so later listings can recover", async () => {
|
||||
let active = 0;
|
||||
const cleanups: Array<() => void> = [];
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createLocalEncryptedCipher, localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
|
||||
import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret, localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
|
||||
|
||||
describe("operation-scoped local encryption", () => {
|
||||
let root: string;
|
||||
@@ -60,4 +60,16 @@ describe("operation-scoped local encryption", () => {
|
||||
code: "provider_unavailable", operation: "resolveVersion",
|
||||
}));
|
||||
});
|
||||
it("derives the same instance secret from an existing key file or environment without creating missing keys", () => {
|
||||
expect(() => deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false }))
|
||||
.toThrow("Secrets master key file is missing");
|
||||
expect(existsSync(keyFile)).toBe(false);
|
||||
const key = randomBytes(32).toString("hex");
|
||||
writeFileSync(keyFile, key, { mode: 0o600 });
|
||||
const fromFile = deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false });
|
||||
unlinkSync(keyFile);
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", key);
|
||||
expect(deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false })).toBe(fromFile);
|
||||
expect(existsSync(keyFile)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -654,8 +654,13 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
|
||||
const available = await seedArchive(healthy.root, { name: `${dueAt - 7 * DAY_MS - 1}-${randomUUID()}` });
|
||||
await startEvidence({ isSessionBusy: scope => scope === busy.scope }); await settled(dueAt);
|
||||
await vi.advanceTimersByTimeAsync(dueAt - Date.now());
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(100); expect(await exists(available)).toBe(false); });
|
||||
expect(await exists(held)).toBe(true); expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000);
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(await exists(available)).toBe(false);
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000);
|
||||
});
|
||||
expect(await exists(held)).toBe(true);
|
||||
await quiet(60_000);
|
||||
});
|
||||
|
||||
|
||||
@@ -315,6 +315,21 @@ function loadSpecRoutes() {
|
||||
}
|
||||
|
||||
describe("openapi routes", () => {
|
||||
it.each(["summary", "tasks"])("documents the board-only compact attention %s contract", projection => {
|
||||
const spec = buildOpenApiSpec() as any;
|
||||
const operation = spec.paths[`/api/companies/{companyId}/attention/${projection}`].get;
|
||||
expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]);
|
||||
expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" });
|
||||
expect(operation.parameters.filter((parameter: { in: string }) => parameter.in === "query")).toEqual([]);
|
||||
const response = operation.responses["200"].content["application/json"].schema;
|
||||
expect(response.properties.totalCount).toMatchObject({ type: "integer", minimum: 0 });
|
||||
expect(response.properties.generatedAt).toMatchObject({ format: "date-time" });
|
||||
expect(response.properties.items).toBeUndefined();
|
||||
expect(response.properties.nextCursor).toBeUndefined();
|
||||
if (projection === "tasks") expect(response.properties.taskIds).toMatchObject({ type: "array", items: { type: "string", format: "uuid" } });
|
||||
for (const code of ["200", "401", "403"]) expect(operation.responses[code]).toBeDefined();
|
||||
});
|
||||
|
||||
it("documents the owned pending-question read as canonical attention items without client scope overrides", () => {
|
||||
const spec = buildOpenApiSpec() as any;
|
||||
const operation = spec.paths["/api/issues/{id}/pending-questions"].get;
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, realpath, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
@@ -90,6 +90,26 @@ suite("durable plugin task credential handoff", () => {
|
||||
expect(await f.credential()).toBe(auth("fixture-refreshed", 2021));
|
||||
expect(f.calls.at(-1)).toBe("complete");
|
||||
});
|
||||
it("retains the Runner credential after a database save failure until a fresh receiver commits", async () => {
|
||||
const f = await fixture();
|
||||
await db.execute(sql.raw(`create function reject_runner_credential_save() returns trigger language plpgsql as $$ begin
|
||||
if new.secret_id = '${f.managed.credentialWriteback!.secretId}'::uuid then raise exception 'fixture receiver save unavailable'; end if; return new; end $$;
|
||||
create constraint trigger reject_runner_credential_save after insert on company_secret_versions
|
||||
deferrable initially deferred for each row execute function reject_runner_credential_save();`));
|
||||
try {
|
||||
await expect(f.settle()).rejects.toThrow();
|
||||
expect(f.calls).toContain("credentials");
|
||||
expect(f.calls).not.toContain("complete");
|
||||
expect(await f.credential()).toBe(auth("fixture-original"));
|
||||
expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "pending" });
|
||||
} finally { await db.execute(sql`drop trigger reject_runner_credential_save on company_secret_versions; drop function reject_runner_credential_save()`); }
|
||||
const receiver = createDb(database.connectionString);
|
||||
expect(await settleDispatchedEnvironmentTask(receiver, f.workers, { companyId: f.companyId, id: f.leaseId })).toBe(true);
|
||||
expect(await f.credential()).toBe(auth("fixture-refreshed", 2021));
|
||||
expect(f.calls.filter(call => call === "credentials")).toHaveLength(2);
|
||||
expect(f.calls.at(-1)).toBe("complete");
|
||||
expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "complete" });
|
||||
});
|
||||
it("does not overwrite an explicit account replacement", async () => {
|
||||
const f = await fixture();
|
||||
await secretService(db).rotate(f.managed.credentialWriteback!.secretId, { value: auth("fixture-reconnected", 2022) }, { userId: f.owner });
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { registerProviderTraceRetentionWork } from "../services/provider-trace-retention-work.js";
|
||||
|
||||
const state = vi.hoisted(() => ({
|
||||
cleanup: vi.fn(async () => []),
|
||||
next: vi.fn<() => Promise<number | null>>(),
|
||||
probe: undefined as undefined | (() => string),
|
||||
unregister: vi.fn(),
|
||||
}));
|
||||
vi.mock("../services/provider-trace-store.js", () => ({
|
||||
providerTraceStore: () => ({ cleanupExpired: state.cleanup, nextRetentionDeadline: state.next }),
|
||||
}));
|
||||
vi.mock("../services/idle-local-work.js", () => ({
|
||||
registerIdleLocalWorkProbe: (probe: () => string) => {
|
||||
state.probe = probe;
|
||||
return state.unregister;
|
||||
},
|
||||
}));
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop();
|
||||
vi.useRealTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("reconstructs trace deadlines, stays quiet between obligations, and discharges the sleep blocker", async () => {
|
||||
vi.useFakeTimers();
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
|
||||
const onError = vi.fn();
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError });
|
||||
state.next.mockResolvedValue(null);
|
||||
const worker = registerProviderTraceRetentionWork(
|
||||
owner as unknown as Db,
|
||||
coordinator,
|
||||
() => true,
|
||||
);
|
||||
expect(state.probe!()).toBe("unknown");
|
||||
await worker.ready;
|
||||
expect(state.probe!()).toBe("none");
|
||||
expect(coordinator.nextWakeAt()).toBeNull();
|
||||
const initial = state.cleanup.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(300_000);
|
||||
expect(state.cleanup).toHaveBeenCalledTimes(initial);
|
||||
expect(state.next).toHaveBeenCalledOnce();
|
||||
|
||||
const deadline = Date.now() + 60_000;
|
||||
state.next.mockResolvedValue(deadline);
|
||||
worker.wake();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(state.probe!()).toBe("present");
|
||||
expect(coordinator.nextWakeAt()).toBe(deadline);
|
||||
const scheduled = state.cleanup.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(59_999);
|
||||
expect(state.cleanup).toHaveBeenCalledTimes(scheduled);
|
||||
state.next.mockResolvedValue(null);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(state.cleanup).toHaveBeenCalledTimes(scheduled + 1);
|
||||
expect(state.probe!()).toBe("none");
|
||||
expect(coordinator.nextWakeAt()).toBeNull();
|
||||
await worker.stop();
|
||||
expect(state.unregister).toHaveBeenCalledOnce();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { it } from "vitest";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
// Keep the dependency-free, actual-CLI acceptance suite usable on a retained
|
||||
// volume while also exercising the same cases in the normal CI test lane.
|
||||
it("passes the read-only recovery cutover CLI acceptance suite", async () => {
|
||||
await run(process.execPath, ["--test", fileURLToPath(new URL("../../../scripts/recovery-cutover.test.mjs", import.meta.url))], {
|
||||
timeout: 45_000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
}, 60_000);
|
||||
@@ -0,0 +1,168 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { registerRunEvidenceMigrationWork } from "../services/run-evidence-migration-work.js";
|
||||
|
||||
const state = vi.hoisted(() => ({
|
||||
provider: "s3",
|
||||
migrate: vi.fn(),
|
||||
probe: undefined as undefined | (() => string),
|
||||
unregister: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
}));
|
||||
vi.mock("../storage/config.js", () => ({
|
||||
loadStorageConfig: () => ({ storageProvider: state.provider }),
|
||||
}));
|
||||
vi.mock("../services/run-evidence-migration.js", () => ({ migrateRunEvidence: state.migrate }));
|
||||
vi.mock("../middleware/logger.js", () => ({ logger: { info: state.info, warn: state.warn } }));
|
||||
vi.mock("../services/idle-local-work.js", () => ({
|
||||
registerIdleLocalWorkProbe: (probe: () => string) => {
|
||||
state.probe = probe;
|
||||
return state.unregister;
|
||||
},
|
||||
}));
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop();
|
||||
coordinator = undefined;
|
||||
state.provider = "s3";
|
||||
state.probe = undefined;
|
||||
vi.resetAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
function setup(canRun = () => true) {
|
||||
vi.useFakeTimers();
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
|
||||
const onError = vi.fn();
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile: () => true, onError });
|
||||
const worker = registerRunEvidenceMigrationWork(owner as unknown as Db, coordinator, canRun);
|
||||
return { worker, onError };
|
||||
}
|
||||
it("does no migration work for local storage", async () => {
|
||||
state.provider = "local_disk";
|
||||
const { worker } = setup();
|
||||
expect(worker).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(3_600_000);
|
||||
expect(state.migrate).not.toHaveBeenCalled();
|
||||
expect(state.probe).toBeUndefined();
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("automatically pages both resources, then leaves completed history quiet", async () => {
|
||||
state.migrate
|
||||
.mockResolvedValueOnce({
|
||||
items: [{ kind: "log", id: "one", state: "migrated" }],
|
||||
nextRunId: "one",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
items: [{ kind: "log", id: "two", state: "migrated" }],
|
||||
nextRunId: "two",
|
||||
})
|
||||
.mockResolvedValue({ items: [] });
|
||||
const { worker, onError } = setup();
|
||||
expect(state.probe!()).toBe("unknown");
|
||||
await worker!.ready;
|
||||
expect(state.migrate).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
apply: true,
|
||||
limit: 1,
|
||||
kinds: ["log", "trace"],
|
||||
}),
|
||||
);
|
||||
expect(state.probe!()).toBe("present");
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
expect(state.migrate).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
afterRunId: "one",
|
||||
kinds: ["log"],
|
||||
}),
|
||||
);
|
||||
expect(state.migrate).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
afterRunId: "two",
|
||||
kinds: ["log"],
|
||||
}),
|
||||
);
|
||||
expect(state.probe!()).toBe("none");
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(3_600_000);
|
||||
expect(state.migrate).toHaveBeenCalledTimes(3);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
await worker!.stop();
|
||||
expect(state.unregister).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("keeps blocked history visible and retries it without stranding later pages", async () => {
|
||||
state.migrate
|
||||
.mockResolvedValueOnce({
|
||||
items: [{ kind: "log", id: "active", state: "blocked", reason: "legacy_writer_not_settled" }],
|
||||
nextRunId: "active",
|
||||
})
|
||||
.mockResolvedValueOnce({ items: [{ kind: "log", id: "later", state: "migrated" }] })
|
||||
.mockResolvedValue({ items: [{ kind: "log", id: "active", state: "migrated" }] });
|
||||
const { worker } = setup();
|
||||
await worker!.ready;
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(state.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: "active", reason: "legacy_writer_not_settled" }),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(state.probe!()).toBe("present");
|
||||
await vi.advanceTimersByTimeAsync(299_999);
|
||||
expect(state.migrate).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(state.migrate).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.anything(),
|
||||
expect.objectContaining({ afterRunId: undefined, kinds: ["log", "trace"] }),
|
||||
);
|
||||
expect(state.probe!()).toBe("none");
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("retains startup work during drain and retries a failed sweep before becoming quiet", async () => {
|
||||
let admitted = false;
|
||||
state.migrate
|
||||
.mockRejectedValueOnce(new Error("storage unavailable"))
|
||||
.mockResolvedValue({ items: [] });
|
||||
const { worker, onError } = setup(() => admitted);
|
||||
await worker!.ready;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(state.migrate).not.toHaveBeenCalled();
|
||||
expect(state.probe!()).toBe("unknown");
|
||||
admitted = true;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(onError).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(state.migrate).toHaveBeenCalledTimes(2);
|
||||
expect(state.probe!()).toBe("none");
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("aborts owned work on shutdown and reconstructs outstanding history on restart", async () => {
|
||||
let started: AbortSignal | undefined;
|
||||
state.migrate.mockImplementationOnce(
|
||||
(_db, options) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
started = options.signal;
|
||||
started!.addEventListener("abort", () => reject(started!.reason), { once: true });
|
||||
}),
|
||||
);
|
||||
const { worker } = setup();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(started?.aborted).toBe(false);
|
||||
await worker!.stop();
|
||||
await worker!.ready;
|
||||
expect(started?.aborted).toBe(true);
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
await coordinator!.stop();
|
||||
state.migrate.mockResolvedValue({ items: [] });
|
||||
const restarted = setup();
|
||||
await restarted.worker!.ready;
|
||||
expect(state.migrate).toHaveBeenCalledTimes(2);
|
||||
expect(state.probe!()).toBe("none");
|
||||
});
|
||||
@@ -20,6 +20,22 @@ import { gzipSync } from "node:zlib";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const cloudRuntimeStartupMocks = vi.hoisted(() => ({
|
||||
initialize: vi.fn(async () => null),
|
||||
bootstrap: vi.fn(),
|
||||
telemetry: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../services/adapter-settings.js", async () => ({
|
||||
...await vi.importActual<typeof import("../services/adapter-settings.js")>("../services/adapter-settings.js"),
|
||||
initializeAdapterSettings: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("../services/cloud-runtime-config.js", () => ({ initializeCloudRuntimeConfig: cloudRuntimeStartupMocks.initialize }));
|
||||
const cloudMigrationMocks = vi.hoisted(() => ({ migrate: vi.fn(async () => null), qualify: vi.fn(async () => null) }));
|
||||
vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudConfiguration: cloudMigrationMocks.migrate }));
|
||||
vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify }));
|
||||
vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) }));
|
||||
|
||||
const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL;
|
||||
const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL;
|
||||
const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON;
|
||||
@@ -343,6 +359,7 @@ vi.mock("../services/native-runtime/native-safe-replacement.js", () => ({
|
||||
|
||||
vi.mock("../config.js", () => ({
|
||||
loadConfig: loadConfigMock,
|
||||
loadCloudBootstrapConfig: cloudRuntimeStartupMocks.bootstrap,
|
||||
}));
|
||||
|
||||
vi.mock("../middleware/logger.js", () => ({
|
||||
@@ -586,6 +603,8 @@ describe("startServer feedback export wiring", () => {
|
||||
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
|
||||
process.env.PAPERCLIP_AGENT_JWT_SECRET = "0123456789abcdef0123456789abcdef";
|
||||
loadConfigMock.mockReturnValue(buildTestConfig());
|
||||
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
resolveHeartbeatSchedulingSuppressionMock.mockReturnValue({
|
||||
suppressed: false,
|
||||
reason: null,
|
||||
@@ -595,6 +614,64 @@ describe("startServer feedback export wiring", () => {
|
||||
process.env.BETTER_AUTH_SECRET = "test-secret";
|
||||
});
|
||||
|
||||
it("awaits saved Cloud configuration before telemetry, authentication and app construction", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
vi.stubEnv("HOST", "0.0.0.0");
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
let release: (() => void) | undefined;
|
||||
cloudRuntimeStartupMocks.initialize.mockImplementationOnce(() => new Promise<null>(resolve => {
|
||||
release = () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ telemetryEnabled: false, authDisableSignUp: true, port: 4321, host: "0.0.0.0", bind: "all" }));
|
||||
resolve(null);
|
||||
};
|
||||
}));
|
||||
try {
|
||||
const starting = startServer();
|
||||
for (let index = 0; index < 30 && !release; index += 1) await new Promise<void>(resolve => setImmediate(resolve));
|
||||
expect(release).toBeTypeOf("function");
|
||||
expect(cloudMigrationMocks.migrate.mock.invocationCallOrder[0]).toBeLessThan(cloudRuntimeStartupMocks.initialize.mock.invocationCallOrder[0]!);
|
||||
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
|
||||
expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
|
||||
expect(createAppMock).not.toHaveBeenCalled();
|
||||
expect(loadConfigMock).not.toHaveBeenCalled();
|
||||
release!();
|
||||
const started = await starting;
|
||||
await settleStartupHousekeeping();
|
||||
expect(cloudRuntimeStartupMocks.telemetry).toHaveBeenCalledExactlyOnceWith({ enabled: false });
|
||||
expect(createBetterAuthInstanceMock).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ authDisableSignUp: true, port: 4321 }), expect.anything());
|
||||
expect(started.listenPort).toBe(4321);
|
||||
expect(cloudMigrationMocks.qualify.mock.invocationCallOrder[0]).toBeLessThan(createAppMock.mock.invocationCallOrder[0]!);
|
||||
} finally {
|
||||
release?.();
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not create telemetry or the app when saved Cloud configuration fails", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
cloudRuntimeStartupMocks.initialize.mockRejectedValueOnce(new Error("persisted config unavailable"));
|
||||
try {
|
||||
await expect(startServer()).rejects.toThrow("persisted config unavailable");
|
||||
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
|
||||
expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
|
||||
expect(createAppMock).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("stops before configuration consumers when the atomic Cloud migration fails", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
cloudMigrationMocks.migrate.mockRejectedValueOnce(new Error("migration did not commit"));
|
||||
try {
|
||||
await expect(startServer()).rejects.toThrow("migration did not commit");
|
||||
expect(cloudRuntimeStartupMocks.initialize).not.toHaveBeenCalled();
|
||||
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
|
||||
expect(createAppMock).not.toHaveBeenCalled();
|
||||
} finally { vi.unstubAllEnvs(); }
|
||||
});
|
||||
|
||||
it.each([false, true])("registers runner evidence independently from the backup scan (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -620,6 +697,12 @@ describe("startServer feedback export wiring", () => {
|
||||
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.nativeRunnerEvidenceRetention)).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([false, true])("registers trace retention independently of heartbeat scheduling (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.providerTraceRetention)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([false, true])("registers skill cache retention only when enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ skillVersionRetentionEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -1509,6 +1592,8 @@ describe("startServer authenticated auth origin setup", () => {
|
||||
vi.clearAllMocks();
|
||||
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
|
||||
loadConfigMock.mockReturnValue(buildTestConfig());
|
||||
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
createBetterAuthInstanceMock.mockReturnValue({});
|
||||
deriveAuthTrustedOriginsMock.mockReturnValue([]);
|
||||
process.env.BETTER_AUTH_SECRET = "test-secret";
|
||||
@@ -1570,6 +1655,8 @@ describe("startServer PAPERCLIP_API_URL handling", () => {
|
||||
vi.clearAllMocks();
|
||||
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
|
||||
loadConfigMock.mockReturnValue(buildTestConfig());
|
||||
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
process.env.BETTER_AUTH_SECRET = "test-secret";
|
||||
delete process.env.PAPERCLIP_API_URL;
|
||||
});
|
||||
|
||||
@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { activityLog, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db";
|
||||
import { activityLog, agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
@@ -11,16 +11,25 @@ import { createHeartbeatRunState } from "../services/heartbeat/run-state.js";
|
||||
import { activityService } from "../services/activity.js";
|
||||
import { documentService } from "../services/documents.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabase = process.env.PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL;
|
||||
if (externalDatabase) {
|
||||
const target = new URL(externalDatabase);
|
||||
if (!["localhost", "127.0.0.1", "[::1]"].includes(target.hostname) || !target.pathname.endsWith("_test"))
|
||||
throw new Error("PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL must name a disposable loopback database ending in _test");
|
||||
}
|
||||
const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("task cold-open reads", () => {
|
||||
let fixture: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let fixture: Pick<Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, "connectionString" | "cleanup">;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
const companyId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
beforeAll(async () => {
|
||||
fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-");
|
||||
if (externalDatabase) {
|
||||
await applyPendingMigrations(externalDatabase);
|
||||
fixture = { connectionString: externalDatabase, cleanup: () => closeRegisteredClients(externalDatabase) };
|
||||
} else fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-");
|
||||
db = createDb(fixture.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Cold open", issuePrefix: `T${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Writer", role: "engineer", adapterType: "codex_local" });
|
||||
@@ -79,6 +88,39 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
expect(response.body.latestCommentId).toBe(response.body.comments[1].id);
|
||||
expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${issueId}/recent-comments`)).status);
|
||||
});
|
||||
it("opens long chats with a contiguous recent window and pages every older message", async () => {
|
||||
const chatId = randomUUID();
|
||||
await db.insert(issues).values({ id: chatId, companyId, title: "Persistent chat", conversationAgentId: agentId, conversationUserId: "local-board", assigneeAgentId: agentId, conversationState: "waiting" });
|
||||
const rows = Array.from({ length: 123 }, (_, i) => ({ id: randomUUID(), companyId, issueId: chatId,
|
||||
authorType: "user" as const, authorUserId: "local-board", body: `Chat ${i}`, createdAt: new Date(1000 + Math.floor(i / 3)) }));
|
||||
await db.insert(issueComments).values(rows);
|
||||
const ordered = [...rows].sort((a, b) => +b.createdAt - +a.createdAt || b.id.localeCompare(a.id));
|
||||
const response = await request(app()).get(`/api/issues/${chatId}/recent-comments`);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.comments.map((row: { id: string }) => row.id)).toEqual(ordered.slice(0, 20).map(row => row.id));
|
||||
expect(response.body.oldestCommentId).toBe(ordered[19].id);
|
||||
expect(response.body.hasMore).toBe(true);
|
||||
const seen = response.body.comments.map((row: { id: string }) => row.id);
|
||||
let cursor = response.body.oldestCommentId;
|
||||
while (cursor) {
|
||||
const page = await request(app()).get(`/api/issues/${chatId}/comments`).query({ order: "desc", limit: 50, after: cursor });
|
||||
expect(page.status).toBe(200);
|
||||
seen.push(...page.body.map((row: { id: string }) => row.id));
|
||||
cursor = page.body.at(-1)?.id;
|
||||
}
|
||||
expect(seen).toEqual(ordered.map(row => row.id));
|
||||
expect(new Set(seen).size).toBe(123);
|
||||
expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${chatId}/recent-comments`)).status);
|
||||
});
|
||||
it("hydrates only requested runs linked to the company and issue", async () => {
|
||||
const otherIssue = randomUUID(), otherRun = randomUUID();
|
||||
await db.insert(issues).values({ id: otherIssue, companyId, title: "Other task" });
|
||||
await db.insert(heartbeatRuns).values({ id: otherRun, companyId, agentId, issueId: otherIssue, status: "succeeded" });
|
||||
const activity = activityService(db);
|
||||
expect((await activity.runsForIssue(companyId, issueId, null, undefined, [runId, otherRun])).map(run => run.runId)).toEqual([runId]);
|
||||
expect(await activity.runsForIssue(randomUUID(), issueId, null, undefined, [runId])).toEqual([]);
|
||||
expect(await activity.runsForIssue(companyId, issueId, null, undefined, [])).toEqual([]);
|
||||
});
|
||||
it.each(["user", null] as const)("preserves initial and latest human instructions without a description (authorType=%s)", async (authorType) => {
|
||||
const bareIssueId = randomUUID();
|
||||
await db.insert(issues).values({ id: bareIssueId, companyId, title: "No description" });
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
let home: string;
|
||||
let localPackage: string;
|
||||
let npmPackage: string;
|
||||
let storePath: string;
|
||||
let sentinel: string;
|
||||
const parserSource = "export function parseStdoutLine() { return { message: 'external parser' }; }";
|
||||
const managedError = { status: 403, details: { code: "adapter_install_platform_managed" } };
|
||||
|
||||
function writePackage(packageDir: string, type: string): void {
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(packageDir, "package.json"), JSON.stringify({
|
||||
name: `fixture-${type}`,
|
||||
type: "module",
|
||||
exports: { ".": "./index.mjs", "./ui-parser": "./ui-parser.mjs" },
|
||||
paperclip: { adapterUiParser: "1.0.0" },
|
||||
}));
|
||||
fs.writeFileSync(path.join(packageDir, "index.mjs"), `
|
||||
import fs from "node:fs";
|
||||
fs.appendFileSync(${JSON.stringify(sentinel)}, ${JSON.stringify(type + "\n")});
|
||||
export function createServerAdapter() {
|
||||
return {
|
||||
type: ${JSON.stringify(type)},
|
||||
agentConfigurationDoc: "External fixture",
|
||||
models: [],
|
||||
execute: async () => ({ exitCode: 41, signal: null, timedOut: false }),
|
||||
testEnvironment: async () => ({ status: "pass", checks: [] }),
|
||||
};
|
||||
}
|
||||
`);
|
||||
fs.writeFileSync(path.join(packageDir, "ui-parser.mjs"), parserSource);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-cloud-adapters-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", home);
|
||||
sentinel = path.join(home, "external-code-executed");
|
||||
localPackage = path.join(home, "local-package");
|
||||
npmPackage = path.join(home, "adapter-plugins", "node_modules", "fixture-npm");
|
||||
storePath = path.join(home, "adapter-plugins.json");
|
||||
writePackage(localPackage, "codex_local");
|
||||
writePackage(npmPackage, "fixture_npm");
|
||||
fs.writeFileSync(storePath, JSON.stringify([
|
||||
{ packageName: "fixture-local", localPath: localPackage, type: "codex_local", installedAt: "2026-01-01" },
|
||||
{ packageName: "fixture-npm", type: "fixture_npm", installedAt: "2026-01-01" },
|
||||
]));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("Cloud image adapter boundary", () => {
|
||||
it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])(
|
||||
"cold boots built-ins without reading or executing persisted packages with %s",
|
||||
async (signal) => {
|
||||
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
|
||||
const registrationBefore = fs.readFileSync(storePath, "utf8");
|
||||
const reads = vi.spyOn(fs, "readFileSync");
|
||||
const registry = await import("./registry.js");
|
||||
await registry.waitForExternalAdapters();
|
||||
const loader = await import("./plugin-loader.js");
|
||||
const store = await import("../services/adapter-plugin-store.js");
|
||||
|
||||
expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
|
||||
expect(registry.requireServerAdapter("process").execute).toBeTypeOf("function");
|
||||
expect(registry.findServerAdapter("fixture_npm")).toBeNull();
|
||||
expect(registry.hasActiveAdapterOverride("codex_local")).toBe(false);
|
||||
expect(await loader.buildExternalAdapters()).toEqual([]);
|
||||
expect(store.listAdapterPlugins()).toEqual([]);
|
||||
expect(store.getAdapterPluginByType("codex_local")).toBeUndefined();
|
||||
expect(loader.getUiParserSource("codex_local")).toBeUndefined();
|
||||
expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
|
||||
await expect(loader.loadExternalAdapterPackage("fixture-local", localPackage)).rejects.toMatchObject(managedError);
|
||||
await expect(loader.loadExternalAdapterPackage("fixture-npm")).rejects.toMatchObject(managedError);
|
||||
await expect(loader.reloadExternalAdapter("codex_local")).rejects.toMatchObject(managedError);
|
||||
expect(() => registry.registerServerAdapter(registry.requireServerAdapter("codex_local"))).toThrow();
|
||||
expect(() => registry.unregisterServerAdapter("codex_local")).toThrow();
|
||||
expect(() => registry.setOverridePaused("codex_local", false)).toThrow();
|
||||
expect(() => store.addAdapterPlugin({ packageName: "fixture", type: "fixture", installedAt: "2026-01-01" })).toThrow();
|
||||
expect(() => store.removeAdapterPlugin("codex_local")).toThrow();
|
||||
expect(() => store.getAdapterPluginsDir()).toThrow();
|
||||
expect(store.getAdapterPluginStorePath()).toBe(storePath);
|
||||
expect(reads.mock.calls.map(([file]) => String(file)).filter(file =>
|
||||
file === storePath || file.startsWith(localPackage + path.sep) || file.startsWith(npmPackage + path.sep),
|
||||
)).toEqual([]);
|
||||
reads.mockRestore();
|
||||
expect(fs.existsSync(sentinel)).toBe(false);
|
||||
expect(fs.readFileSync(storePath, "utf8")).toBe(registrationBefore);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves local and npm package loading, overrides, reload, and lazy UI parsing on self-hosted instances", async () => {
|
||||
const loader = await import("./plugin-loader.js");
|
||||
expect(loader.getOrExtractUiParserSource("codex_local")).toBe(parserSource);
|
||||
const registry = await import("./registry.js");
|
||||
await registry.waitForExternalAdapters();
|
||||
|
||||
const override = registry.requireServerAdapter("codex_local");
|
||||
expect(override.agentConfigurationDoc).toBe("External fixture");
|
||||
expect(registry.hasActiveAdapterOverride("codex_local")).toBe(true);
|
||||
expect(registry.requireServerAdapter("fixture_npm").agentConfigurationDoc).toBe("External fixture");
|
||||
expect(await override.execute({} as never)).toMatchObject({ exitCode: 41 });
|
||||
expect(fs.readFileSync(sentinel, "utf8").trim().split("\n")).toEqual(["codex_local", "fixture_npm"]);
|
||||
expect(loader.getUiParserSource("fixture_npm")).toBe(parserSource);
|
||||
const reloaded = await loader.reloadExternalAdapter("codex_local");
|
||||
expect(reloaded?.type).toBe("codex_local");
|
||||
expect(await reloaded?.execute({} as never)).toMatchObject({ exitCode: 41 });
|
||||
expect(registry.setOverridePaused("codex_local", true)).toBe(true);
|
||||
expect(registry.findActiveServerAdapter("codex_local")?.agentConfigurationDoc).not.toBe("External fixture");
|
||||
expect(registry.setOverridePaused("codex_local", false)).toBe(true);
|
||||
registry.unregisterServerAdapter("codex_local");
|
||||
expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
|
||||
});
|
||||
|
||||
it("does not expose a cached registration or parser after a managed signal becomes present", async () => {
|
||||
const loader = await import("./plugin-loader.js");
|
||||
const store = await import("../services/adapter-plugin-store.js");
|
||||
await loader.loadExternalAdapterPackage("fixture-local", localPackage);
|
||||
expect(loader.getUiParserSource("codex_local")).toBe(parserSource);
|
||||
expect(store.listAdapterPlugins()).toHaveLength(2);
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", "{}");
|
||||
expect(store.listAdapterPlugins()).toEqual([]);
|
||||
expect(loader.getUiParserSource("codex_local")).toBeUndefined();
|
||||
expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -15,6 +15,8 @@ import { pathToFileURL } from "node:url";
|
||||
import type { ServerAdapterModule } from "./types.js";
|
||||
import { validateAdapterLoginCapability } from "@paperclipai/adapter-utils";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
|
||||
|
||||
import {
|
||||
listAdapterPlugins,
|
||||
@@ -30,6 +32,7 @@ import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
|
||||
const uiParserCache = new Map<string, string>();
|
||||
|
||||
export function getUiParserSource(adapterType: string): string | undefined {
|
||||
if (isCloudManagedInstance()) return undefined;
|
||||
return uiParserCache.get(adapterType);
|
||||
}
|
||||
|
||||
@@ -38,6 +41,7 @@ export function getUiParserSource(adapterType: string): string | undefined {
|
||||
* Makes the ui-parser.js endpoint self-healing.
|
||||
*/
|
||||
export function getOrExtractUiParserSource(adapterType: string): string | undefined {
|
||||
if (isCloudManagedInstance()) return undefined;
|
||||
const cached = uiParserCache.get(adapterType);
|
||||
if (cached) return cached;
|
||||
|
||||
@@ -182,6 +186,7 @@ export async function loadExternalAdapterPackage(
|
||||
packageName: string,
|
||||
localPath?: string,
|
||||
): Promise<ServerAdapterModule> {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const packageDir = localPath
|
||||
? path.resolve(localPath)
|
||||
: path.resolve(getAdapterPluginsDir(), "node_modules", packageName);
|
||||
@@ -221,6 +226,7 @@ async function loadFromRecord(record: AdapterPluginRecord): Promise<ServerAdapte
|
||||
export async function reloadExternalAdapter(
|
||||
type: string,
|
||||
): Promise<ServerAdapterModule | null> {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const record = getAdapterPluginByType(type);
|
||||
if (!record) return null;
|
||||
|
||||
@@ -271,6 +277,7 @@ export async function reloadExternalAdapter(
|
||||
* Build all external adapter modules from the plugin store.
|
||||
*/
|
||||
export async function buildExternalAdapters(): Promise<ServerAdapterModule[]> {
|
||||
if (isCloudManagedInstance()) return [];
|
||||
const results: ServerAdapterModule[] = [];
|
||||
|
||||
const storeRecords = listAdapterPlugins();
|
||||
|
||||
@@ -136,7 +136,8 @@ import {
|
||||
import { agentConfigurationDoc as piAgentConfigurationDoc } from "@paperclipai/adapter-pi-local";
|
||||
import { BUILTIN_ADAPTER_TYPES } from "./builtin-adapter-types.js";
|
||||
import { buildExternalAdapters } from "./plugin-loader.js";
|
||||
import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
|
||||
import { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
|
||||
import { NativeEnvironmentProbeCleanupError } from "../vendor/paperclip-runner/index.js";
|
||||
import { processAdapter } from "./process/index.js";
|
||||
import { httpAdapter } from "./http/index.js";
|
||||
@@ -997,11 +998,6 @@ registerBuiltInAdapters();
|
||||
// applies the registry fallback for externals overriding a built-in type.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Cached sync wrapper — the store is a simple JSON file read, safe to call frequently. */
|
||||
function getDisabledAdapterTypesFromStore(): string[] {
|
||||
return getDisabledAdapterTypes();
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge an external adapter module with host-provided session management.
|
||||
*
|
||||
@@ -1034,6 +1030,7 @@ export function resolveExternalAdapterRegistration(
|
||||
* and avoid racing against the loading window.
|
||||
*/
|
||||
const externalAdaptersReady: Promise<void> = (async () => {
|
||||
if (isCloudManagedInstance()) return;
|
||||
try {
|
||||
const externalAdapters = await buildExternalAdapters();
|
||||
for (const externalAdapter of externalAdapters) {
|
||||
@@ -1069,6 +1066,7 @@ export function waitForExternalAdapters(): Promise<void> {
|
||||
}
|
||||
|
||||
export function registerServerAdapter(adapter: ServerAdapterModule): void {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
if (BUILTIN_ADAPTER_TYPES.has(adapter.type) && !builtinFallbacks.has(adapter.type)) {
|
||||
const existing = adaptersByType.get(adapter.type);
|
||||
if (existing) {
|
||||
@@ -1079,6 +1077,7 @@ export function registerServerAdapter(adapter: ServerAdapterModule): void {
|
||||
}
|
||||
|
||||
export function unregisterServerAdapter(type: string): void {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
if (type === processAdapter.type || type === httpAdapter.type) return;
|
||||
if (builtinFallbacks.has(type)) {
|
||||
pausedOverrides.delete(type);
|
||||
@@ -1186,8 +1185,7 @@ export function listServerAdapters(): ServerAdapterModule[] {
|
||||
* Used for menus and agent creation flows — disabled adapters remain
|
||||
* functional for existing agents but hidden from selection.
|
||||
*/
|
||||
export function listEnabledServerAdapters(): ServerAdapterModule[] {
|
||||
const disabled = getDisabledAdapterTypesFromStore();
|
||||
export function listEnabledServerAdapters(disabled: string[]): ServerAdapterModule[] {
|
||||
const disabledSet = disabled.length > 0 ? new Set(disabled) : null;
|
||||
return disabledSet
|
||||
? Array.from(adaptersByType.values()).filter((a) => !disabledSet.has(a.type))
|
||||
@@ -1227,6 +1225,7 @@ export async function detectAdapterModel(
|
||||
* an override or was already in the requested state.
|
||||
*/
|
||||
export function setOverridePaused(type: string, paused: boolean): boolean {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
if (!builtinFallbacks.has(type)) return false;
|
||||
const wasPaused = pausedOverrides.has(type);
|
||||
if (paused && !wasPaused) {
|
||||
|
||||
+13
-4
@@ -183,7 +183,9 @@ import {
|
||||
ensureBundledPlugins,
|
||||
resolveBundledCatalogRoot,
|
||||
resolveBundledPluginInstalls,
|
||||
resolveManagedBundledPluginKeys,
|
||||
} from "./services/bundled-plugins.js";
|
||||
import { createBundledPluginPackageResolver } from "./services/bundled-plugin-package.js";
|
||||
import { readDistributionPluginCatalog, distributionPluginActivationGuard } from "./services/distribution-plugin-catalog.js";
|
||||
import {
|
||||
createPluginWorkerManager,
|
||||
@@ -623,7 +625,7 @@ export async function createApp(
|
||||
res.set("Cache-Control", "no-store"); res.set("Referrer-Policy", "no-referrer");
|
||||
res.redirect(303, await chatChannels.githubWizard.identityCallback(String(req.query.state ?? ""), String(req.query.code ?? ""), req.actor));
|
||||
});
|
||||
const managedAutoInstallKeys = opts.managedPluginAutoInstall ?? null;
|
||||
const managedAutoInstallKeys = resolveManagedBundledPluginKeys(opts.managedPluginAutoInstall);
|
||||
const bundledCatalogRoot =
|
||||
opts.bundledPluginCatalogRoot ?? resolveBundledCatalogRoot(process.env);
|
||||
const distributionPlugins = readDistributionPluginCatalog(bundledCatalogRoot, BUNDLED_PLUGIN_CATALOG);
|
||||
@@ -636,6 +638,7 @@ export async function createApp(
|
||||
distributionPlugins,
|
||||
},
|
||||
);
|
||||
const bundledPackageResolver = managedAutoInstallKeys === null ? undefined : createBundledPluginPackageResolver(bundledPluginInstalls);
|
||||
const managedBundledPluginKeys =
|
||||
managedAutoInstallKeys !== null
|
||||
? bundledPluginInstalls.map((install) => install.pluginKey)
|
||||
@@ -906,7 +909,11 @@ export async function createApp(
|
||||
{
|
||||
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
|
||||
migrationDb: opts.pluginMigrationDb,
|
||||
assertPackageActivation: distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
|
||||
assertPackageActivation: bundledPackageResolver?.assertActivation ?? distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
|
||||
resolvePackageRoot: bundledPackageResolver?.resolve,
|
||||
prebuiltOnly: managedAutoInstallKeys !== null,
|
||||
enableLocalFilesystem: managedAutoInstallKeys === null,
|
||||
enableNpmDiscovery: managedAutoInstallKeys === null,
|
||||
},
|
||||
{
|
||||
workerManager,
|
||||
@@ -965,6 +972,7 @@ export async function createApp(
|
||||
);
|
||||
api.use(
|
||||
adapterRoutes({
|
||||
db,
|
||||
resolveRunnerTarget: (companyId, environmentId) => resolveAgentRunnerTargetForCompany(db, companyId, environmentId),
|
||||
getNativeRunnerEnabled: async () =>
|
||||
(await instanceSettingsService(db).getExperimental())
|
||||
@@ -1014,6 +1022,7 @@ export async function createApp(
|
||||
app.use(
|
||||
pluginUiStaticRoutes(db, {
|
||||
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
|
||||
resolvePackageRoot: bundledPackageResolver?.resolve,
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -1253,7 +1262,7 @@ export async function createApp(
|
||||
void toolDispatcher.initialize().catch((err) => {
|
||||
logger.error({ err }, "Failed to initialize plugin tool dispatcher");
|
||||
});
|
||||
const devWatcher = createPluginDevWatcher(
|
||||
const devWatcher = managedAutoInstallKeys !== null ? null : createPluginDevWatcher(
|
||||
lifecycle,
|
||||
async (pluginId) =>
|
||||
(await pluginRegistry.getById(pluginId))?.packagePath ?? null,
|
||||
@@ -1299,7 +1308,7 @@ export async function createApp(
|
||||
// owns provisioning); self-hosted leaves an operator's uninstall alone.
|
||||
// Disabled plugins never start automatically. Added distribution permissions
|
||||
// still enter upgrade_pending so enabling them requires an operator decision.
|
||||
{ reinstallUninstalled: managedAutoInstallKeys !== null },
|
||||
{ reinstallUninstalled: managedAutoInstallKeys !== null, preservePackagePath: managedAutoInstallKeys !== null },
|
||||
)
|
||||
.then(() => loader.loadAll())
|
||||
.then((result) => {
|
||||
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
} from "@paperclipai/shared";
|
||||
import { ZodError } from "zod";
|
||||
import { resolvePaperclipConfigPath } from "./paths.js";
|
||||
import { isCloudManagedInstance } from "./services/cloud-instance.js";
|
||||
import { getCloudRuntimeConfigSnapshot, toPaperclipConfig } from "./services/cloud-runtime-config-snapshot.js";
|
||||
|
||||
function formatConfigValidationError(error: ZodError): string {
|
||||
return error.issues
|
||||
@@ -19,6 +21,7 @@ function formatConfigValidationError(error: ZodError): string {
|
||||
}
|
||||
|
||||
export function readConfigFile(): PaperclipConfig | null {
|
||||
if (isCloudManagedInstance()) return toPaperclipConfig(getCloudRuntimeConfigSnapshot().config);
|
||||
const configPath = resolvePaperclipConfigPath();
|
||||
|
||||
if (!fs.existsSync(configPath)) return null;
|
||||
|
||||
+50
-60
@@ -1,4 +1,7 @@
|
||||
import { readConfigFile, resolveDeploymentMode } from "./config-file.js";
|
||||
import { isCloudManagedInstance } from "./services/cloud-instance.js";
|
||||
import { StartupRefusalError } from "./startup-refusals.js";
|
||||
import type { PaperclipConfig } from "@paperclipai/shared";
|
||||
import { parseChatWebhookPublicBaseUrl } from "./chat-webhook-public-url.js";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, realpathSync } from "node:fs";
|
||||
@@ -13,13 +16,11 @@ import {
|
||||
BIND_MODES,
|
||||
DEPLOYMENT_EXPOSURES,
|
||||
SECRET_PROVIDERS,
|
||||
STORAGE_PROVIDERS,
|
||||
type BindMode,
|
||||
type AuthBaseUrlMode,
|
||||
type DeploymentExposure,
|
||||
type DeploymentMode,
|
||||
type SecretProvider,
|
||||
type StorageProvider,
|
||||
inferBindModeFromHost,
|
||||
resolveRuntimeBind,
|
||||
validateConfiguredBindMode,
|
||||
@@ -28,37 +29,39 @@ import {
|
||||
resolveDefaultBackupDir,
|
||||
resolveDefaultEmbeddedPostgresDir,
|
||||
resolveDefaultSecretsKeyFilePath,
|
||||
resolveDefaultStorageDir,
|
||||
resolveHomeAwarePath,
|
||||
} from "./home-paths.js";
|
||||
import { loadStorageConfig, type StorageConfig } from "./storage/config.js";
|
||||
|
||||
const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
|
||||
if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
|
||||
loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
|
||||
}
|
||||
if (!isCloudManagedInstance()) {
|
||||
const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
|
||||
if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
|
||||
loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
|
||||
}
|
||||
|
||||
const CWD_ENV_PATH = resolve(process.cwd(), ".env");
|
||||
const cwdEnvExists = existsSync(CWD_ENV_PATH);
|
||||
const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
: CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
|
||||
if (shouldLoadWorkingDirectoryEnv({
|
||||
cwdEnvExists,
|
||||
isPaperclipEnvFile: isSameFile,
|
||||
})) {
|
||||
loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
|
||||
}
|
||||
const CWD_ENV_PATH = resolve(process.cwd(), ".env");
|
||||
const cwdEnvExists = existsSync(CWD_ENV_PATH);
|
||||
const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
: CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
|
||||
if (shouldLoadWorkingDirectoryEnv({
|
||||
cwdEnvExists,
|
||||
isPaperclipEnvFile: isSameFile,
|
||||
})) {
|
||||
loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
|
||||
}
|
||||
|
||||
if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
|
||||
applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
|
||||
if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
|
||||
applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
|
||||
}
|
||||
maybeRepairLegacyWorktreeConfigAndEnvFiles();
|
||||
}
|
||||
maybeRepairLegacyWorktreeConfigAndEnvFiles();
|
||||
|
||||
const TAILSCALE_DETECT_TIMEOUT_MS = 3000;
|
||||
|
||||
type DatabaseMode = "embedded-postgres" | "postgres";
|
||||
|
||||
export interface Config {
|
||||
export interface Config extends StorageConfig {
|
||||
deploymentMode: DeploymentMode;
|
||||
deploymentExposure: DeploymentExposure;
|
||||
bind: BindMode;
|
||||
@@ -111,13 +114,6 @@ export interface Config {
|
||||
secretsProvider: SecretProvider;
|
||||
secretsStrictMode: boolean;
|
||||
secretsMasterKeyFilePath: string;
|
||||
storageProvider: StorageProvider;
|
||||
storageLocalDiskBaseDir: string;
|
||||
storageS3Bucket: string;
|
||||
storageS3Region: string;
|
||||
storageS3Endpoint: string | undefined;
|
||||
storageS3Prefix: string;
|
||||
storageS3ForcePathStyle: boolean;
|
||||
feedbackExportBackendUrl: string | undefined;
|
||||
feedbackExportBackendToken: string | undefined;
|
||||
heartbeatSchedulerEnabled: boolean;
|
||||
@@ -147,18 +143,16 @@ function detectTailnetBindHost(): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
export function loadConfig(): Config {
|
||||
const fileConfig = readConfigFile();
|
||||
export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()): Config {
|
||||
const fileDatabaseMode =
|
||||
(fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
|
||||
(isCloudManagedInstance() || fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
|
||||
|
||||
const fileDbUrl =
|
||||
fileDatabaseMode === "postgres"
|
||||
!isCloudManagedInstance() && fileDatabaseMode === "postgres"
|
||||
? fileConfig?.database.connectionString
|
||||
: undefined;
|
||||
const fileDatabaseBackup = fileConfig?.database.backup;
|
||||
const fileSecrets = fileConfig?.secrets;
|
||||
const fileStorage = fileConfig?.storage;
|
||||
|
||||
const providerFromEnvRaw = process.env.PAPERCLIP_SECRETS_PROVIDER;
|
||||
const providerFromEnv =
|
||||
@@ -168,25 +162,7 @@ export function loadConfig(): Config {
|
||||
const providerFromFile = fileSecrets?.provider;
|
||||
const secretsProvider: SecretProvider = providerFromEnv ?? providerFromFile ?? "local_encrypted";
|
||||
|
||||
const storageProviderFromEnvRaw = process.env.PAPERCLIP_STORAGE_PROVIDER;
|
||||
const storageProviderFromEnv =
|
||||
storageProviderFromEnvRaw && STORAGE_PROVIDERS.includes(storageProviderFromEnvRaw as StorageProvider)
|
||||
? (storageProviderFromEnvRaw as StorageProvider)
|
||||
: null;
|
||||
const storageProvider: StorageProvider = storageProviderFromEnv ?? fileStorage?.provider ?? "local_disk";
|
||||
const storageLocalDiskBaseDir = resolveHomeAwarePath(
|
||||
process.env.PAPERCLIP_STORAGE_LOCAL_DIR ??
|
||||
fileStorage?.localDisk?.baseDir ??
|
||||
resolveDefaultStorageDir(),
|
||||
);
|
||||
const storageS3Bucket = process.env.PAPERCLIP_STORAGE_S3_BUCKET ?? fileStorage?.s3?.bucket ?? "paperclip";
|
||||
const storageS3Region = process.env.PAPERCLIP_STORAGE_S3_REGION ?? fileStorage?.s3?.region ?? "us-east-1";
|
||||
const storageS3Endpoint = process.env.PAPERCLIP_STORAGE_S3_ENDPOINT ?? fileStorage?.s3?.endpoint ?? undefined;
|
||||
const storageS3Prefix = process.env.PAPERCLIP_STORAGE_S3_PREFIX ?? fileStorage?.s3?.prefix ?? "";
|
||||
const storageS3ForcePathStyle =
|
||||
process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE !== undefined
|
||||
? process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE === "true"
|
||||
: (fileStorage?.s3?.forcePathStyle ?? false);
|
||||
const storage = loadStorageConfig(fileConfig);
|
||||
const feedbackExportBackendUrl =
|
||||
process.env.PAPERCLIP_FEEDBACK_EXPORT_BACKEND_URL?.trim() ||
|
||||
process.env.PAPERCLIP_TELEMETRY_BACKEND_URL?.trim() ||
|
||||
@@ -443,13 +419,7 @@ export function loadConfig(): Config {
|
||||
fileSecrets?.localEncrypted.keyFilePath ??
|
||||
resolveDefaultSecretsKeyFilePath(),
|
||||
),
|
||||
storageProvider,
|
||||
storageLocalDiskBaseDir,
|
||||
storageS3Bucket,
|
||||
storageS3Region,
|
||||
storageS3Endpoint,
|
||||
storageS3Prefix,
|
||||
storageS3ForcePathStyle,
|
||||
...storage,
|
||||
feedbackExportBackendUrl,
|
||||
feedbackExportBackendToken,
|
||||
heartbeatSchedulerEnabled: process.env.HEARTBEAT_SCHEDULER_ENABLED !== "false",
|
||||
@@ -460,3 +430,23 @@ export function loadConfig(): Config {
|
||||
announcementsFeedUrl: process.env.PAPERCLIP_ANNOUNCEMENTS_FEED_URL?.trim() || "https://pages.paperclip.ing/announcements/v1/current.json",
|
||||
};
|
||||
}
|
||||
|
||||
export type CloudBootstrapConfig = {
|
||||
databaseMode: "postgres";
|
||||
databaseUrl: string;
|
||||
databaseMigrationUrl: string | undefined;
|
||||
};
|
||||
|
||||
export function loadCloudBootstrapConfig(): CloudBootstrapConfig {
|
||||
if (!isCloudManagedInstance()) throw new Error("Cloud bootstrap configuration requires a managed instance");
|
||||
const url = process.env.DATABASE_URL?.trim();
|
||||
if (!url) throw new StartupRefusalError("database-contract-unmet", "Cloud bootstrap requires DATABASE_URL from the environment");
|
||||
const migrationUrl = process.env.DATABASE_MIGRATION_URL?.trim() || undefined;
|
||||
for (const [name, value] of [["DATABASE_URL", url], ["DATABASE_MIGRATION_URL", migrationUrl]]) {
|
||||
if (!value) continue;
|
||||
let parsed: URL;
|
||||
try { parsed = new URL(value); } catch { throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); }
|
||||
if (!["postgres:", "postgresql:"].includes(parsed.protocol)) throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`);
|
||||
}
|
||||
return { databaseMode: "postgres", databaseUrl: url, databaseMigrationUrl: migrationUrl };
|
||||
}
|
||||
+53
-21
@@ -1,3 +1,5 @@
|
||||
import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js";
|
||||
import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js";
|
||||
import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js";
|
||||
import { registerDatabaseBackupWork } from "./services/database-backup-work.js";
|
||||
import { registerToolConnectionHealthWork } from "./services/tool-connection-health-work.js";
|
||||
@@ -104,7 +106,8 @@ import {
|
||||
} from "@paperclipai/db";
|
||||
import detectPort from "detect-port";
|
||||
import { createApp } from "./app.js";
|
||||
import { loadConfig } from "./config.js";
|
||||
import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
|
||||
import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
|
||||
import { logger } from "./middleware/logger.js";
|
||||
import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
|
||||
import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
|
||||
@@ -186,6 +189,9 @@ import {
|
||||
loadWithoutCoordinatedShutdownSignalHooks,
|
||||
} from "./shutdown.js";
|
||||
import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js";
|
||||
import { initializeAdapterSettings } from "./services/adapter-settings.js";
|
||||
import { migrateCloudConfiguration } from "./services/cloud-configuration-migration.js";
|
||||
import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
|
||||
import { systemdNotify } from "./services/systemd-notify.js";
|
||||
import { flushInFlightRunLogMirrors } from "./services/run-log-store.js";
|
||||
import {
|
||||
@@ -251,6 +257,19 @@ async function endDatabaseClient(client: unknown, timeoutSeconds: number): Promi
|
||||
await sql.end({ timeout: timeoutSeconds });
|
||||
}
|
||||
|
||||
function initializeRuntimeConfigConsumers(config: ReturnType<typeof loadConfig>): void {
|
||||
initTelemetry({ enabled: config.telemetryEnabled });
|
||||
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
|
||||
}
|
||||
}
|
||||
|
||||
export async function startServer(): Promise<StartedServer> {
|
||||
const startupDatabase: StartupDatabaseTeardown = { close: null };
|
||||
try {
|
||||
@@ -278,17 +297,8 @@ async function startServerWithDatabaseTeardown(
|
||||
// sentry.ts.
|
||||
await sentryReady;
|
||||
ensureDecisionSigningSecret();
|
||||
let config = loadConfig();
|
||||
initTelemetry({ enabled: config.telemetryEnabled });
|
||||
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
|
||||
}
|
||||
const startupConfig = isCloudManagedInstance() ? loadCloudBootstrapConfig() : loadConfig();
|
||||
if ("deploymentMode" in startupConfig) initializeRuntimeConfigConsumers(startupConfig);
|
||||
|
||||
type MigrationSummary =
|
||||
| "skipped"
|
||||
@@ -390,7 +400,7 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
}
|
||||
|
||||
function assertCloudDatabaseContract(): void {
|
||||
function assertCloudDatabaseContract(config: Config): void {
|
||||
if (config.deploymentMode !== "authenticated" || config.deploymentExposure !== "public") {
|
||||
return;
|
||||
}
|
||||
@@ -483,17 +493,19 @@ async function startServerWithDatabaseTeardown(
|
||||
let startupDbInfo:
|
||||
| { mode: "external-postgres"; connectionString: string }
|
||||
| { mode: "embedded-postgres"; dataDir: string; port: number };
|
||||
assertCloudDatabaseContract();
|
||||
if (config.databaseUrl) {
|
||||
const migrationUrl = config.databaseMigrationUrl ?? config.databaseUrl;
|
||||
if ("deploymentMode" in startupConfig) assertCloudDatabaseContract(startupConfig);
|
||||
if (startupConfig.databaseUrl) {
|
||||
const migrationUrl = startupConfig.databaseMigrationUrl ?? startupConfig.databaseUrl;
|
||||
migrationSummary = await ensureMigrations(migrationUrl, "PostgreSQL");
|
||||
|
||||
db = createDb(config.databaseUrl);
|
||||
pluginMigrationDb = config.databaseMigrationUrl ? createDb(config.databaseMigrationUrl) : db;
|
||||
|
||||
db = createDb(startupConfig.databaseUrl);
|
||||
pluginMigrationDb = startupConfig.databaseMigrationUrl ? createDb(startupConfig.databaseMigrationUrl) : db;
|
||||
logger.info("Using external PostgreSQL via DATABASE_URL/config");
|
||||
activeDatabaseConnectionString = config.databaseUrl;
|
||||
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
|
||||
activeDatabaseConnectionString = startupConfig.databaseUrl;
|
||||
startupDbInfo = { mode: "external-postgres", connectionString: startupConfig.databaseUrl };
|
||||
} else {
|
||||
if (!("deploymentMode" in startupConfig)) throw new Error("Cloud bootstrap requires an external PostgreSQL connection");
|
||||
const config = startupConfig;
|
||||
const moduleName = "embedded-postgres";
|
||||
let EmbeddedPostgres: EmbeddedPostgresCtor;
|
||||
try {
|
||||
@@ -711,12 +723,19 @@ async function startServerWithDatabaseTeardown(
|
||||
};
|
||||
startupDatabase.close = closeDatabaseClients;
|
||||
setSentryRunLinkResolver(createSentryRunLinkResolver(db));
|
||||
await migrateCloudConfiguration(db as any);
|
||||
await initializeAdapterSettings(db as any);
|
||||
|
||||
await initializeCloudRuntimeConfig(db as any);
|
||||
let config = "deploymentMode" in startupConfig ? startupConfig : loadConfig();
|
||||
|
||||
// A claimed warm-pool stack may restart while its provider environment still
|
||||
// names the pool host. Restore the signed, durable identity before Better
|
||||
// Auth, routes, or child-runtime configuration capture any public URL.
|
||||
const restoredCloudRuntimeIdentity = await initializeCloudRuntimeIdentity(db as any);
|
||||
if (restoredCloudRuntimeIdentity) config = loadConfig();
|
||||
if (isCloudManagedInstance()) initializeRuntimeConfigConsumers(config);
|
||||
|
||||
const isWarmStandby = await createCloudWarmStandby(db as any);
|
||||
|
||||
if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) {
|
||||
@@ -963,6 +982,10 @@ async function startServerWithDatabaseTeardown(
|
||||
// document parsed fail-closed above (`plugins.autoInstall`). Absent env means
|
||||
// self-hosted: createApp falls back to its built-in kubernetes-only default.
|
||||
const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null;
|
||||
const cloudQualification = await qualifyCloudConfigurationForCurrentImage(db as any);
|
||||
if (cloudQualification && !cloudQualification.configurationPackagesEligible) {
|
||||
logger.warn({ scope: cloudQualification.scope, blockerCodes: cloudQualification.blockers.map((blocker) => blocker.code) }, "Cloud configuration and package qualification has blockers");
|
||||
}
|
||||
const app = await createApp(db as any, {
|
||||
prepareIdleDatabaseBackup: idleDatabaseBackupEnabled
|
||||
? async () => (await runServerDatabaseBackup("idle")) !== null
|
||||
@@ -1344,6 +1367,13 @@ async function startServerWithDatabaseTeardown(
|
||||
}));
|
||||
});
|
||||
server.on("close", () => { void taskDrainExpiryWork.stop(); });
|
||||
const runEvidenceMigrationWork = registerRunEvidenceMigrationWork(db, deliveryWork,
|
||||
() => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(), { storageProvider: config.storageProvider });
|
||||
if (runEvidenceMigrationWork) trackHeartbeatSchedulerWork(runEvidenceMigrationWork.ready);
|
||||
const providerTraceRetentionWork = registerProviderTraceRetentionWork(db, deliveryWork,
|
||||
() => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby());
|
||||
trackHeartbeatSchedulerWork(providerTraceRetentionWork.ready);
|
||||
server.on("close", providerTraceRetentionWork.stop);
|
||||
const executionSchedule = executionControlSchedule(db);
|
||||
const workspaceRecoveryRuntime = environmentRuntimeService(db, { pluginWorkerManager });
|
||||
const nativeRunnerEvidenceRetentionWorker = config.nativeHarnessBackupExpirationEnabled
|
||||
@@ -2215,6 +2245,8 @@ async function startServerWithDatabaseTeardown(
|
||||
unsubscribeNativeWorkspaceFinalization?.();
|
||||
unsubscribeRetainedNativeSessionCleanup?.();
|
||||
await deliveryWork.stop();
|
||||
await runEvidenceMigrationWork?.stop();
|
||||
await providerTraceRetentionWork.stop();
|
||||
await skillVersionRetentionWorker?.dispose();
|
||||
await workspaceRecoveryWork.stop();
|
||||
unregisterTerminalWorkspaceProbe?.();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolveDefaultConfigPath } from "./home-paths.js";
|
||||
import { isCloudManagedInstance } from "./services/cloud-instance.js";
|
||||
|
||||
const PAPERCLIP_CONFIG_BASENAME = "config.json";
|
||||
const PAPERCLIP_ENV_FILENAME = ".env";
|
||||
@@ -26,6 +27,7 @@ function findConfigFileFromAncestors(startDir: string): string | null {
|
||||
export function resolvePaperclipConfigPath(overridePath?: string): string {
|
||||
if (overridePath) return path.resolve(overridePath);
|
||||
if (process.env.PAPERCLIP_CONFIG) return path.resolve(process.env.PAPERCLIP_CONFIG);
|
||||
if (isCloudManagedInstance()) return resolveDefaultConfigPath();
|
||||
return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { createDb } from "@paperclipai/db";
|
||||
import { cloudConfigurationMigrationService } from "./services/cloud-configuration-migration.js";
|
||||
import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
|
||||
|
||||
export async function runCloudConfigurationQualification(args: string[], env: NodeJS.ProcessEnv = process.env): Promise<number> {
|
||||
const allowed = new Set(["--migrate", "--expected-package-evidence", "--recheck-external-revision"]);
|
||||
let migrate = false;
|
||||
let expectedPackageEvidenceDigest: string | undefined;
|
||||
let recheckRevision: number | undefined;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
if (!allowed.has(argument)) throw new Error(`Unknown qualification argument: ${argument}`);
|
||||
if (argument === "--migrate") migrate = true;
|
||||
if (argument === "--expected-package-evidence") {
|
||||
expectedPackageEvidenceDigest = args[++index];
|
||||
if (!expectedPackageEvidenceDigest || !/^sha256:[a-f0-9]{64}$/.test(expectedPackageEvidenceDigest)) throw new Error("A SHA-256 package evidence digest is required");
|
||||
}
|
||||
if (argument === "--recheck-external-revision") {
|
||||
const value = args[++index];
|
||||
if (!value || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error("A nonnegative receipt revision is required");
|
||||
recheckRevision = Number(value);
|
||||
}
|
||||
}
|
||||
const url = env.DATABASE_URL?.trim();
|
||||
if (!url) throw new Error("DATABASE_URL is required for Cloud configuration qualification");
|
||||
const db = createDb(url);
|
||||
try {
|
||||
const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
|
||||
if (migrate) await service.migrate();
|
||||
if (recheckRevision !== undefined) await service.recheckLegacyExternalRegistrations(recheckRevision);
|
||||
const report = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv: env, expectedPackageEvidenceDigest });
|
||||
if (!report) throw new Error("Cloud configuration qualification requires a managed instance");
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
return report.configurationPackagesEligible && (!expectedPackageEvidenceDigest || report.samePackageOverlapEligible) ? 0 : 2;
|
||||
} finally {
|
||||
await db.$client.end({ timeout: 5 });
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
||||
runCloudConfigurationQualification(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : "Cloud configuration qualification failed"}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -373,6 +373,14 @@ export function activityRoutes(db: Db) {
|
||||
if (!issue) return;
|
||||
if (!(await assertIssueReadAllowed(req, res, issue))) return;
|
||||
const viewerUserId = req.actor.type === "board" ? getActorInfo(req).actorId : null;
|
||||
if (req.query.runIds !== undefined) {
|
||||
const parsed = z.array(z.string().uuid()).min(1).max(100).safeParse(
|
||||
typeof req.query.runIds === "string" ? req.query.runIds.split(",") : null,
|
||||
);
|
||||
if (!parsed.success) throw badRequest("runIds must contain 1 to 100 UUIDs");
|
||||
res.json(await svc.runsForIssue(issue.companyId, issue.id, viewerUserId, undefined, parsed.data));
|
||||
return;
|
||||
}
|
||||
const result = await (req.query.view === "recent"
|
||||
? svc.runsForIssue(issue.companyId, issue.id, viewerUserId, 2)
|
||||
: svc.runsForIssue(issue.companyId, issue.id, viewerUserId));
|
||||
|
||||
@@ -19,11 +19,13 @@ import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { Router } from "express";
|
||||
import { companies, type Db } from "@paperclipai/db";
|
||||
import { adapterSettingsService } from "../services/adapter-settings.js";
|
||||
import { logActivity, publishActivity, type ActivityPublication } from "../services/activity-log.js";
|
||||
import {
|
||||
listServerAdapters,
|
||||
findServerAdapter,
|
||||
findActiveServerAdapter,
|
||||
listEnabledServerAdapters,
|
||||
registerServerAdapter,
|
||||
resolveExternalAdapterRegistration,
|
||||
unregisterServerAdapter,
|
||||
@@ -37,8 +39,6 @@ import {
|
||||
removeAdapterPlugin,
|
||||
getAdapterPluginByType,
|
||||
getAdapterPluginsDir,
|
||||
getDisabledAdapterTypes,
|
||||
setAdapterDisabled,
|
||||
} from "../services/adapter-plugin-store.js";
|
||||
import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
|
||||
import type { ServerAdapterModule, AdapterConfigSchema } from "../adapters/types.js";
|
||||
@@ -49,9 +49,9 @@ import type {
|
||||
import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { badRequest, forbidden } from "../errors.js";
|
||||
import { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
|
||||
import { getHiddenSettings } from "../services/settings-visibility.js";
|
||||
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin } from "./authz.js";
|
||||
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js";
|
||||
import { BUILTIN_ADAPTER_TYPES } from "../adapters/builtin-adapter-types.js";
|
||||
import { agentRunnerAvailability, type RunnerTarget } from "../services/agent-runner-selection.js";
|
||||
import type { AgentRunnerAvailability } from "@paperclipai/shared";
|
||||
@@ -60,22 +60,6 @@ import { resolveAntigravityCandidateAdmission } from "../services/native-runtime
|
||||
const execFileAsync = promisify(execFile);
|
||||
let warnedMalformedAntigravityAdmission = false;
|
||||
|
||||
/**
|
||||
* Floor: on cloud-managed instances adapter code is bundled into the platform
|
||||
* image; fetching and loading external adapter packages at runtime stays off
|
||||
* for every actor, including instance admins. Adapter code executes in the
|
||||
* server process, so a runtime install would let an instance admin read the
|
||||
* platform trust anchors from the process environment (mirrors the
|
||||
* bundled-only plugin install floor in plugin-install-guard.ts).
|
||||
*/
|
||||
function assertAdapterCodeInstallAllowed() {
|
||||
if (isCloudManagedInstance()) {
|
||||
throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
|
||||
code: "adapter_install_platform_managed",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Floor: when the hosting operator hides the Adapters settings surface
|
||||
* (`instance.adapters` in PAPERCLIP_HIDDEN_SETTINGS), adapter management
|
||||
@@ -209,7 +193,7 @@ function buildAdapterInfo(adapter: ServerAdapterModule, externalRecord: AdapterP
|
||||
loaded: true, // If it's in the registry, it's loaded
|
||||
disabled: disabledSet.has(adapter.type),
|
||||
capabilities: buildAdapterCapabilities(adapter),
|
||||
...agentRunnerAvailability(adapter.type, target),
|
||||
...agentRunnerAvailability(adapter.type, target, [...disabledSet]),
|
||||
...(adapter.acp ? { acp: adapter.acp } : {}),
|
||||
overriddenBuiltin: externalRecord ? BUILTIN_ADAPTER_TYPES.has(adapter.type) : undefined,
|
||||
overridePaused: BUILTIN_ADAPTER_TYPES.has(adapter.type) ? isOverridePaused(adapter.type) : undefined,
|
||||
@@ -265,11 +249,13 @@ function registerWithSessionManagement(adapter: ServerAdapterModule): void {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function adapterRoutes(options: {
|
||||
db?: Db;
|
||||
getNativeRunnerEnabled?: () => Promise<boolean>;
|
||||
getOpenAiDotEnabled?: () => Promise<boolean>;
|
||||
resolveRunnerTarget?: (companyId: string, environmentId?: string) => Promise<RunnerTarget>;
|
||||
} = {}) {
|
||||
const router = Router();
|
||||
const settings = adapterSettingsService(options.db);
|
||||
|
||||
/**
|
||||
* GET /api/adapters
|
||||
@@ -303,7 +289,7 @@ export function adapterRoutes(options: {
|
||||
const externalRecords = new Map(
|
||||
listAdapterPlugins().map((r) => [r.type, r]),
|
||||
);
|
||||
const disabledSet = new Set(getDisabledAdapterTypes());
|
||||
const disabledSet = new Set(await settings.getDisabledTypes());
|
||||
// One shared implementation, with independent provider rollouts. Explicit
|
||||
// adapter-admin disabling still applies to both choices.
|
||||
|
||||
@@ -343,7 +329,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.post("/adapters/install", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertAdapterCodeInstallAllowed();
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const { packageName, isLocalPath = false, version } = req.body as AdapterInstallRequest;
|
||||
@@ -477,7 +463,7 @@ export function adapterRoutes(options: {
|
||||
}
|
||||
|
||||
const externalRecord = getAdapterPluginByType(adapterType);
|
||||
const disabledSet = new Set(getDisabledAdapterTypes());
|
||||
const disabledSet = new Set(await settings.getDisabledTypes());
|
||||
res.json(buildAdapterInfo(adapter, externalRecord, disabledSet));
|
||||
});
|
||||
|
||||
@@ -509,7 +495,28 @@ export function adapterRoutes(options: {
|
||||
return;
|
||||
}
|
||||
|
||||
const changed = setAdapterDisabled(adapterType, disabled);
|
||||
const publications: ActivityPublication[] = [];
|
||||
const changed = await settings.setDisabled(adapterType, disabled, async (tx, current) => {
|
||||
const actor = getActorInfo(req);
|
||||
const companyIds = await tx.select({ id: companies.id }).from(companies);
|
||||
for (const company of companyIds) {
|
||||
await logActivity(tx as Db, {
|
||||
companyId: company.id,
|
||||
...actor,
|
||||
action: "instance.adapter.updated",
|
||||
entityType: "instance_settings",
|
||||
entityId: "adapter-settings:v1",
|
||||
details: { adapterType, disabled, revision: current.revision },
|
||||
}, publications);
|
||||
}
|
||||
});
|
||||
for (const publication of publications) {
|
||||
try {
|
||||
publishActivity(publication);
|
||||
} catch (error) {
|
||||
logger.error({ err: error, companyId: publication.companyId }, "failed to publish adapter settings activity");
|
||||
}
|
||||
}
|
||||
|
||||
if (changed) {
|
||||
logger.info({ type: adapterType, disabled }, "Adapter enabled/disabled");
|
||||
@@ -528,6 +535,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.patch("/adapters/:type/override", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertExternalAdapterCodeAllowed();
|
||||
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
@@ -558,6 +566,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.delete("/adapters/:type", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const adapterType = req.params.type;
|
||||
@@ -635,6 +644,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.post("/adapters/:type/reload", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const type = req.params.type;
|
||||
@@ -688,7 +698,7 @@ export function adapterRoutes(options: {
|
||||
// package name, but without the risk of losing the store record.
|
||||
router.post("/adapters/:type/reinstall", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertAdapterCodeInstallAllowed();
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const type = req.params.type;
|
||||
|
||||
@@ -154,7 +154,7 @@ import type {
|
||||
} from "@paperclipai/adapter-utils";
|
||||
import { evaluateCodexCredentialReadiness } from "@paperclipai/adapter-codex-local/server";
|
||||
import type { AdapterAuthSignal, AdapterAuthSignalResponse, CodexAccountBindingClaim } from "@paperclipai/shared";
|
||||
import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
|
||||
import { adapterSettingsService } from "../services/adapter-settings.js";
|
||||
import { skillVersionSelectionMap } from "../services/runtime-skill-selections.js";
|
||||
import { isFixedClaudeOAuthBinding, secretService } from "../services/secrets.js";
|
||||
import { authorizationDeniedDetails } from "../services/authorization.js";
|
||||
@@ -1859,15 +1859,6 @@ export function agentRoutes(
|
||||
* Adapter validation for the paths that CHOOSE a harness for a new agent
|
||||
* (hire + create), as opposed to the paths that operate on an existing one.
|
||||
*
|
||||
* A disabled adapter is one this instance cannot run — most often because a
|
||||
* declarative registry (PAPERCLIP_ADAPTERS) curated it out, which
|
||||
* reconcileAdapterAvailability turns into a disabled type at boot. Registered
|
||||
* but disabled still passes assertKnownAdapterType, so an agent could be
|
||||
* created on it and then fail EVERY run at lease time with
|
||||
* `Adapter "..." is not in the configured adapter registry` — an error that
|
||||
* arrives minutes later, in a run log, with no way back to the choice that
|
||||
* caused it. Refuse at selection time instead, and name what can be chosen.
|
||||
*
|
||||
* Existing agents on a now-disabled adapter are deliberately untouched
|
||||
* (listEnabledServerAdapters documents the same rule: hidden from selection,
|
||||
* still functional for agents that already use them).
|
||||
@@ -1902,7 +1893,7 @@ export function agentRoutes(
|
||||
);
|
||||
}
|
||||
}
|
||||
const disabled = new Set(getDisabledAdapterTypes());
|
||||
const disabled = new Set(await adapterSettingsService(db).getDisabledTypes());
|
||||
if (!disabled.has(adapterType)) return adapterType;
|
||||
const available = listServerAdapters()
|
||||
.map((a) => a.type)
|
||||
|
||||
@@ -17,6 +17,22 @@ export function attentionRoutes(db: Db) {
|
||||
const router = Router();
|
||||
const svc = attentionService(db);
|
||||
|
||||
router.get("/companies/:companyId/attention/summary", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
assertBoard(req);
|
||||
if (!req.actor.userId) { res.status(403).json({ error: "Board user context required" }); return; }
|
||||
res.json(await svc.summary(companyId, { userId: req.actor.userId, actor: req.actor }));
|
||||
});
|
||||
|
||||
router.get("/companies/:companyId/attention/tasks", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
assertBoard(req);
|
||||
if (!req.actor.userId) { res.status(403).json({ error: "Board user context required" }); return; }
|
||||
res.json(await svc.taskIndex(companyId, { userId: req.actor.userId, actor: req.actor }));
|
||||
});
|
||||
|
||||
router.get("/companies/:companyId/attention/history", async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
|
||||
@@ -16449,12 +16449,23 @@ export function issueRoutes(
|
||||
router.get("/issues/:id/recent-comments", async (req, res) => {
|
||||
const issue = await getAccessibleResource(req, res, getIssueById(req, req.params.id as string), "Issue not found");
|
||||
if (!issue || !(await assertIssueReadAllowed(req, res, issue))) return;
|
||||
if (issue.conversationAgentId) {
|
||||
const recent = await svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 21 });
|
||||
const comments = recent.slice(0, 20);
|
||||
res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, {
|
||||
comments,
|
||||
latestCommentId: comments[0]?.id ?? null,
|
||||
oldestCommentId: comments.at(-1)?.id ?? null,
|
||||
hasMore: recent.length > 20,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
const [recent, human, firstInstruction] = await Promise.all([
|
||||
svc.listComments(issue.id, { order: "desc", limit: 5 }),
|
||||
svc.listComments(issue.id, { order: "desc", limit: 1, humanOnly: true }),
|
||||
svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 5 }),
|
||||
svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 1, humanOnly: true }),
|
||||
issue.description?.trim()
|
||||
? Promise.resolve([])
|
||||
: svc.listComments(issue.id, { order: "asc", limit: 1, humanOnly: true }),
|
||||
: svc.listComments(issue.id, { companyId: issue.companyId, order: "asc", limit: 1, humanOnly: true }),
|
||||
]);
|
||||
const comments = [...new Map([...firstInstruction, ...human, ...recent.slice(0, 4)].map((comment) => [comment.id, comment])).values()];
|
||||
res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, {
|
||||
@@ -16494,7 +16505,8 @@ export function issueRoutes(
|
||||
limitRaw && Number.isFinite(limitRaw) && limitRaw > 0
|
||||
? Math.min(Math.floor(limitRaw), MAX_ISSUE_COMMENT_LIMIT)
|
||||
: null;
|
||||
const comments = await svc.listComments(id, {
|
||||
const comments = await svc.listComments(issue.id, {
|
||||
companyId: issue.companyId,
|
||||
afterCommentId,
|
||||
order,
|
||||
limit,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { MUSE_ASSET_VERSION, museStopBoundarySchema, museDetectorCleanupAttestationSchema, createGrokInvitationSchema, configureGrokRoutineSchema, verifyGrokRoutineSchema, recoverGrokRoutineSchema, pairGrokBindingSchema, stopGrokWorkSchema, retryGrokDeliverySchema, updateFastResponseSchema } from "@paperclipai/shared";
|
||||
import { slackTestMessageSchema, slackTestMessageResultSchema, slackManagerAuthorizationSchema, slackSetupOptionsSchema, slackManagedProvisionSchema, slackManagedResultSchema, slackRegistrationSchema, slackSetupActionSchema, slackInstallAuthorizationSchema, slackRegistrationStateSchema, slackAppConfigurationSchema, slackAvatarStateSchema, slackAccountStateSchema } from "@paperclipai/shared";
|
||||
import { experimentalApiMetadata } from "./experimental-api-metadata.js";
|
||||
import { ATTENTION_SOURCE_KINDS } from "@paperclipai/shared";
|
||||
import { agentCaseHandoffSchema, caseWorkflowDocumentTextSchema, doneCaseHandoffSchema, createWorkflowSchema, updateWorkflowSchema, listWorkflowsSchema, readWorkflowQuerySchema } from "@paperclipai/shared";
|
||||
import {
|
||||
experimentalApiPaths,
|
||||
@@ -1417,6 +1418,8 @@ const BOARD_ONLY_OPERATIONS = new Set([
|
||||
"GET /api/issues/{id}/decision-receipts",
|
||||
"GET /api/issues/{id}/chat-handoffs",
|
||||
"GET /api/companies/{companyId}/attention/history",
|
||||
"GET /api/companies/{companyId}/attention/summary",
|
||||
"GET /api/companies/{companyId}/attention/tasks",
|
||||
"POST /api/issues/{id}/interactions/{interactionId}/dismiss",
|
||||
"POST /api/issues/{id}/retry",
|
||||
"GET /api/companies/{companyId}/prompt-injection-prevention",
|
||||
@@ -4596,7 +4599,7 @@ registry.registerPath({
|
||||
path: "/api/issues/{id}/recent-comments",
|
||||
tags: ["issues"],
|
||||
summary: "Read a bounded recent task exchange",
|
||||
description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.",
|
||||
description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Agent chats instead return the latest 20 contiguous comments and oldestCommentId for backward cursor paging. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.",
|
||||
request: { params: z.object({ id: z.string() }) },
|
||||
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
|
||||
});
|
||||
@@ -6088,8 +6091,8 @@ registry.registerPath({
|
||||
path: "/api/issues/{id}/runs",
|
||||
tags: ["activity"],
|
||||
summary: "List runs for an issue",
|
||||
request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries.") }) },
|
||||
responses: { 200: r.ok(), 401: r.unauthorized },
|
||||
request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries."), runIds: z.string().optional().describe("Comma-separated list of 1 to 100 run UUIDs. Returns only those runs linked to this accessible issue.") }) },
|
||||
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
|
||||
});
|
||||
|
||||
registry.registerPath({
|
||||
@@ -6160,6 +6163,29 @@ registry.registerPath({
|
||||
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden },
|
||||
});
|
||||
|
||||
const attentionSummaryResponseSchema = z.object({
|
||||
companyId: z.string().uuid(),
|
||||
generatedAt: z.string().datetime(),
|
||||
totalCount: z.number().int().nonnegative(),
|
||||
deskBadgeCount: z.number().int().nonnegative(),
|
||||
countsBySourceKind: z.record(z.enum(ATTENTION_SOURCE_KINDS), z.number().int().nonnegative()),
|
||||
}).strict();
|
||||
|
||||
for (const [projection, responseSchema] of [
|
||||
["summary", attentionSummaryResponseSchema],
|
||||
["tasks", attentionSummaryResponseSchema.extend({ taskIds: z.array(z.string().uuid()) })],
|
||||
] as const) {
|
||||
registry.registerPath({
|
||||
method: "get",
|
||||
path: `/api/companies/{companyId}/attention/${projection}`,
|
||||
tags: ["inbox"],
|
||||
summary: projection === "summary" ? "Get canonical attention counts" : "Get complete attention task membership",
|
||||
description: "Requires a board user with company access. Reads the complete canonical attention set with current task privacy, addressee, dismissal, snooze, and archive rules. Returns counts without decision cards, images, plan bodies, or pagination. The tasks projection also returns distinct accessible task IDs. This read grants no resolution authority.",
|
||||
request: { params: z.object({ companyId: z.string() }) },
|
||||
responses: { 200: r.ok(responseSchema), 401: r.unauthorized, 403: r.forbidden },
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Decisions ──────────────────────────────────────────────────────────────
|
||||
|
||||
registry.registerPath({
|
||||
|
||||
@@ -32,6 +32,7 @@ import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import crypto from "node:crypto";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { PluginRecord } from "@paperclipai/shared";
|
||||
import { pluginRegistryService } from "../services/plugin-registry.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { assertCompanyAccess } from "./authz.js";
|
||||
@@ -192,6 +193,7 @@ export interface PluginUiStaticRouteOptions {
|
||||
* Defaults to the standard `~/.paperclip/plugins/` location.
|
||||
*/
|
||||
localPluginDir: string;
|
||||
resolvePackageRoot?: (plugin: PluginRecord) => string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -304,7 +306,7 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
|
||||
|
||||
if (typeof devUiUrl === "string" && devUiUrl.length > 0) {
|
||||
// Dev proxy is only available in development mode
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
if (options.resolvePackageRoot || process.env.NODE_ENV === "production") {
|
||||
log.warn(
|
||||
{ pluginId: plugin.id },
|
||||
"plugin-ui-static: devUiUrl ignored in production",
|
||||
@@ -405,12 +407,15 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
|
||||
}
|
||||
|
||||
// Step 3: Resolve the plugin's UI directory
|
||||
const uiDir = resolvePluginUiDir(
|
||||
options.localPluginDir,
|
||||
plugin.packageName,
|
||||
manifest.entrypoints.ui,
|
||||
plugin.packagePath,
|
||||
);
|
||||
let uiDir: string | null;
|
||||
try {
|
||||
uiDir = options.resolvePackageRoot
|
||||
? path.resolve(options.resolvePackageRoot(plugin), manifest.entrypoints.ui)
|
||||
: resolvePluginUiDir(options.localPluginDir, plugin.packageName, manifest.entrypoints.ui, plugin.packagePath);
|
||||
} catch {
|
||||
res.status(503).json({ error: "Plugin package is unavailable in the current image" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!uiDir) {
|
||||
log.warn(
|
||||
|
||||
@@ -94,8 +94,8 @@ function loadOrCreateMasterKey(allowCreate = true, env: LocalEncryptedKeyEnv = p
|
||||
}
|
||||
|
||||
/** Derive a purpose-specific instance secret without exposing or reusing the vault key. */
|
||||
export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env): string {
|
||||
return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(true, env), Buffer.alloc(0),
|
||||
export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env, options: { allowCreate?: boolean } = {}): string {
|
||||
return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(options.allowCreate ?? true, env), Buffer.alloc(0),
|
||||
`paperclip:instance-secret:v1\0${purpose}`, 32)).toString("base64url");
|
||||
}
|
||||
|
||||
|
||||
@@ -428,7 +428,8 @@ export function activityService(db: Db) {
|
||||
return limit === undefined ? query : query.limit(limit);
|
||||
},
|
||||
|
||||
runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number) => {
|
||||
runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number, selectedRunIds?: string[]) => {
|
||||
if (selectedRunIds && !selectedRunIds.length) return [];
|
||||
scheduleRunLivenessBackfill(companyId, issueId);
|
||||
const runsQuery = db
|
||||
.select({
|
||||
@@ -479,6 +480,7 @@ export function activityService(db: Db) {
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
...(selectedRunIds ? [inArray(heartbeatRuns.id, selectedRunIds)] : []),
|
||||
or(
|
||||
eq(heartbeatRuns.issueId, issueId),
|
||||
sql`exists (
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { forbidden } from "../errors.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
|
||||
export function assertExternalAdapterCodeAllowed(): void {
|
||||
if (isCloudManagedInstance()) {
|
||||
throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
|
||||
code: "adapter_install_platform_managed",
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,8 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolvePaperclipHomeDir } from "../home-paths.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "./adapter-code-policy.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -43,8 +45,8 @@ interface AdapterSettings {
|
||||
// Paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function adapterPluginPaths() {
|
||||
const paperclipDir = resolvePaperclipHomeDir();
|
||||
function adapterPluginPaths(homeOverride?: string) {
|
||||
const paperclipDir = resolvePaperclipHomeDir(homeOverride);
|
||||
return {
|
||||
adapterPluginsDir: path.join(paperclipDir, "adapter-plugins"),
|
||||
adapterPluginsStorePath: path.join(paperclipDir, "adapter-plugins.json"),
|
||||
@@ -79,6 +81,7 @@ function ensureDirs(): string {
|
||||
}
|
||||
|
||||
function readStore(): AdapterPluginRecord[] {
|
||||
if (isCloudManagedInstance()) return [];
|
||||
const { adapterPluginsStorePath } = adapterPluginPaths();
|
||||
if (storeCache?.path === adapterPluginsStorePath) return storeCache.records;
|
||||
try {
|
||||
@@ -130,11 +133,16 @@ function writeSettings(settings: AdapterSettings): void {
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function getAdapterPluginStorePath(homeOverride?: string): string {
|
||||
return adapterPluginPaths(homeOverride).adapterPluginsStorePath;
|
||||
}
|
||||
|
||||
export function listAdapterPlugins(): AdapterPluginRecord[] {
|
||||
return readStore();
|
||||
}
|
||||
|
||||
export function addAdapterPlugin(record: AdapterPluginRecord): void {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const store = [...readStore()];
|
||||
const idx = store.findIndex((r) => r.type === record.type);
|
||||
if (idx >= 0) {
|
||||
@@ -146,6 +154,7 @@ export function addAdapterPlugin(record: AdapterPluginRecord): void {
|
||||
}
|
||||
|
||||
export function removeAdapterPlugin(type: string): boolean {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const store = [...readStore()];
|
||||
const idx = store.findIndex((r) => r.type === type);
|
||||
if (idx < 0) return false;
|
||||
@@ -159,6 +168,7 @@ export function getAdapterPluginByType(type: string): AdapterPluginRecord | unde
|
||||
}
|
||||
|
||||
export function getAdapterPluginsDir(): string {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
return ensureDirs();
|
||||
}
|
||||
|
||||
|
||||
@@ -32,8 +32,7 @@ describe("reconcileAdapterAvailability", () => {
|
||||
]);
|
||||
expect(result.enabled).toEqual(["opencode_local"]);
|
||||
expect(result.disabled.sort()).toEqual(["claude_local", "pi_local"]);
|
||||
expect(setAdapterDisabled).toHaveBeenCalledWith("claude_local", true);
|
||||
expect(setAdapterDisabled).toHaveBeenCalledWith("opencode_local", false);
|
||||
expect(setAdapterDisabled).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("throws when a declared adapter has no installed implementation", () => {
|
||||
|
||||
@@ -1,73 +1,8 @@
|
||||
/**
|
||||
* Declarative adapter-registry bootstrap.
|
||||
*
|
||||
* One source (`PAPERCLIP_ADAPTERS` inline JSON, or `PAPERCLIP_ADAPTERS_FILE` a
|
||||
* path to a JSON file) feeds two consumers:
|
||||
* 1. Availability: reconcile the file-backed disabled-set so the picker shows
|
||||
* exactly the declared, enabled set (runs for any instance).
|
||||
* 2. k8s runtime: the same registry rides on the Kubernetes environment config
|
||||
* (see execution-policy-bootstrap) so the plugin resolves runtime
|
||||
* image/envKeys/allowFqdns/probe/defaultEnv from it.
|
||||
*
|
||||
* Parsing is pure + fails loud on malformed/invalid config, mirroring
|
||||
* execution-policy-bootstrap.
|
||||
*/
|
||||
import fs from "node:fs";
|
||||
import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
|
||||
import type { AdapterRegistryEntryParsed } from "@paperclipai/shared";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { listServerAdapters } from "../adapters/registry.js";
|
||||
import { setAdapterDisabled } from "./adapter-plugin-store.js";
|
||||
export { parseAdapterRegistryEnv, type AdapterRegistryEnv } from "./adapter-registry-config.js";
|
||||
|
||||
export type AdapterRegistryEnv = Record<string, string | undefined>;
|
||||
|
||||
/**
|
||||
* Parse the declarative registry from env. Returns null when unconfigured
|
||||
* (built-in defaults; local/OSS unchanged). Throws on malformed/invalid input.
|
||||
*/
|
||||
export function parseAdapterRegistryEnv(
|
||||
env: AdapterRegistryEnv = process.env,
|
||||
): AdapterRegistryEntryParsed[] | null {
|
||||
const inline = env.PAPERCLIP_ADAPTERS?.trim();
|
||||
const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
|
||||
if (!inline && !filePath) return null;
|
||||
|
||||
let rawText: string;
|
||||
if (inline) {
|
||||
rawText = inline;
|
||||
} else {
|
||||
try {
|
||||
rawText = fs.readFileSync(filePath as string, "utf-8");
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(rawText);
|
||||
} catch (err) {
|
||||
throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
const result = adapterRegistrySchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
|
||||
.map((i) => `${i.path.join(".")}: ${i.message}`)
|
||||
.join("; ")}`,
|
||||
);
|
||||
}
|
||||
return result.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile availability: every known server adapter NOT enabled in the declared
|
||||
* registry is disabled; declared+enabled ones are enabled. Throws if a declared
|
||||
* adapterType has no installed adapter (cannot offer a harness with no
|
||||
* implementation). No-op when `registry` is null.
|
||||
*/
|
||||
export function reconcileAdapterAvailability(
|
||||
registry: AdapterRegistryEntryParsed[] | null,
|
||||
): { enabled: string[]; disabled: string[] } {
|
||||
@@ -88,10 +23,9 @@ export function reconcileAdapterAvailability(
|
||||
for (const type of knownTypes) {
|
||||
const entry = declared.get(type);
|
||||
const shouldEnable = entry !== undefined && entry.enabled !== false;
|
||||
setAdapterDisabled(type, !shouldEnable);
|
||||
(shouldEnable ? enabled : disabled).push(type);
|
||||
}
|
||||
|
||||
logger.info({ enabled, disabled }, "reconciled adapter availability from PAPERCLIP_ADAPTERS");
|
||||
logger.info({ enabled, disabled }, "validated adapter availability from PAPERCLIP_ADAPTERS");
|
||||
return { enabled, disabled };
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import fs from "node:fs";
|
||||
import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
|
||||
|
||||
export type AdapterRegistryEnv = Record<string, string | undefined>;
|
||||
|
||||
let cached: { inline: string | undefined; filePath: string | undefined; registry: AdapterRegistryEntryParsed[] } | undefined;
|
||||
|
||||
export function parseAdapterRegistryEnv(
|
||||
env: AdapterRegistryEnv = process.env,
|
||||
): AdapterRegistryEntryParsed[] | null {
|
||||
const inline = env.PAPERCLIP_ADAPTERS?.trim();
|
||||
const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
|
||||
if (!inline && !filePath) return null;
|
||||
if (cached && cached.inline === inline && cached.filePath === filePath) return structuredClone(cached.registry);
|
||||
|
||||
let rawText: string;
|
||||
if (inline) {
|
||||
rawText = inline;
|
||||
} else {
|
||||
try {
|
||||
rawText = fs.readFileSync(filePath as string, "utf-8");
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(rawText);
|
||||
} catch (err) {
|
||||
throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
const result = adapterRegistrySchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
|
||||
.map((i) => `${i.path.join(".")}: ${i.message}`)
|
||||
.join("; ")}`,
|
||||
);
|
||||
}
|
||||
cached = { inline, filePath, registry: result.data };
|
||||
return structuredClone(result.data);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../adapters/registry.js", () => ({ listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }] }));
|
||||
import { adapterSettingsService, type AdapterSettingsDb } from "./adapter-settings.js";
|
||||
import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
|
||||
import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
|
||||
|
||||
describe("local adapter settings compatibility", () => {
|
||||
let directory: string;
|
||||
beforeEach(() => {
|
||||
directory = fs.mkdtempSync(path.join(os.tmpdir(), "adapter-settings-local-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", directory);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS", undefined);
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS_FILE", undefined);
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("reads and writes the existing file without a database", async () => {
|
||||
fs.writeFileSync(path.join(directory, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local"] }));
|
||||
const service = adapterSettingsService(undefined);
|
||||
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
|
||||
expect(await service.setDisabled("codex_local", true)).toBe(true);
|
||||
expect(getDisabledAdapterTypes()).toEqual(["claude_local", "codex_local"]);
|
||||
expect(setAdapterDisabled("claude_local", false)).toBe(true);
|
||||
expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
|
||||
expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: ["codex_local"] });
|
||||
});
|
||||
|
||||
it("loads an operator file once and leaves the saved file intact", async () => {
|
||||
setAdapterDisabled("codex_local", true);
|
||||
const file = path.join(directory, "operator-adapters.json");
|
||||
fs.writeFileSync(file, JSON.stringify([{ adapterType: "codex_local", enabled: true }]));
|
||||
const env = { PAPERCLIP_ADAPTERS_FILE: file };
|
||||
parseAdapterRegistryEnv(env);
|
||||
const read = vi.spyOn(fs, "readFileSync");
|
||||
const service = adapterSettingsService(undefined, { runtimeEnv: env });
|
||||
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
|
||||
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
expect(getDisabledAdapterTypes()).toEqual(["codex_local"]);
|
||||
});
|
||||
|
||||
it("restores the saved choice when its audit fails and audits a later retry", async () => {
|
||||
const transaction = vi.fn(async (callback: (tx: AdapterSettingsDb) => Promise<unknown>) => callback(db));
|
||||
const db = { transaction } as unknown as AdapterSettingsDb;
|
||||
const service = adapterSettingsService(db);
|
||||
const audit = vi.fn().mockRejectedValueOnce(new Error("audit unavailable")).mockResolvedValue(undefined);
|
||||
|
||||
await expect(service.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
|
||||
expect(await service.getDisabledTypes()).toEqual([]);
|
||||
expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: [] });
|
||||
expect(await service.setDisabled("codex_local", true, audit)).toBe(true);
|
||||
expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
|
||||
expect(audit).toHaveBeenCalledTimes(2);
|
||||
expect(transaction).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps a failed audit rollback from overwriting a concurrent save", async () => {
|
||||
const db = { transaction: async (callback: (tx: AdapterSettingsDb) => Promise<unknown>) => callback(db) } as unknown as AdapterSettingsDb;
|
||||
const first = adapterSettingsService(db);
|
||||
const second = adapterSettingsService(db);
|
||||
let failAudit!: (reason: Error) => void;
|
||||
const auditWait = new Promise<void>((_resolve, reject) => { failAudit = reject; });
|
||||
const audit = vi.fn(() => auditWait);
|
||||
const rejected = expect(first.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
|
||||
await vi.waitFor(() => expect(audit).toHaveBeenCalledOnce());
|
||||
const nextAudit = vi.fn(async () => undefined);
|
||||
const next = second.setDisabled("codex_local", true, nextAudit);
|
||||
await Promise.resolve();
|
||||
expect(nextAudit).not.toHaveBeenCalled();
|
||||
|
||||
failAudit(new Error("audit unavailable"));
|
||||
await rejected;
|
||||
expect(await next).toBe(true);
|
||||
expect(await first.getDisabledTypes()).toEqual(["codex_local"]);
|
||||
expect(nextAudit).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,228 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { activityLog, applyPendingMigrations, companies, createDb, instanceSettings, type Db } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
|
||||
const legacy = vi.hoisted(() => ({ get: vi.fn(() => [] as string[]), set: vi.fn() }));
|
||||
vi.mock("./adapter-plugin-store.js", () => ({
|
||||
getDisabledAdapterTypes: legacy.get,
|
||||
setAdapterDisabled: legacy.set,
|
||||
listAdapterPlugins: () => [],
|
||||
getAdapterPluginByType: () => undefined,
|
||||
}));
|
||||
vi.mock("../adapters/registry.js", () => ({
|
||||
listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }],
|
||||
findServerAdapter: (type: string) => ({ type }),
|
||||
findActiveServerAdapter: (type: string) => ({ type }),
|
||||
isOverridePaused: () => false,
|
||||
hasActiveAdapterOverride: () => false,
|
||||
waitForExternalAdapters: async () => {},
|
||||
}));
|
||||
vi.mock("../adapters/plugin-loader.js", () => ({}));
|
||||
import { adapterSettingsService, initializeAdapterSettings } from "./adapter-settings.js";
|
||||
import { adapterRoutes } from "../routes/adapters.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
import * as activities from "./activity-log.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL;
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("durable managed adapter settings", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let peer: Db;
|
||||
let directory: string;
|
||||
const managed = { runtimeEnv: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "synthetic-test-token" } };
|
||||
const service = () => adapterSettingsService(db, managed);
|
||||
|
||||
beforeAll(async () => {
|
||||
database = externalDatabaseUrl
|
||||
? { connectionString: externalDatabaseUrl, cleanup: async () => {} }
|
||||
: await startEmbeddedPostgresTestDatabase("adapter-settings-");
|
||||
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
|
||||
db = createDb(database.connectionString);
|
||||
peer = createDb(database.connectionString);
|
||||
await db.insert(companies).values({ name: "Adapter settings test", issuePrefix: "AST" });
|
||||
}, 90_000);
|
||||
afterAll(async () => {
|
||||
await Promise.all([db?.$client.end({ timeout: 0 }), peer?.$client.end({ timeout: 0 })]);
|
||||
await database?.cleanup();
|
||||
}, 60_000);
|
||||
beforeEach(async () => {
|
||||
directory = await fs.mkdtemp(path.join(os.tmpdir(), "adapter-settings-import-"));
|
||||
await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, "adapter-settings:v1"));
|
||||
legacy.get.mockClear();
|
||||
legacy.set.mockClear();
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-test-token");
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function app() {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", source: "local_implicit", userId: "local-board", isInstanceAdmin: true };
|
||||
next();
|
||||
});
|
||||
app.use(adapterRoutes({ db }));
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
}
|
||||
|
||||
it("persists across independent connections and reconstructed stores without file reads", async () => {
|
||||
await expect(service().getDisabledTypes()).rejects.toThrow("not been initialized");
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
expect(await service().setDisabled("codex_local", true)).toBe(true);
|
||||
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
|
||||
await db.$client.end({ timeout: 0 });
|
||||
db = createDb(database.connectionString);
|
||||
expect(await service().inspectStored()).toEqual({ version: 1, revision: 1, disabledTypes: ["codex_local"] });
|
||||
expect(legacy.get).not.toHaveBeenCalled();
|
||||
expect(legacy.set).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("acknowledges the committed save when its live activity event fails", async () => {
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
const publication = vi.spyOn(activities, "publishActivity").mockImplementation(() => { throw new Error("event unavailable"); });
|
||||
const response = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(publication).toHaveBeenCalled();
|
||||
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
|
||||
const events = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
|
||||
expect(events.some((event) => event.details?.adapterType === "codex_local")).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves simultaneous first writes and concurrent removals from other connections", async () => {
|
||||
const writers = [service(), adapterSettingsService(peer, managed)];
|
||||
await Promise.all(Array.from({ length: 12 }, (_, index) => writers[index % 2].setDisabled(`adapter_${index}`, true)));
|
||||
expect((await service().inspectStored())?.disabledTypes).toHaveLength(12);
|
||||
await Promise.all([
|
||||
writers[0].setDisabled("adapter_0", false),
|
||||
writers[1].setDisabled("adapter_1", false),
|
||||
writers[0].setDisabled("extra", true),
|
||||
]);
|
||||
const stored = await service().inspectStored();
|
||||
expect(stored?.revision).toBe(15);
|
||||
expect(stored?.disabledTypes).not.toContain("adapter_0");
|
||||
expect(stored?.disabledTypes).not.toContain("adapter_1");
|
||||
expect(stored?.disabledTypes).toContain("extra");
|
||||
expect(await writers[1].setDisabled("extra", true)).toBe(false);
|
||||
expect((await service().inspectStored())?.revision).toBe(15);
|
||||
});
|
||||
|
||||
it("imports legacy choices once before admission and ignores stale files after the DB record exists", async () => {
|
||||
const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
|
||||
const file = path.join(directory, "adapter-settings.json");
|
||||
await fs.writeFile(file, JSON.stringify({ disabledTypes: ["codex_local", "future_adapter"] }));
|
||||
const results = await Promise.all([
|
||||
initializeAdapterSettings(db, { runtimeEnv }),
|
||||
initializeAdapterSettings(peer, { runtimeEnv }),
|
||||
]);
|
||||
expect(results.filter((result) => result?.seeded)).toHaveLength(1);
|
||||
expect(await service().getDisabledTypes()).toEqual(["codex_local", "future_adapter"]);
|
||||
await service().setDisabled("codex_local", false);
|
||||
await fs.writeFile(file, "invalid stale JSON");
|
||||
expect((await initializeAdapterSettings(peer, { runtimeEnv }))?.seeded).toBe(false);
|
||||
expect(await service().getDisabledTypes()).toEqual(["future_adapter"]);
|
||||
});
|
||||
|
||||
it("uses defaults only for an absent legacy file and rejects malformed or oversized input", async () => {
|
||||
const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
|
||||
const file = path.join(directory, "adapter-settings.json");
|
||||
await fs.writeFile(file, JSON.stringify({ disabledTypes: [42] }));
|
||||
await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("Persisted adapter settings are invalid");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await fs.writeFile(file, Buffer.alloc(65_537));
|
||||
await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("size limit");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await fs.unlink(file);
|
||||
expect((await initializeAdapterSettings(db, { runtimeEnv }))?.current.disabledTypes).toEqual([]);
|
||||
expect(await service().getDisabledTypes()).toEqual([]);
|
||||
});
|
||||
|
||||
it("composes atomic import and receipts with the caller transaction", async () => {
|
||||
await expect(db.transaction(async (tx) => {
|
||||
expect(await service().seedIfAbsent({ disabledTypes: ["codex_local"] }, tx)).toEqual({
|
||||
seeded: true, current: { version: 1, revision: 0, disabledTypes: ["codex_local"] },
|
||||
});
|
||||
throw new Error("receipt failed");
|
||||
})).rejects.toThrow("receipt failed");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await Promise.all([
|
||||
service().seedIfAbsent({ disabledTypes: ["codex_local"] }),
|
||||
adapterSettingsService(peer, managed).seedIfAbsent({ disabledTypes: ["claude_local"] }),
|
||||
]).then((results) => {
|
||||
expect(results.filter((result) => result.seeded)).toHaveLength(1);
|
||||
expect(results[0].current).toEqual(results[1].current);
|
||||
});
|
||||
});
|
||||
|
||||
it("applies operator restrictions without rewriting saved choices", async () => {
|
||||
await service().setDisabled("codex_local", true);
|
||||
const env = { ...managed.runtimeEnv, PAPERCLIP_ADAPTERS: JSON.stringify([{ adapterType: "codex_local", enabled: true }]) };
|
||||
const overlay = adapterSettingsService(peer, { runtimeEnv: env });
|
||||
expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
|
||||
expect(await overlay.inspectStored()).toEqual(await service().inspectStored());
|
||||
await expect(overlay.setDisabled("claude_local", false)).rejects.toThrow("hosting operator");
|
||||
await expect(overlay.setDisabled("codex_local", true)).rejects.toThrow("hosting operator");
|
||||
expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
|
||||
expect(await service().getDisabledTypes()).toEqual(["codex_local"]);
|
||||
});
|
||||
|
||||
it("refreshes the disabled harness at each new-agent admission", async () => {
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
const input = { adapterType: "codex_local", runner: "legacy" as const };
|
||||
expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
|
||||
await adapterSettingsService(peer, managed).setDisabled("codex_local", true);
|
||||
await expect(resolveNewAgentRunnerForCompany(db, "company", input)).rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } });
|
||||
await adapterSettingsService(peer, managed).setDisabled("codex_local", false);
|
||||
expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
|
||||
});
|
||||
|
||||
it("rolls back the setting when its audit write fails", async () => {
|
||||
await expect(service().setDisabled("codex_local", true, async () => { throw new Error("audit failed"); })).rejects.toThrow("audit failed");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await expect(adapterSettingsService(peer, managed).getDisabledTypes()).rejects.toThrow("not been initialized");
|
||||
});
|
||||
|
||||
it("publishes catalog changes across stores and records the instance mutation", async () => {
|
||||
const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ type: "codex_local", disabled: true, changed: true });
|
||||
const catalog = await request(app()).get("/adapters");
|
||||
expect(catalog.status).toBe(200);
|
||||
expect(catalog.body.find((entry: { type: string }) => entry.type === "codex_local").disabled).toBe(true);
|
||||
const [audit] = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
|
||||
expect(audit.details).toMatchObject({ adapterType: "codex_local", disabled: true, revision: 1 });
|
||||
});
|
||||
|
||||
it("returns failure when PostgreSQL rejects the update and keeps the durable choice", async () => {
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
await db.execute(sql`CREATE FUNCTION reject_adapter_settings() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test adapter settings failure'; END $$`);
|
||||
await db.execute(sql`CREATE TRIGGER reject_adapter_settings BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_adapter_settings()`);
|
||||
try {
|
||||
const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
|
||||
expect(res.status).toBe(500);
|
||||
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual([]);
|
||||
} finally {
|
||||
await db.execute(sql`DROP TRIGGER reject_adapter_settings ON instance_settings`);
|
||||
await db.execute(sql`DROP FUNCTION reject_adapter_settings()`);
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed on malformed stored settings and unavailable database", async () => {
|
||||
await db.insert(instanceSettings).values({ singletonKey: "adapter-settings:v1", general: { version: 2, disabledTypes: [] } });
|
||||
await expect(service().getDisabledTypes()).rejects.toThrow("Persisted adapter settings are invalid");
|
||||
await expect(adapterSettingsService(undefined, managed).getDisabledTypes()).rejects.toThrow("database is required");
|
||||
expect(legacy.get).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { instanceSettings } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { open } from "node:fs/promises";
|
||||
import { constants } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { z } from "zod";
|
||||
import { listServerAdapters } from "../adapters/registry.js";
|
||||
import { forbidden } from "../errors.js";
|
||||
import { resolvePaperclipHomeDir } from "../home-paths.js";
|
||||
import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
|
||||
import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
|
||||
const SINGLETON_KEY = "adapter-settings:v1";
|
||||
let localSettingsMutation: Promise<unknown> = Promise.resolve();
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type AdapterSettingsDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
|
||||
export type StoredAdapterSettings = { version: 1; revision: number; disabledTypes: string[] };
|
||||
const storedSettingsSchema = z.object({
|
||||
version: z.literal(1),
|
||||
revision: z.number().int().nonnegative(),
|
||||
disabledTypes: z.array(z.string().refine((type) => type.trim().length > 0)),
|
||||
});
|
||||
|
||||
function parseStored(value: Record<string, unknown>): StoredAdapterSettings {
|
||||
const parsed = storedSettingsSchema.safeParse(value);
|
||||
if (!parsed.success) throw new Error("Persisted adapter settings are invalid");
|
||||
return { ...parsed.data, disabledTypes: [...new Set(parsed.data.disabledTypes)].sort() };
|
||||
}
|
||||
|
||||
export function effectiveDisabledAdapterTypes(saved: string[], knownTypes: string[] | (() => string[]), env: Record<string, string | undefined> = process.env): string[] {
|
||||
const registry = parseAdapterRegistryEnv(env);
|
||||
if (!registry) return [...saved];
|
||||
const enabled = new Set(registry.filter((entry) => entry.enabled !== false).map((entry) => entry.adapterType));
|
||||
const types = typeof knownTypes === "function" ? knownTypes() : knownTypes;
|
||||
return [...new Set(types.filter((type) => !enabled.has(type)))].sort();
|
||||
}
|
||||
|
||||
export function adapterSettingsService(db: AdapterSettingsDb | undefined, options: { runtimeEnv?: Record<string, string | undefined> } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
const managed = isCloudManagedInstance(env);
|
||||
|
||||
function requireDb(): AdapterSettingsDb {
|
||||
if (!db) throw new Error("A database is required for managed adapter settings");
|
||||
return db;
|
||||
}
|
||||
|
||||
async function inspectStored(handle: AdapterSettingsDb = requireDb()): Promise<StoredAdapterSettings | null> {
|
||||
const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).limit(1);
|
||||
return row ? parseStored(row.general) : null;
|
||||
}
|
||||
|
||||
async function seedIfAbsent(value: { disabledTypes: string[] }, handle: AdapterSettingsDb = requireDb()) {
|
||||
const snapshot = parseStored({ version: 1, revision: 0, disabledTypes: value.disabledTypes });
|
||||
const inserted = await handle.insert(instanceSettings).values({ singletonKey: SINGLETON_KEY, general: snapshot })
|
||||
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
|
||||
const current = await inspectStored(handle);
|
||||
if (!current) throw new Error("Adapter settings seed was not persisted");
|
||||
return { seeded: inserted.length > 0, current };
|
||||
}
|
||||
|
||||
return {
|
||||
inspectStored,
|
||||
seedIfAbsent,
|
||||
async getDisabledTypes(): Promise<string[]> {
|
||||
const current = managed ? await inspectStored() : null;
|
||||
if (managed && !current) throw new Error("Managed adapter settings have not been initialized");
|
||||
const stored = managed ? current!.disabledTypes : getDisabledAdapterTypes();
|
||||
return effectiveDisabledAdapterTypes(stored, () => listServerAdapters().map((adapter) => adapter.type), env);
|
||||
},
|
||||
async setDisabled(type: string, disabled: boolean, afterChange?: (handle: AdapterSettingsDb, current: StoredAdapterSettings) => Promise<void>): Promise<boolean> {
|
||||
if (!type.trim()) throw new Error("Adapter type is required");
|
||||
const registry = parseAdapterRegistryEnv(env);
|
||||
if (registry && disabled === registry.some((entry) => entry.adapterType === type && entry.enabled !== false)) {
|
||||
throw forbidden("Adapter availability is managed by the hosting operator", { code: "settings_operator_managed" });
|
||||
}
|
||||
if (!managed) {
|
||||
const mutation = localSettingsMutation.then(async () => {
|
||||
const previous = getDisabledAdapterTypes().includes(type);
|
||||
const changed = setAdapterDisabled(type, disabled);
|
||||
if (!changed || !db || !afterChange) return changed;
|
||||
try {
|
||||
await db.transaction(async (tx) => {
|
||||
await afterChange(tx, { version: 1, revision: 0, disabledTypes: getDisabledAdapterTypes() });
|
||||
});
|
||||
} catch (error) {
|
||||
setAdapterDisabled(type, previous);
|
||||
throw error;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
localSettingsMutation = mutation.catch(() => undefined);
|
||||
return mutation;
|
||||
}
|
||||
return requireDb().transaction(async (tx) => {
|
||||
await seedIfAbsent({ disabledTypes: [] }, tx);
|
||||
const [row] = await tx.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).for("update");
|
||||
if (!row) throw new Error("Adapter settings are unavailable");
|
||||
const current = parseStored(row.general);
|
||||
const types = new Set(current.disabledTypes);
|
||||
if (types.has(type) === disabled) return false;
|
||||
if (disabled) types.add(type); else types.delete(type);
|
||||
const next: StoredAdapterSettings = { version: 1, revision: current.revision + 1, disabledTypes: [...types].sort() };
|
||||
await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
|
||||
await afterChange?.(tx, next);
|
||||
return true;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type AdapterSettingsService = Pick<ReturnType<typeof adapterSettingsService>, "getDisabledTypes" | "setDisabled">;
|
||||
|
||||
export async function initializeAdapterSettings(db: AdapterSettingsDb, options: { runtimeEnv?: Record<string, string | undefined> } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
if (!isCloudManagedInstance(env)) return null;
|
||||
const service = adapterSettingsService(db, options);
|
||||
const current = await service.inspectStored();
|
||||
if (current) return { seeded: false, current };
|
||||
return service.seedIfAbsent(await readLegacyAdapterSettings(env) ?? { disabledTypes: [] });
|
||||
}
|
||||
|
||||
export async function readLegacyAdapterSettings(env: Record<string, string | undefined> = process.env): Promise<{ disabledTypes: string[] } | null> {
|
||||
const filePath = path.join(resolvePaperclipHomeDir(env.PAPERCLIP_HOME), "adapter-settings.json");
|
||||
try {
|
||||
const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size > 65_536) throw new Error("Legacy adapter settings exceed the size limit or are not a regular file");
|
||||
const buffer = Buffer.alloc(65_537);
|
||||
let total = 0;
|
||||
while (total < buffer.length) {
|
||||
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
|
||||
if (bytesRead === 0) break;
|
||||
total += bytesRead;
|
||||
}
|
||||
if (total > 65_536) throw new Error("Legacy adapter settings exceed the size limit");
|
||||
const value = JSON.parse(buffer.subarray(0, total).toString("utf8"));
|
||||
return { disabledTypes: parseStored({ version: 1, revision: 0, disabledTypes: value?.disabledTypes }).disabledTypes };
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import type { AgentPublicIdentity } from "@paperclipai/shared";
|
||||
import { notFound } from "../errors.js";
|
||||
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
|
||||
import { persistActivity } from "./activity-log.js";
|
||||
import { createAgentIdentityRedactor } from "./agent-identity-redaction.js";
|
||||
|
||||
const publicColumns = {
|
||||
algorithm: agentIdentityKeys.algorithm,
|
||||
@@ -17,6 +18,16 @@ export function agentIdentityService(db: Db) {
|
||||
const ownership = (companyId: string, agentId: string) => and(
|
||||
eq(agentIdentityKeys.companyId, companyId), eq(agentIdentityKeys.agentId, agentId),
|
||||
);
|
||||
async function resolveIdentity(row: typeof agentIdentityKeys.$inferSelect) {
|
||||
const privateKeyPem = await localEncryptedProvider.resolveVersion({ material: row.privateKeyMaterial, externalRef: null });
|
||||
const privateKey = createPrivateKey(privateKeyPem);
|
||||
const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" });
|
||||
const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" });
|
||||
if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" ||
|
||||
!derived.equals(stored) || fingerprint(stored) !== row.keyId)
|
||||
throw new Error("Agent identity public and private keys do not match");
|
||||
return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem };
|
||||
}
|
||||
|
||||
return {
|
||||
// Reading an identity must never provision it or decrypt private material.
|
||||
@@ -24,6 +35,10 @@ export function agentIdentityService(db: Db) {
|
||||
const [row] = await db.select(publicColumns).from(agentIdentityKeys).where(ownership(companyId, agentId));
|
||||
return row ? { ...row, createdAt: row.createdAt.toISOString() } : null;
|
||||
},
|
||||
async createExistingLogRedactor(companyId: string, agentId: string) {
|
||||
const [row] = await db.select().from(agentIdentityKeys).where(ownership(companyId, agentId));
|
||||
return createAgentIdentityRedactor(row ? (await resolveIdentity(row)).privateKeyPem : undefined);
|
||||
},
|
||||
|
||||
async ensureAgentIdentity(companyId: string, agentId: string) {
|
||||
return db.transaction(async (tx) => {
|
||||
@@ -49,18 +64,8 @@ export function agentIdentityService(db: Db) {
|
||||
details: { algorithm: "Ed25519", keyId },
|
||||
});
|
||||
}
|
||||
const privateKeyPem = await localEncryptedProvider.resolveVersion({
|
||||
material: row.privateKeyMaterial, externalRef: null,
|
||||
});
|
||||
// Fail closed on corruption or a wrong master key; never replace a key.
|
||||
const privateKey = createPrivateKey(privateKeyPem);
|
||||
const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" });
|
||||
const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" });
|
||||
if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" ||
|
||||
!derived.equals(stored) || fingerprint(stored) !== row.keyId) {
|
||||
throw new Error("Agent identity public and private keys do not match");
|
||||
}
|
||||
return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem };
|
||||
return resolveIdentity(row);
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
@@ -3,6 +3,10 @@ import { agentHarnessType, agentRunner, paperclipRunnerProfileForHarness, paperc
|
||||
const state = vi.hoisted(() => ({ disabled: [] as string[], overridden: new Set<string>(), settings: {} as Record<string, unknown>, getEnvironment: vi.fn(), bindings: vi.fn(), activeTemplate: vi.fn().mockResolvedValue(null), secretPaths: vi.fn().mockResolvedValue(new Set(["apiKey"])), managedEnvironment: vi.fn(), kubernetesEnvironment: vi.fn(), ensureKubernetesEnvironment: vi.fn(), ssh: vi.fn(), resolveEnvironment: vi.fn(), sandboxQualification: vi.fn(), pluginByKey: vi.fn() }));
|
||||
vi.mock("../adapters/registry.js", () => ({ findActiveServerAdapter: (type: string) => ({ type }), hasActiveAdapterOverride: (type: string) => state.overridden.has(type), waitForExternalAdapters: async () => {} }));
|
||||
vi.mock("./adapter-plugin-store.js", () => ({ getDisabledAdapterTypes: () => state.disabled }));
|
||||
vi.mock("./adapter-settings.js", async (original) => ({
|
||||
...await original(),
|
||||
adapterSettingsService: () => ({ getDisabledTypes: async () => state.disabled }),
|
||||
}));
|
||||
vi.mock("./instance-settings.js", () => ({ instanceSettingsService: () => ({ get: async () => state.settings }) }));
|
||||
vi.mock("./environments.js", () => ({ environmentService: () => ({ getById: state.getEnvironment, listBoundCompanyIds: state.bindings, findManagedSandboxEnvironment: state.managedEnvironment, findKubernetesEnvironment: state.kubernetesEnvironment, ensureKubernetesEnvironment: state.ensureKubernetesEnvironment }) }));
|
||||
vi.mock("./environment-custom-images.js", () => ({ environmentCustomImageService: () => ({ getActiveTemplate: state.activeTemplate }) }));
|
||||
@@ -351,7 +355,7 @@ describe("server-owned Codex runner selection", () => {
|
||||
state.settings = { general: { executionMode: "kubernetes" } };
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company");
|
||||
expect(selected).toEqual({ driver: "sandbox" });
|
||||
expect(agentRunnerAvailability("codex_local", selected)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
|
||||
expect(agentRunnerAvailability("codex_local", selected, state.disabled)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
|
||||
expect(state.kubernetesEnvironment).toHaveBeenCalledWith("company");
|
||||
expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.resolveEnvironment).not.toHaveBeenCalled();
|
||||
@@ -477,14 +481,14 @@ describe("server-owned Codex runner selection", () => {
|
||||
config: { provider: "daytona", snapshot: "replacement-snapshot" }, metadata: { managedByPaperclip: true } });
|
||||
state.resolveEnvironment.mockRejectedValue(new Error("Saved provider secret is unavailable."));
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("paperclip");
|
||||
expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("paperclip");
|
||||
expect(await resolveNewAgentRunnerForCompany({} as never, "company", {
|
||||
adapterType, defaultEnvironmentId: "sandbox", adapterConfig: { model: adapterType === "opencode_local" ? "provider/model" : undefined },
|
||||
})).toMatchObject({ adapterType: "paperclip_runner" });
|
||||
expect(state.sandboxQualification).not.toHaveBeenCalled();
|
||||
expect(state.resolveEnvironment).not.toHaveBeenCalled();
|
||||
state.disabled.push("paperclip_runner");
|
||||
expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("legacy");
|
||||
});
|
||||
it.each(["self-hosted", "custom-environment", "captured-template", "stale-marker"])("keeps qualification for a %s sandbox", async kind => {
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", kind === "self-hosted" ? undefined : JSON.stringify({ ...managedCloudConfig,
|
||||
@@ -495,7 +499,7 @@ describe("server-owned Codex runner selection", () => {
|
||||
metadata: { managedByPaperclip: kind !== "custom-environment" } });
|
||||
state.activeTemplate.mockResolvedValue(kind === "captured-template" ? { id: "template", provider: "daytona", templateRef: "custom-snapshot" } : null);
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", selected).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability("codex_local", selected, state.disabled).defaultRunner).toBe("legacy");
|
||||
expect(state.sandboxQualification).toHaveBeenCalled();
|
||||
});
|
||||
it("ignores a stale captured image and preserves a matching custom image without resolving credentials", async () => {
|
||||
@@ -507,17 +511,17 @@ describe("server-owned Codex runner selection", () => {
|
||||
sourceEnvironmentConfigFingerprint: fingerprint({ ...config, snapshot: "previous-snapshot" }, { excludePaths: ["apiKey"] }) };
|
||||
state.activeTemplate.mockResolvedValue(template);
|
||||
const current = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", current).defaultRunner).toBe("paperclip");
|
||||
expect(agentRunnerAvailability("codex_local", current, state.disabled).defaultRunner).toBe("paperclip");
|
||||
expect(await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "codex_local", runner: "paperclip", defaultEnvironmentId: "sandbox" }))
|
||||
.toMatchObject({ adapterType: "paperclip_runner" });
|
||||
expect(state.sandboxQualification).not.toHaveBeenCalled();
|
||||
state.activeTemplate.mockResolvedValue({ ...template, sourceEnvironmentConfigFingerprint: fingerprint(config, { excludePaths: ["apiKey"] }) });
|
||||
const custom = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", custom).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability("codex_local", custom, state.disabled).defaultRunner).toBe("legacy");
|
||||
expect(state.sandboxQualification).toHaveBeenCalled();
|
||||
state.getEnvironment.mockResolvedValue({ id: "sandbox", driver: "sandbox", status: "active", config: { ...config, timeoutMs: "invalid" }, metadata: { managedByPaperclip: true } });
|
||||
const invalid = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", invalid).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability("codex_local", invalid, state.disabled).defaultRunner).toBe("legacy");
|
||||
expect(state.resolveEnvironment).not.toHaveBeenCalled();
|
||||
});
|
||||
it("checks company access before trusting a managed Cloud image", async () => {
|
||||
|
||||
@@ -5,6 +5,9 @@ import { resolveClaudeModel } from "@paperclipai/adapter-claude-local";
|
||||
import { DEFAULT_GROK_LOCAL_MODEL } from "@paperclipai/adapter-grok-local";
|
||||
import { findActiveServerAdapter, hasActiveAdapterOverride, waitForExternalAdapters } from "../adapters/registry.js";
|
||||
import { getDisabledAdapterTypes } from "./adapter-plugin-store.js";
|
||||
import { adapterSettingsService, effectiveDisabledAdapterTypes } from "./adapter-settings.js";
|
||||
import { listServerAdapters } from "../adapters/registry.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
import { claudeRunnerUnsupportedSettings, PaperclipRunnerProviderProfileError, resolvePaperclipRunnerNativeProviderInput } from "./native-runtime/provider-profile.js";
|
||||
import { forbidden, unprocessable } from "../errors.js";
|
||||
import { runSshCommand } from "@paperclipai/adapter-utils/ssh";
|
||||
@@ -24,6 +27,7 @@ export type RunnerTarget = {
|
||||
export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi";
|
||||
export interface AgentRunnerSelection {
|
||||
adapterType?: string | null;
|
||||
disabledTypes?: string[];
|
||||
adapterConfig?: Record<string, unknown>;
|
||||
runner?: AgentRunnerChoice;
|
||||
target?: RunnerTarget;
|
||||
@@ -32,8 +36,13 @@ export interface AgentRunnerSelection {
|
||||
onConfigurationWarnings?: (warnings: AgentConfigurationWarning[]) => void;
|
||||
}
|
||||
|
||||
function runnerAdaptersAvailable(harness: string, driver: string): boolean {
|
||||
const disabled = new Set(getDisabledAdapterTypes());
|
||||
function localDisabledTypes(): string[] {
|
||||
if (isCloudManagedInstance()) throw new Error("Managed runner selection requires a fresh adapter settings snapshot");
|
||||
return effectiveDisabledAdapterTypes(getDisabledAdapterTypes(), () => listServerAdapters().map((adapter) => adapter.type));
|
||||
}
|
||||
|
||||
function runnerAdaptersAvailable(harness: string, driver: string, disabledTypes: string[]): boolean {
|
||||
const disabled = new Set(disabledTypes);
|
||||
return !!paperclipRunnerProfileForHarness(harness)
|
||||
&& !disabled.has(harness) && !disabled.has("paperclip_runner")
|
||||
&& !!findActiveServerAdapter(harness) && !!findActiveServerAdapter("paperclip_runner")
|
||||
@@ -41,13 +50,13 @@ function runnerAdaptersAvailable(harness: string, driver: string): boolean {
|
||||
&& isEnvironmentDriverSupportedForAdapter("paperclip_runner", driver);
|
||||
}
|
||||
|
||||
export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget()): AgentRunnerAvailability {
|
||||
export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget(), disabledTypes: string[] = localDisabledTypes()): AgentRunnerAvailability {
|
||||
if (target.runnerTasks || target.managedCloudImage) {
|
||||
return runnerAdaptersAvailable(harness, target.driver)
|
||||
return runnerAdaptersAvailable(harness, target.driver, disabledTypes)
|
||||
? { supportedRunners: target.runnerTasks ? ["paperclip"] : ["paperclip", "legacy"], defaultRunner: "paperclip" }
|
||||
: { supportedRunners: ["legacy"], defaultRunner: "legacy" };
|
||||
}
|
||||
const supported = runnerAdaptersAvailable(harness, target.driver)
|
||||
const supported = runnerAdaptersAvailable(harness, target.driver, disabledTypes)
|
||||
&& (target.driver !== "sandbox" || Boolean(target.platform && target.architecture
|
||||
&& (target.platform !== "linux" || (target.libc === "glibc" && target.glibcVersion))))
|
||||
&& (["claude_local", "grok_local"].includes(harness)
|
||||
@@ -166,11 +175,12 @@ export function resolveNewAgentRunner(input: AgentRunnerSelection): { adapterTyp
|
||||
function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; adapterConfig: Record<string, unknown> } {
|
||||
const adapterType = input.adapterType ?? "process";
|
||||
let config = { ...input.adapterConfig };
|
||||
const disabledTypes = input.disabledTypes ?? localDisabledTypes();
|
||||
const harness = agentHarnessType(adapterType, config);
|
||||
const native = adapterType === "paperclip_runner";
|
||||
if (paperclipRunnerProfileForHarness(adapterType) && getDisabledAdapterTypes().includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
|
||||
if (paperclipRunnerProfileForHarness(adapterType) && disabledTypes.includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
|
||||
if (native && (input.runner === undefined || input.runner === "paperclip" || (harness !== "codex_local" && input.runner !== "legacy"))) return { adapterType, adapterConfig: normalizeLegacyRunnerProvider(config) };
|
||||
const availability = agentRunnerAvailability(harness, input.target);
|
||||
const availability = agentRunnerAvailability(harness, input.target, disabledTypes);
|
||||
const choice = input.runner ?? "auto";
|
||||
const useNative = choice === "paperclip" || (choice === "auto" && availability.defaultRunner === "paperclip");
|
||||
if (!useNative) {
|
||||
@@ -318,13 +328,14 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string;
|
||||
}
|
||||
|
||||
export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, input: AgentRunnerSelection & { defaultEnvironmentId?: string | null }) {
|
||||
input = { ...input, disabledTypes: await adapterSettingsService(db).getDisabledTypes() };
|
||||
if (!paperclipRunnerProfileForHarness(agentHarnessType(input.adapterType ?? "process", input.adapterConfig))
|
||||
|| input.runner === "legacy" || (input.adapterType === "paperclip_runner" && (input.runner === undefined || input.runner === "paperclip"))) return resolveNewAgentRunner(input);
|
||||
const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId);
|
||||
const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId, input.disabledTypes);
|
||||
return resolveNewAgentRunner({ ...input, target });
|
||||
}
|
||||
|
||||
export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null): Promise<RunnerTarget> {
|
||||
export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null, disabledTypes?: string[]): Promise<RunnerTarget> {
|
||||
await waitForExternalAdapters();
|
||||
const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
|
||||
let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget()
|
||||
@@ -356,7 +367,9 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
|
||||
const qualified = await readSandboxPlatformQualification(db, environment, companyId);
|
||||
if (qualified) target = { driver, ...qualified };
|
||||
}
|
||||
if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver))) {
|
||||
const disabledForSsh = driver === "ssh" && environment
|
||||
? disabledTypes ?? await adapterSettingsService(db).getDisabledTypes() : [];
|
||||
if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver, disabledForSsh))) {
|
||||
const parsed = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment);
|
||||
if (parsed.driver !== "ssh") throw unprocessable("The selected SSH environment is unavailable.");
|
||||
let output: string;
|
||||
|
||||
@@ -1105,7 +1105,7 @@ export function aiConnectionService(db: Db) {
|
||||
async function refreshQuotaCredential(row: Parameters<typeof credential>[0], failedValue: string, signal: AbortSignal) {
|
||||
return withAccountHomeSecretMutationLock(undefined, row.connection.companyId, async () => {
|
||||
const recovery = quotaCredentialRecovery(row.connection.companyId, row.grant.id);
|
||||
let savingReplacement = false;
|
||||
let replacement: { value: string; secretId: string; version: number; epoch: number; base: string; grantUpdatedAt: number; versionSelector: "latest" | number } | undefined;
|
||||
const save = () => db.transaction(async (tx) => {
|
||||
// Do not wait on a transaction that might itself be waiting for the file
|
||||
// lock. All database locks must be available before consuming OAuth tokens.
|
||||
@@ -1117,11 +1117,20 @@ export function aiConnectionService(db: Db) {
|
||||
if (!grant || grant.status !== "active") throw new Error("credentials_unavailable");
|
||||
const ref = grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential");
|
||||
if (!ref) throw new Error("credentials_unavailable");
|
||||
await tx.select({ id: companySecrets.id }).from(companySecrets).where(and(
|
||||
const [secret] = await tx.select({ id: companySecrets.id, version: companySecrets.latestVersion, epoch: companySecrets.aiSessionEpoch }).from(companySecrets).where(and(
|
||||
eq(companySecrets.id, ref.secretId), eq(companySecrets.companyId, companyId),
|
||||
)).for("update", { noWait: true });
|
||||
if (!secret) throw new Error("credentials_unavailable");
|
||||
const current = await aiConnectionService(tx as unknown as Db).credential({ connection: row.connection, grant });
|
||||
signal.throwIfAborted();
|
||||
if (replacement) {
|
||||
// A lost commit response is reconciled from the authoritative secret.
|
||||
// Reconnect, revocation and a competing saved generation always win.
|
||||
if (current === replacement.value) return current;
|
||||
if (ref.secretId !== replacement.secretId || secret.version !== replacement.version
|
||||
|| secret.epoch !== replacement.epoch || current !== replacement.base
|
||||
|| grant.updatedAt.getTime() !== replacement.grantUpdatedAt
|
||||
|| (ref.versionSelector ?? "latest") !== replacement.versionSelector) return current;
|
||||
} else signal.throwIfAborted();
|
||||
let pending = await recovery.read();
|
||||
if (pending && (pending.secretId !== ref.secretId || pending.connectionId !== row.connection.id || pending.baseHash !== quotaCredentialHash(current))) {
|
||||
// Reconnect may reuse this grant. Once the current authorized token
|
||||
@@ -1141,10 +1150,9 @@ export function aiConnectionService(db: Db) {
|
||||
or(sql`${heartbeatRuns.contextSnapshot}->'aiConnection'->>'grantId' = ${grant.id}`,
|
||||
sql`${agents.runtimeConfig}->'aiConnection'->>'provider' = 'openai'`, eq(agents.adapterType, "codex_local")),
|
||||
)).limit(1);
|
||||
if (active && !pending) throw new Error("provider_unavailable");
|
||||
let refreshed = pending?.value;
|
||||
if (active && !pending && !replacement) throw new Error("provider_unavailable");
|
||||
let refreshed = replacement?.value ?? pending?.value;
|
||||
if (!refreshed) {
|
||||
await recovery.prepare();
|
||||
const auth = JSON.parse(current);
|
||||
const refreshToken = auth.tokens?.refresh_token;
|
||||
if (typeof refreshToken !== "string" || !refreshToken) throw new Error("authentication_required");
|
||||
@@ -1169,12 +1177,12 @@ export function aiConnectionService(db: Db) {
|
||||
...(typeof tokens.refresh_token === "string" && tokens.refresh_token ? { refresh_token: tokens.refresh_token } : {}),
|
||||
...(typeof tokens.id_token === "string" && tokens.id_token ? { id_token: tokens.id_token } : {}),
|
||||
}, last_refresh: new Date().toISOString() });
|
||||
// This fsynced encrypted journal survives transaction rollback and host
|
||||
// restart. Never exchange the consumed token again after a failed save.
|
||||
await recovery.write({ companyId, grantId: grant.id, connectionId: row.connection.id,
|
||||
secretId: ref.secretId, baseHash: quotaCredentialHash(current), value: refreshed });
|
||||
}
|
||||
savingReplacement = true;
|
||||
// Only this logical refresh owns the replacement. Save retries must not
|
||||
// consume another single-use token. Process loss before commit can require
|
||||
// signing in again in v1; no secondary credential vault is created.
|
||||
replacement ??= { value: refreshed, secretId: ref.secretId, version: secret.version, epoch: secret.epoch, base: current,
|
||||
grantUpdatedAt: grant.updatedAt.getTime(), versionSelector: ref.versionSelector ?? "latest" };
|
||||
// Persist the rotated token even if the quota deadline expires after the
|
||||
// successful exchange; losing it would strand all subsequent executions.
|
||||
await secretService(tx).rotate(ref.secretId, { value: refreshed, preserveAiSessionEpoch: true }, { userId: grant.subjectUserId });
|
||||
@@ -1191,9 +1199,9 @@ export function aiConnectionService(db: Db) {
|
||||
await recovery.clear().catch((error) => logger.warn({ err: error, grantId: row.grant.id }, "Saved quota credential recovery cleanup pending"));
|
||||
return value;
|
||||
} catch (error) {
|
||||
if (!savingReplacement || attempt >= 2) throw error;
|
||||
// Saving retries have no provider side effect. Persistent failures
|
||||
// retain the journal for the next quota poll, including after restart.
|
||||
if (!replacement || attempt >= 2) throw error;
|
||||
// Retry only the returned value. Required database reads still fail
|
||||
// closed; exhaustion is an error, never a successful refresh.
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -31,6 +31,8 @@ import { deriveProjectUrlKey } from "@paperclipai/shared";
|
||||
import type {
|
||||
AttentionDecisionVerb,
|
||||
AttentionFeed,
|
||||
AttentionSummary,
|
||||
AttentionTaskIndex,
|
||||
AttentionFeedQuery,
|
||||
AttentionDetailImage,
|
||||
AttentionItem,
|
||||
@@ -163,6 +165,8 @@ type AttentionListOptions = AttentionFeedQuery & {
|
||||
/** Internal-only escape hatch for callers that need one stable, unpaginated feed snapshot. */
|
||||
allowUnscopedAll?: boolean;
|
||||
excludeSources?: readonly AttentionSourceKind[];
|
||||
/** Count the canonical complete set without card bodies or presentation reads. */
|
||||
summaryOnly?: boolean;
|
||||
};
|
||||
|
||||
type AttentionServiceOptions = {
|
||||
@@ -554,7 +558,7 @@ function parseActivityBoundary(value: string | undefined, field: "activitySince"
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function enrichAttentionItems(db: Db, companyId: string, items: AttentionItem[], now: number) {
|
||||
async function enrichAttentionItems(db: Db, companyId: string, items: AttentionItem[], now: number, includePresentation = true) {
|
||||
if (items.length === 0) return items;
|
||||
const sourceIds = [...new Set(items.map((item) => item.subject.id))];
|
||||
const queueRows = await db
|
||||
@@ -605,7 +609,7 @@ async function enrichAttentionItems(db: Db, companyId: string, items: AttentionI
|
||||
...items.map(readMetadataAgentId),
|
||||
...triageRows.map((row) => row.setByAgentId),
|
||||
].filter((value): value is string => Boolean(value)))];
|
||||
const agentNameById = new Map(agentIds.length === 0 ? [] : await db
|
||||
const agentNameById = new Map(!includePresentation || agentIds.length === 0 ? [] : await db
|
||||
.select({ id: agents.id, name: agents.name })
|
||||
.from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), inArray(agents.id, agentIds)))
|
||||
@@ -1173,7 +1177,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
|
||||
OPEN_DECISION_MAX_LIMIT,
|
||||
);
|
||||
return {
|
||||
const service = {
|
||||
list: async (companyId: string, options: AttentionListOptions = {}): Promise<AttentionFeed> => {
|
||||
if (options.all && !options.queue && !options.allowUnscopedAll) {
|
||||
throw badRequest("all requires a queue filter");
|
||||
@@ -1187,6 +1191,9 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
]);
|
||||
const includeDismissed = options.includeDismissed === true;
|
||||
const now = serviceOptions.now?.() ?? Date.now();
|
||||
const summaryOnly = options.summaryOnly === true;
|
||||
const loadImages = (...args: Parameters<typeof issueImageMap>) => summaryOnly
|
||||
? Promise.resolve(new Map<string, AttentionDetailImage[]>()) : issueImageMap(...args);
|
||||
const collected: AttentionItem[] = [];
|
||||
|
||||
const add = (item: AttentionItem) => {
|
||||
@@ -1204,7 +1211,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
status: approvals.status,
|
||||
requestedByAgentId: approvals.requestedByAgentId,
|
||||
requestedByUserId: approvals.requestedByUserId,
|
||||
payload: approvals.payload,
|
||||
payload: summaryOnly ? sql<Record<string, unknown>>`'{}'::jsonb` : approvals.payload,
|
||||
createdAt: approvals.createdAt,
|
||||
updatedAt: approvals.updatedAt,
|
||||
})
|
||||
@@ -1271,7 +1278,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
status: issueThreadInteractions.status,
|
||||
title: issueThreadInteractions.title,
|
||||
summary: issueThreadInteractions.summary,
|
||||
payload: issueThreadInteractions.payload,
|
||||
payload: summaryOnly ? sql<Record<string, unknown>>`jsonb_build_object('target', ${issueThreadInteractions.payload}->'target', 'qualityReview', ${issueThreadInteractions.payload}->'qualityReview')` : issueThreadInteractions.payload,
|
||||
addresseeAgentId: issueThreadInteractions.addresseeAgentId,
|
||||
addresseeUserId: issueThreadInteractions.addresseeUserId,
|
||||
createdByAgentId: issueThreadInteractions.createdByAgentId,
|
||||
@@ -1318,22 +1325,23 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
// A pending human request remains actionable in a persistent chat even
|
||||
// though its idle container is intentionally absent from work queues.
|
||||
issueSummaryMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition, true),
|
||||
issueImageMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
|
||||
planDocumentMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
|
||||
loadImages(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
|
||||
summaryOnly ? Promise.resolve(new Map<string, PlanDocumentSummary>())
|
||||
: planDocumentMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
|
||||
]);
|
||||
|
||||
for (const interaction of visibleInteractionRows) {
|
||||
const issue = interactionIssueMap.get(interaction.issueId) ?? null;
|
||||
if (!issue) continue;
|
||||
const payload = readRecord(interaction.payload);
|
||||
const detail = interactionDetail({
|
||||
const detail = summaryOnly ? null : interactionDetail({
|
||||
kind: interaction.kind,
|
||||
payload,
|
||||
issue,
|
||||
planDocument: interactionPlanDocumentMap.get(interaction.issueId) ?? null,
|
||||
images: issueImages(interactionImageMap, interaction.issueId),
|
||||
});
|
||||
const isPlanTarget = detail.kind === "plan_approval";
|
||||
const isPlanTarget = interaction.kind === "request_confirmation" && isPlanDocumentTarget(payload);
|
||||
const dedupKey = `interaction:${interaction.id}`;
|
||||
add(createItem({
|
||||
companyId,
|
||||
@@ -1380,7 +1388,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
originAgentId: decisions.originAgentId,
|
||||
ruleKey: decisions.ruleKey,
|
||||
title: decisions.title,
|
||||
body: decisions.body,
|
||||
body: summaryOnly ? sql<string>`''` : decisions.body,
|
||||
status: decisions.status,
|
||||
expiresAt: decisions.expiresAt,
|
||||
originIssueId: decisions.originIssueId,
|
||||
@@ -1511,7 +1519,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
recoveryRows.flatMap((row) => [row.sourceIssueId, row.recoveryIssueId]),
|
||||
readCondition,
|
||||
),
|
||||
issueImageMap(db, companyId, recoveryRows.map((row) => row.sourceIssueId), readCondition),
|
||||
loadImages(db, companyId, recoveryRows.map((row) => row.sourceIssueId), readCondition),
|
||||
]);
|
||||
|
||||
for (const recovery of recoveryRows) {
|
||||
@@ -1580,7 +1588,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const [blockedIssueSummaries, terminalBlockerSummaries, blockerImageMap, blockingIssues] = await Promise.all([
|
||||
issueSummaryMap(db, companyId, blockedIssues.map((issue) => issue.id), readCondition),
|
||||
issueSummaryMap(db, companyId, terminalBlockerIssueIds, readCondition),
|
||||
issueImageMap(
|
||||
loadImages(
|
||||
db,
|
||||
companyId,
|
||||
[...blockedIssues.map((issue) => issue.id), ...terminalBlockerIssueIds],
|
||||
@@ -1721,7 +1729,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const [reviewAttentionByIssueId, reviewIssueMap, reviewImageMap] = await Promise.all([
|
||||
issueService(db).listReviewAttention(companyId, reviewRows),
|
||||
issueSummaryMap(db, companyId, reviewIssueIds, readCondition),
|
||||
issueImageMap(db, companyId, reviewIssueIds, readCondition),
|
||||
loadImages(db, companyId, reviewIssueIds, readCondition),
|
||||
]);
|
||||
|
||||
for (const review of reviewRows) {
|
||||
@@ -1787,7 +1795,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
failedIssueIds,
|
||||
readCondition,
|
||||
),
|
||||
issueImageMap(db, companyId, failedIssueIds, readCondition),
|
||||
loadImages(db, companyId, failedIssueIds, readCondition),
|
||||
listFailedRunsWithNewerRun(db, companyId, failedRows.map((row, index) => ({
|
||||
id: row.id,
|
||||
agentId: row.agentId,
|
||||
@@ -1962,7 +1970,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
|
||||
const collectedItems = [...deduped.values()].sort(compareAttentionItems);
|
||||
await decisionQueueService(db).materializeSeededQueues(companyId, collectedItems);
|
||||
const enrichedItems = await enrichAttentionItems(db, companyId, collectedItems, now);
|
||||
const enrichedItems = await enrichAttentionItems(db, companyId, collectedItems, now, !summaryOnly);
|
||||
|
||||
const activitySince = parseActivityBoundary(options.activitySince, "activitySince");
|
||||
const activityUntil = parseActivityBoundary(options.activityUntil, "activityUntil");
|
||||
@@ -1980,6 +1988,15 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
return true;
|
||||
});
|
||||
|
||||
const countsBySourceKind = emptyCounts();
|
||||
for (const item of visibleItems) countsBySourceKind[item.sourceKind] += 1;
|
||||
const summary: AttentionSummary = {
|
||||
companyId, generatedAt: new Date().toISOString(), totalCount: visibleItems.length,
|
||||
deskBadgeCount: visibleItems.filter(item => isNewToday(item, now) || isDecideNow(item, now)).length,
|
||||
countsBySourceKind,
|
||||
};
|
||||
if (summaryOnly) return { ...summary, nextCursor: null, items: visibleItems };
|
||||
|
||||
const sort = options.sort ?? "activity";
|
||||
if (sort !== "activity" && sort !== "decide") throw badRequest("sort must be 'activity' or 'decide'");
|
||||
const selectedComparator = sort === "decide"
|
||||
@@ -2051,22 +2068,23 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
}
|
||||
}
|
||||
}
|
||||
const countsBySourceKind = emptyCounts();
|
||||
for (const item of rankedItems) countsBySourceKind[item.sourceKind] += 1;
|
||||
|
||||
return {
|
||||
companyId,
|
||||
generatedAt: new Date().toISOString(),
|
||||
totalCount: rankedItems.length,
|
||||
// Desk badge: distinct items that surfaced
|
||||
// today OR carry an explicit decide-by deadline due today/past. Counted
|
||||
// over the full ranked set (pre-pagination) so the sidebar badge stays
|
||||
// company-wide accurate even on a small first page.
|
||||
deskBadgeCount: rankedItems.filter((item) => isNewToday(item, now) || isDecideNow(item, now)).length,
|
||||
nextCursor,
|
||||
countsBySourceKind,
|
||||
items,
|
||||
};
|
||||
return { ...summary, nextCursor, items };
|
||||
},
|
||||
};
|
||||
return {
|
||||
...service,
|
||||
summary: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionSummary> => {
|
||||
const { items: _items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
|
||||
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
|
||||
});
|
||||
return summary;
|
||||
},
|
||||
taskIndex: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionTaskIndex> => {
|
||||
const { items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
|
||||
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
|
||||
});
|
||||
return { ...summary, taskIds: [...new Set(items.flatMap(item => item.relatedIssue
|
||||
? [item.relatedIssue.id] : item.subject.kind === "issue" ? [item.subject.id] : []))] };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import type { PaperclipPluginManifestV1, PluginRecord } from "@paperclipai/shared";
|
||||
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import { assertDistributionManifestCapabilities } from "./distribution-plugin-catalog.js";
|
||||
|
||||
export type BundledPluginRegistration = Pick<PluginRecord, "pluginKey" | "packageName" | "version" | "manifestJson">;
|
||||
|
||||
export interface BundledPluginPackageIdentity {
|
||||
kind: "bundled_image";
|
||||
catalogKey: string;
|
||||
pluginKey: string;
|
||||
packageName: string;
|
||||
version: string;
|
||||
digest?: string;
|
||||
}
|
||||
|
||||
export type BundledPluginPackageInspection =
|
||||
| { compatible: true; identity: BundledPluginPackageIdentity }
|
||||
| { compatible: false; pluginKey: string; code: "incompatible_bundled_package" | "missing_manifest_evidence"; reason: string };
|
||||
|
||||
function containedEntrypoint(root: string, declared: unknown, directory: boolean): string {
|
||||
const relative = typeof declared === "string" ? declared.replace(/^\.\//, "").replace(/\/$/, "") : "";
|
||||
if (!relative || path.isAbsolute(relative) || path.win32.isAbsolute(relative) || relative.includes("\\") || relative.split("/").length > 32 || relative.split("/").some((part) => !part || part === "." || part === "..")) {
|
||||
throw new Error("Bundled plugin entrypoint must stay inside its package");
|
||||
}
|
||||
const target = fs.realpathSync(path.join(root, relative));
|
||||
const resolvedRelative = path.relative(root, target);
|
||||
if (!resolvedRelative || resolvedRelative.startsWith("..") || path.isAbsolute(resolvedRelative)) {
|
||||
throw new Error("Bundled plugin entrypoint escapes its package");
|
||||
}
|
||||
const stat = fs.statSync(target);
|
||||
if (directory ? !stat.isDirectory() : !stat.isFile()) throw new Error("Bundled plugin entrypoint is not prebuilt");
|
||||
return relative;
|
||||
}
|
||||
|
||||
export function createBundledPluginPackageResolver(installs: readonly ResolvedBundledPlugin[]) {
|
||||
const byKey = new Map(installs.map((install) => [install.pluginKey, install]));
|
||||
|
||||
function read(install: ResolvedBundledPlugin) {
|
||||
const root = fs.realpathSync(install.localPath);
|
||||
if (root !== path.resolve(install.localPath)) throw new Error("Bundled plugin package root changed after catalog resolution");
|
||||
containedEntrypoint(root, "package.json", false);
|
||||
const metadataPath = path.join(root, "package.json");
|
||||
if (fs.statSync(metadataPath).size > 128 * 1024) throw new Error("Bundled plugin package metadata exceeds verification limits");
|
||||
const pkg = JSON.parse(fs.readFileSync(metadataPath, "utf8"));
|
||||
if (typeof pkg.name !== "string" || typeof pkg.version !== "string" || !pkg.paperclipPlugin) {
|
||||
throw new Error("Bundled plugin package identity or entrypoints are missing");
|
||||
}
|
||||
if (install.distribution && pkg.version !== install.distribution.version) throw new Error("Bundled plugin package version does not match the catalog");
|
||||
const manifest = containedEntrypoint(root, pkg.paperclipPlugin.manifest, false);
|
||||
const worker = containedEntrypoint(root, pkg.paperclipPlugin.worker, false);
|
||||
const ui = pkg.paperclipPlugin.ui === undefined ? undefined : containedEntrypoint(root, pkg.paperclipPlugin.ui, true);
|
||||
if (install.distribution && (worker !== install.distribution.entrypoints.worker || ui !== install.distribution.entrypoints.ui)) {
|
||||
throw new Error("Bundled plugin entrypoints do not match the verified package");
|
||||
}
|
||||
return { root, pkg, manifest, worker, ui };
|
||||
}
|
||||
|
||||
function assertManifest(install: ResolvedBundledPlugin, manifest: PaperclipPluginManifestV1, previousManifest?: PaperclipPluginManifestV1) {
|
||||
const bundle = read(install);
|
||||
if (manifest.id !== install.pluginKey || manifest.version !== bundle.pkg.version) throw new Error("Bundled plugin manifest identity/version does not match the image");
|
||||
const worker = containedEntrypoint(bundle.root, manifest.entrypoints.worker, false);
|
||||
const ui = manifest.entrypoints.ui === undefined ? undefined : containedEntrypoint(bundle.root, manifest.entrypoints.ui, true);
|
||||
if (worker !== bundle.worker || ui !== bundle.ui) throw new Error("Bundled plugin manifest entrypoints do not match the image");
|
||||
assertDistributionManifestCapabilities(manifest);
|
||||
if (previousManifest) {
|
||||
const approved = new Set(previousManifest.capabilities);
|
||||
if (manifest.capabilities.some((capability) => !approved.has(capability))) throw new Error("Bundled plugin capabilities require approval");
|
||||
}
|
||||
return bundle;
|
||||
}
|
||||
|
||||
function resolve(plugin: BundledPluginRegistration): string {
|
||||
const install = byKey.get(plugin.pluginKey);
|
||||
if (!install) throw new Error(`Bundled plugin is absent or not selected in this deployment: ${plugin.pluginKey}`);
|
||||
const bundle = assertManifest(install, plugin.manifestJson);
|
||||
if (plugin.packageName !== bundle.pkg.name || plugin.version !== bundle.pkg.version) throw new Error("Bundled plugin registration identity/version does not match the image");
|
||||
return bundle.root;
|
||||
}
|
||||
|
||||
return {
|
||||
resolve,
|
||||
inspect(plugin: BundledPluginRegistration, verifiedManifest?: PaperclipPluginManifestV1): BundledPluginPackageInspection {
|
||||
try {
|
||||
resolve(plugin);
|
||||
const install = byKey.get(plugin.pluginKey)!;
|
||||
if (!verifiedManifest) return { compatible: false, pluginKey: plugin.pluginKey, code: "missing_manifest_evidence", reason: "Image manifest evidence is required" };
|
||||
assertManifest(install, verifiedManifest);
|
||||
if (!isDeepStrictEqual(verifiedManifest, plugin.manifestJson)) throw new Error("Approved plugin manifest does not match the image");
|
||||
return { compatible: true, identity: {
|
||||
kind: "bundled_image", catalogKey: install.key, pluginKey: plugin.pluginKey,
|
||||
packageName: plugin.packageName, version: plugin.version,
|
||||
...(install.distribution ? { digest: install.distribution.digest } : {}),
|
||||
} };
|
||||
} catch (error) {
|
||||
return { compatible: false, pluginKey: plugin.pluginKey, code: "incompatible_bundled_package", reason: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
},
|
||||
assertActivation(input: { pluginKey?: string; packageRoot: string; manifest?: PaperclipPluginManifestV1; previousManifest?: PaperclipPluginManifestV1 }) {
|
||||
const install = input.pluginKey ? byKey.get(input.pluginKey) : installs.find((candidate) => candidate.localPath === input.packageRoot);
|
||||
if (!install || path.resolve(input.packageRoot) !== install.localPath) throw new Error("Bundled plugin is absent or not selected in this deployment");
|
||||
read(install);
|
||||
if (input.manifest) assertManifest(install, input.manifest, input.previousManifest);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export function resolveBundledPluginRuntimeLoader(
|
||||
moduleDirectory = path.dirname(fileURLToPath(import.meta.url)),
|
||||
): string {
|
||||
let directory = fs.realpathSync(moduleDirectory);
|
||||
let imageRoot: string | undefined;
|
||||
for (let depth = 0; depth < 5; depth++) {
|
||||
const metadataPath = path.join(directory, "package.json");
|
||||
const metadata = fs.lstatSync(metadataPath, { throwIfNoEntry: false });
|
||||
if (metadata?.isFile() && metadata.size <= 128 * 1024
|
||||
&& JSON.parse(fs.readFileSync(metadataPath, "utf8")).name === "@paperclipai/server") {
|
||||
imageRoot = path.dirname(directory);
|
||||
break;
|
||||
}
|
||||
const parent = path.dirname(directory);
|
||||
if (parent === directory) break;
|
||||
directory = parent;
|
||||
}
|
||||
if (!imageRoot) throw new Error("Managed plugin server package is missing from the current image");
|
||||
const declared = path.join(imageRoot, "cli/node_modules/tsx/dist/loader.mjs");
|
||||
let loader: string;
|
||||
try {
|
||||
loader = fs.realpathSync(declared);
|
||||
if (!fs.statSync(loader).isFile()) throw new Error("not a file");
|
||||
} catch {
|
||||
throw new Error("Managed plugin runtime loader is missing from the current image");
|
||||
}
|
||||
const roots = [path.join(imageRoot, "cli"), path.join(imageRoot, "node_modules")];
|
||||
if (!roots.some((root) => {
|
||||
const relative = path.relative(root, loader);
|
||||
return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
|
||||
})) throw new Error("Managed plugin runtime loader escapes the current image");
|
||||
return loader;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { assertDistributionManifestCapabilities, readDistributionPluginCatalog, type DistributionPlugin } from "./distribution-plugin-catalog.js";
|
||||
|
||||
@@ -118,6 +119,13 @@ export const BUNDLED_PLUGIN_CATALOG: readonly BundledPluginCatalogEntry[] = [
|
||||
*/
|
||||
export const SELF_HOSTED_AUTO_INSTALL_KEYS: readonly string[] = ["kubernetes"];
|
||||
|
||||
export function resolveManagedBundledPluginKeys(
|
||||
keys: readonly string[] | null | undefined,
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): readonly string[] | null {
|
||||
return keys ?? (isCloudManagedInstance(env) ? [] : null);
|
||||
}
|
||||
|
||||
export function resolveBundledCatalogRoot(
|
||||
env: Record<string, string | undefined>,
|
||||
): string {
|
||||
@@ -203,7 +211,7 @@ export function resolveBundledPluginInstalls(
|
||||
`bundled plugin "${key}" resolves to "${localPath}", outside the bundled catalog root "${opts.catalogRoot}"; refusing to start`,
|
||||
);
|
||||
}
|
||||
resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath });
|
||||
resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath: opts.enforceCatalogRoot ? canonicalize(localPath) : localPath });
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
@@ -269,6 +277,7 @@ async function reconcileBundledPluginManifest(
|
||||
deps: BundledPluginProvisionerDeps,
|
||||
bundleManifestExists: (localPath: string) => boolean,
|
||||
verifiedManifest?: PaperclipPluginManifestV1,
|
||||
preservePackagePath = false,
|
||||
): Promise<"upgrade_pending" | undefined> {
|
||||
try {
|
||||
// Managed reinstalls take the install path instead; this branch means the
|
||||
@@ -283,7 +292,7 @@ async function reconcileBundledPluginManifest(
|
||||
const approved = new Set(existing.manifestJson.capabilities ?? []);
|
||||
requiresApproval = bundleManifest.capabilities.some((capability) => !approved.has(capability));
|
||||
}
|
||||
const rebindPackage = install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
|
||||
const rebindPackage = !preservePackagePath && install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
|
||||
const refreshDistribution = install.distribution && !isDeepStrictEqual(bundleManifest, existing.manifestJson);
|
||||
if (bundleManifest.version === existing.version && !rebindPackage && !requiresApproval && !refreshDistribution) return;
|
||||
await deps.registry.update(existing.id, {
|
||||
@@ -389,7 +398,7 @@ async function reenableErroredBundledPlugin(
|
||||
export async function ensureBundledPlugins(
|
||||
installs: readonly ResolvedBundledPlugin[],
|
||||
deps: BundledPluginProvisionerDeps,
|
||||
opts: { reinstallUninstalled: boolean },
|
||||
opts: { reinstallUninstalled: boolean; preservePackagePath?: boolean },
|
||||
): Promise<void> {
|
||||
const bundleManifestExists = deps.bundleManifestExists ?? defaultBundleManifestExists;
|
||||
for (const install of installs) {
|
||||
@@ -411,7 +420,7 @@ export async function ensureBundledPlugins(
|
||||
// existing install, because the auto-install below skips a present
|
||||
// plugin. Distribution entries also replace a legacy/npm package path
|
||||
// before loadAll resolves the worker. Configuration and status stay put.
|
||||
const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest);
|
||||
const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest, opts.preservePackagePath);
|
||||
if (reconciledStatus === "upgrade_pending") continue;
|
||||
if (existing.status === "error") {
|
||||
await reenableErroredBundledPlugin(existing, install, deps);
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
|
||||
|
||||
export function cloudBundledPackageDigest(packageRoot: string, options: { catalogRoot?: string } = {}): string {
|
||||
const root = fs.realpathSync(packageRoot);
|
||||
const roots = [{ key: "package", directory: root }];
|
||||
let imageNodeModules: string | undefined;
|
||||
if (options.catalogRoot) {
|
||||
const catalogRoot = fs.realpathSync(options.catalogRoot);
|
||||
const relative = path.relative(catalogRoot, root);
|
||||
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Image package is outside the selected catalog root");
|
||||
imageNodeModules = path.resolve(catalogRoot, "../../node_modules");
|
||||
roots.push(
|
||||
{ key: "plugin-sdk", directory: path.join(catalogRoot, "sdk") },
|
||||
{ key: "shared", directory: path.resolve(catalogRoot, "../shared") },
|
||||
{ key: "image-node-modules", directory: imageNodeModules },
|
||||
);
|
||||
}
|
||||
roots.sort((left, right) => right.directory.length - left.directory.length);
|
||||
const inventory = new Map<string, [string, string, string, number]>();
|
||||
const directories = new Set<string>();
|
||||
const files = new Set<string>();
|
||||
const buffer = Buffer.alloc(65_536);
|
||||
let bytes = 0;
|
||||
let entries = 0;
|
||||
|
||||
function logical(file: string): string {
|
||||
for (const candidate of roots) {
|
||||
const relative = path.relative(candidate.directory, file);
|
||||
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
|
||||
if (relative.split(path.sep).length > 32) throw new Error("Image package exceeds the directory depth limit");
|
||||
return `${candidate.key}/${relative.split(path.sep).join("/")}`;
|
||||
}
|
||||
}
|
||||
throw new Error(options.catalogRoot ? "Image dependency escapes its authorized image roots" : "Image package symlink escapes its package");
|
||||
}
|
||||
|
||||
function follow(target: string) {
|
||||
logical(target);
|
||||
if (imageNodeModules) {
|
||||
const relative = path.relative(imageNodeModules, target).split(path.sep);
|
||||
if ((relative.length === 1 && relative[0] === "")
|
||||
|| (relative[0] === ".pnpm" && (relative.length === 1 || (relative.length === 2 && relative[1] === "node_modules")))) {
|
||||
throw new Error("Image dependency must select an individual package");
|
||||
}
|
||||
if (relative[0] === ".pnpm" && relative.length > 2 && relative[1] && relative[1] !== "node_modules") {
|
||||
walk(path.join(imageNodeModules, ".pnpm", relative[1]));
|
||||
return;
|
||||
}
|
||||
}
|
||||
walk(target);
|
||||
}
|
||||
|
||||
function walk(file: string) {
|
||||
const name = logical(file);
|
||||
const stat = fs.lstatSync(file);
|
||||
if (stat.isSymbolicLink()) {
|
||||
const target = fs.realpathSync(file);
|
||||
inventory.set(name, [name, "link", logical(target), stat.mode & 0o777]);
|
||||
follow(target);
|
||||
} else if (stat.isDirectory()) {
|
||||
const real = fs.realpathSync(file);
|
||||
logical(real);
|
||||
if (directories.has(real)) return;
|
||||
directories.add(real);
|
||||
for (const child of fs.readdirSync(real).sort()) {
|
||||
if (++entries > 50_000) throw new Error("Image package exceeds the entry limit");
|
||||
walk(path.join(real, child));
|
||||
}
|
||||
} else if (stat.isFile()) {
|
||||
const real = fs.realpathSync(file);
|
||||
if (files.has(real)) return;
|
||||
files.add(real);
|
||||
if (path.basename(file) === "package.json" && stat.size > 128 * 1024) throw new Error("Image package metadata exceeds the byte limit");
|
||||
if (bytes + stat.size > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
|
||||
const handle = fs.openSync(file, "r");
|
||||
const hash = createHash("sha256");
|
||||
try {
|
||||
while (true) {
|
||||
const count = fs.readSync(handle, buffer);
|
||||
if (!count) break;
|
||||
bytes += count;
|
||||
if (bytes > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
|
||||
hash.update(buffer.subarray(0, count));
|
||||
}
|
||||
} finally { fs.closeSync(handle); }
|
||||
inventory.set(name, [name, "file", `sha256:${hash.digest("hex")}`, stat.mode & 0o777]);
|
||||
} else throw new Error("Image package contains a special file");
|
||||
}
|
||||
walk(root);
|
||||
if (!inventory.size) throw new Error("Image package is empty");
|
||||
return cloudConfigurationDigest([...inventory.values()].sort((left, right) => left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import { open } from "node:fs/promises";
|
||||
import { z } from "zod";
|
||||
|
||||
function canonical(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(canonical);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, child]) => [key, canonical(child)]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function cloudConfigurationDigest(value: unknown): string {
|
||||
return `sha256:${createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex")}`;
|
||||
}
|
||||
|
||||
export const externalRegistrationEvidenceSchema = z.object({
|
||||
sourceLocatorDigest: z.string(),
|
||||
sourceDigest: z.string(),
|
||||
state: z.enum(["missing", "parsed", "invalid", "unreadable"]),
|
||||
adapterTypes: z.array(z.string()),
|
||||
blocker: z.enum(["unsupported_external_adapters", "invalid_external_registration_metadata", "unreadable_external_registration_metadata"]).nullable(),
|
||||
}).strict();
|
||||
export type ExternalRegistrationEvidence = z.infer<typeof externalRegistrationEvidenceSchema>;
|
||||
|
||||
const legacyRegistrationSchema = z.array(z.object({
|
||||
type: z.string().regex(/^[a-zA-Z0-9_.-]{1,128}$/),
|
||||
packageName: z.string().min(1).max(1024),
|
||||
version: z.string().max(128).optional(),
|
||||
localPath: z.string().max(4096).optional(),
|
||||
disabled: z.boolean().optional(),
|
||||
}).passthrough()).max(100);
|
||||
|
||||
export async function inspectLegacyExternalRegistrations(filePath: string): Promise<ExternalRegistrationEvidence> {
|
||||
const sourceLocatorDigest = cloudConfigurationDigest(filePath);
|
||||
try {
|
||||
const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
let raw: unknown;
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid legacy registration file");
|
||||
const buffer = Buffer.alloc(65_537);
|
||||
let total = 0;
|
||||
while (total < buffer.length) {
|
||||
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
|
||||
if (!bytesRead) break;
|
||||
total += bytesRead;
|
||||
}
|
||||
if (total > 65_536) throw new Error("oversized legacy registration file");
|
||||
raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
const records = legacyRegistrationSchema.safeParse(raw);
|
||||
if (!records.success) {
|
||||
return { sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state: "invalid" }), state: "invalid", adapterTypes: [], blocker: "invalid_external_registration_metadata" };
|
||||
}
|
||||
const metadata = records.data.map((record) => ({ type: record.type, packageName: record.packageName, version: record.version, disabled: record.disabled, hasLocalPath: record.localPath !== undefined }));
|
||||
return {
|
||||
sourceLocatorDigest, sourceDigest: cloudConfigurationDigest(metadata), state: "parsed",
|
||||
adapterTypes: [...new Set(metadata.map((record) => record.type))].sort(),
|
||||
blocker: metadata.length ? "unsupported_external_adapters" : null,
|
||||
};
|
||||
} catch (error) {
|
||||
const absent = (error as NodeJS.ErrnoException).code === "ENOENT";
|
||||
const invalid = error instanceof SyntaxError || (error instanceof Error && /legacy registration file/.test(error.message));
|
||||
const state = absent ? "missing" : invalid ? "invalid" : "unreadable";
|
||||
return {
|
||||
sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state }), state, adapterTypes: [],
|
||||
blocker: absent ? null : invalid ? "invalid_external_registration_metadata" : "unreadable_external_registration_metadata",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { instanceSettings, plugins, type Db } from "@paperclipai/db";
|
||||
import { eq, inArray, ne } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { adapterSettingsService, readLegacyAdapterSettings } from "./adapter-settings.js";
|
||||
import { getAdapterPluginStorePath } from "./adapter-plugin-store.js";
|
||||
import { cloudRuntimeConfigService, readLegacyCloudRuntimeConfig } from "./cloud-runtime-config.js";
|
||||
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
|
||||
import { cloudConfigurationDigest, externalRegistrationEvidenceSchema, inspectLegacyExternalRegistrations } from "./cloud-configuration-evidence.js";
|
||||
import { cloudPackageEvidenceSchema, inspectCloudPackages, qualificationBlockerSchema } from "./cloud-package-qualification.js";
|
||||
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
|
||||
const RECEIPT_KEY = "cloud-configuration-migration:v1";
|
||||
const CONFIGURATION_KEYS = ["adapter-settings:v1", "cloud-runtime-config:v1"];
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type CloudConfigurationDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
|
||||
const sourceSchema = z.object({
|
||||
authority: z.enum(["database", "legacy_file", "default"]), sourceDigest: z.string(),
|
||||
importedRevision: z.number().int().nonnegative(),
|
||||
}).strict();
|
||||
const qualificationSchema = z.object({
|
||||
scope: z.literal("configuration_and_packages"), checkedAt: z.string(),
|
||||
adapterRevision: z.number().int().nonnegative(), configRevision: z.number().int().nonnegative(),
|
||||
configurationDigest: z.string(), packageEvidenceDigest: z.string(),
|
||||
packageInspectionDurationMs: z.number().nonnegative(),
|
||||
expectedPackageEvidenceDigest: z.string().nullable(),
|
||||
configurationPackagesEligible: z.boolean(), samePackageOverlapEligible: z.boolean(),
|
||||
packages: z.array(cloudPackageEvidenceSchema), blockers: z.array(qualificationBlockerSchema),
|
||||
overlapBlockers: z.array(qualificationBlockerSchema),
|
||||
}).strict();
|
||||
const receiptSchema = z.object({
|
||||
version: z.literal(1), revision: z.number().int().nonnegative(),
|
||||
migration: z.object({ completedAt: z.string(), adapter: sourceSchema, config: sourceSchema, externalRegistrations: externalRegistrationEvidenceSchema }).strict(),
|
||||
qualification: qualificationSchema.nullable(), lastPackageMismatch: qualificationSchema.nullable(),
|
||||
}).strict();
|
||||
export type CloudConfigurationReceipt = z.infer<typeof receiptSchema>;
|
||||
|
||||
function parseReceipt(value: unknown): CloudConfigurationReceipt {
|
||||
const result = receiptSchema.safeParse(value);
|
||||
if (!result.success) throw new Error("Persisted Cloud configuration migration receipt is invalid");
|
||||
return result.data;
|
||||
}
|
||||
|
||||
export function cloudConfigurationMigrationService(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv; now?: () => Date } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
const now = () => (options.now?.() ?? new Date()).toISOString();
|
||||
const adapters = adapterSettingsService(db, { runtimeEnv: env });
|
||||
const config = cloudRuntimeConfigService(db);
|
||||
|
||||
function assertManaged() {
|
||||
if (!isCloudManagedInstance(env)) throw new Error("Cloud configuration qualification requires a managed instance");
|
||||
}
|
||||
async function readRow(handle: CloudConfigurationDb, lock = false) {
|
||||
const query = handle.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, RECEIPT_KEY));
|
||||
const [row] = lock ? await query.for("update") : await query;
|
||||
return row ?? null;
|
||||
}
|
||||
async function currentConfiguration(handle: CloudConfigurationDb) {
|
||||
await handle.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
|
||||
.orderBy(instanceSettings.singletonKey).for("update");
|
||||
const adapter = await adapters.inspectStored(handle);
|
||||
const runtime = await config.inspectStored(handle);
|
||||
if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
|
||||
return { adapter, runtime };
|
||||
}
|
||||
async function registrationSnapshot(handle: CloudConfigurationDb, lock = false) {
|
||||
const query = handle.select({
|
||||
pluginKey: plugins.pluginKey, packageName: plugins.packageName, version: plugins.version,
|
||||
manifestJson: plugins.manifestJson, status: plugins.status,
|
||||
}).from(plugins).where(ne(plugins.status, "uninstalled")).orderBy(plugins.pluginKey);
|
||||
return lock ? query.for("share") : query;
|
||||
}
|
||||
async function save(handle: CloudConfigurationDb, id: string, receipt: CloudConfigurationReceipt) {
|
||||
const validated = parseReceipt(receipt);
|
||||
const updated = await handle.update(instanceSettings).set({ general: validated, updatedAt: new Date() }).where(eq(instanceSettings.id, id)).returning({ id: instanceSettings.id });
|
||||
if (updated.length !== 1) throw new Error("Cloud configuration migration receipt was not persisted");
|
||||
return validated;
|
||||
}
|
||||
|
||||
return {
|
||||
async inspectStored(): Promise<CloudConfigurationReceipt | null> {
|
||||
const row = await readRow(db);
|
||||
return row ? parseReceipt(row.general) : null;
|
||||
},
|
||||
async migrate(): Promise<CloudConfigurationReceipt> {
|
||||
assertManaged();
|
||||
return db.transaction(async (tx) => {
|
||||
const inserted = await tx.insert(instanceSettings).values({ singletonKey: RECEIPT_KEY, general: { version: 1 } })
|
||||
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
|
||||
const row = await readRow(tx, true);
|
||||
if (!row) throw new Error("Cloud configuration migration receipt is unavailable");
|
||||
if (!inserted.length) {
|
||||
const receipt = parseReceipt(row.general);
|
||||
await currentConfiguration(tx);
|
||||
return receipt;
|
||||
}
|
||||
await tx.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
|
||||
.orderBy(instanceSettings.singletonKey).for("update");
|
||||
const storedAdapter = await adapters.inspectStored(tx);
|
||||
const storedConfig = await config.inspectStored(tx);
|
||||
const adapterCandidate = storedAdapter ? null : await readLegacyAdapterSettings(env);
|
||||
const configCandidate = storedConfig ? null : await readLegacyCloudRuntimeConfig(env);
|
||||
const adapterSeed = storedAdapter ? null : await adapters.seedIfAbsent({ disabledTypes: adapterCandidate?.disabledTypes ?? [] }, tx);
|
||||
const configSeed = storedConfig ? null : await config.seedIfAbsent(configCandidate, tx);
|
||||
const adapter = storedAdapter ?? adapterSeed!.current;
|
||||
const runtime = storedConfig ?? configSeed!.current;
|
||||
const adapterFromDatabase = Boolean(storedAdapter || !adapterSeed?.seeded);
|
||||
const configFromDatabase = Boolean(storedConfig || !configSeed?.seeded);
|
||||
const receipt: CloudConfigurationReceipt = {
|
||||
version: 1, revision: 0,
|
||||
migration: {
|
||||
completedAt: now(),
|
||||
adapter: { authority: adapterFromDatabase ? "database" : adapterCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(adapterFromDatabase ? { disabledTypes: adapter.disabledTypes } : adapterCandidate), importedRevision: adapter.revision },
|
||||
config: { authority: configFromDatabase ? "database" : configCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(configFromDatabase ? runtime.config : configCandidate), importedRevision: runtime.revision },
|
||||
externalRegistrations: await inspectLegacyExternalRegistrations(getAdapterPluginStorePath(env.PAPERCLIP_HOME)),
|
||||
},
|
||||
qualification: null, lastPackageMismatch: null,
|
||||
};
|
||||
return save(tx, row.id, receipt);
|
||||
});
|
||||
},
|
||||
async recheckLegacyExternalRegistrations(expectedRevision: number): Promise<CloudConfigurationReceipt> {
|
||||
assertManaged();
|
||||
return db.transaction(async (tx) => {
|
||||
const row = await readRow(tx, true);
|
||||
if (!row) throw new Error("Cloud configuration migration has not completed");
|
||||
const receipt = parseReceipt(row.general);
|
||||
if (receipt.revision !== expectedRevision) throw new Error("Cloud configuration migration receipt revision conflict");
|
||||
const filePath = getAdapterPluginStorePath(env.PAPERCLIP_HOME);
|
||||
if (cloudConfigurationDigest(filePath) !== receipt.migration.externalRegistrations.sourceLocatorDigest) {
|
||||
throw new Error("External registration recheck requires the original legacy source location");
|
||||
}
|
||||
const evidence = await inspectLegacyExternalRegistrations(filePath);
|
||||
if (evidence.state !== "parsed") throw new Error("External registration recheck requires a present, valid registration file");
|
||||
receipt.migration.externalRegistrations = evidence;
|
||||
receipt.revision += 1;
|
||||
receipt.qualification = null;
|
||||
return save(tx, row.id, receipt);
|
||||
});
|
||||
},
|
||||
async qualify(input: {
|
||||
installs: readonly ResolvedBundledPlugin[];
|
||||
catalogRoot?: string;
|
||||
loadManifest: (packageRoot: string) => Promise<PaperclipPluginManifestV1 | null>;
|
||||
expectedPackageEvidenceDigest?: string;
|
||||
imageCatalogUnavailable?: boolean;
|
||||
}) {
|
||||
assertManaged();
|
||||
const adapter = await adapters.inspectStored();
|
||||
const runtime = await config.inspectStored();
|
||||
if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
|
||||
const configurationBefore = cloudConfigurationDigest({ adapter, runtime });
|
||||
const registrationsBefore = await registrationSnapshot(db);
|
||||
const evidence = await inspectCloudPackages({ ...input, registrations: registrationsBefore });
|
||||
return db.transaction(async (tx) => {
|
||||
const row = await readRow(tx, true);
|
||||
if (!row) throw new Error("Cloud configuration migration has not completed");
|
||||
const receipt = parseReceipt(row.general);
|
||||
const current = await currentConfiguration(tx);
|
||||
const registrations = await registrationSnapshot(tx, true);
|
||||
const blockers = [...evidence.blockers];
|
||||
if (configurationBefore !== cloudConfigurationDigest(current) || cloudConfigurationDigest(registrationsBefore) !== cloudConfigurationDigest(registrations)) {
|
||||
blockers.push({ code: "configuration_changed_during_inspection", reason: "Configuration or plugin registration changed during image inspection; run qualification again" });
|
||||
}
|
||||
if (input.imageCatalogUnavailable) blockers.push({ code: "current_image_catalog_unavailable", reason: "The current image package catalog could not be verified" });
|
||||
const external = receipt.migration.externalRegistrations;
|
||||
if (external.blocker) blockers.push({ code: external.blocker, reason: "Legacy external adapter registration metadata requires an explicit operator resolution" });
|
||||
const overlapBlockers = input.expectedPackageEvidenceDigest && input.expectedPackageEvidenceDigest !== evidence.packageEvidenceDigest
|
||||
? [{ code: "package_evidence_changed", reason: "Current packages do not match the expected package evidence digest" }] : [];
|
||||
const qualification: z.infer<typeof qualificationSchema> = {
|
||||
scope: "configuration_and_packages", checkedAt: now(),
|
||||
adapterRevision: current.adapter.revision, configRevision: current.runtime.revision,
|
||||
configurationDigest: cloudConfigurationDigest(current), packageEvidenceDigest: evidence.packageEvidenceDigest,
|
||||
packageInspectionDurationMs: evidence.packageInspectionDurationMs,
|
||||
expectedPackageEvidenceDigest: input.expectedPackageEvidenceDigest ?? null,
|
||||
configurationPackagesEligible: blockers.length === 0,
|
||||
samePackageOverlapEligible: Boolean(input.expectedPackageEvidenceDigest && blockers.length === 0 && overlapBlockers.length === 0),
|
||||
packages: evidence.packages, blockers, overlapBlockers,
|
||||
};
|
||||
receipt.revision += 1;
|
||||
receipt.qualification = qualification;
|
||||
if (evidence.blockers.length || input.imageCatalogUnavailable || overlapBlockers.length) receipt.lastPackageMismatch = qualification;
|
||||
await save(tx, row.id, receipt);
|
||||
return { receiptRevision: receipt.revision, ...qualification };
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function migrateCloudConfiguration(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
|
||||
if (!isCloudManagedInstance(options.runtimeEnv ?? process.env)) return null;
|
||||
return cloudConfigurationMigrationService(db, options).migrate();
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { BUNDLED_PLUGIN_CATALOG, resolveBundledCatalogRoot, resolveBundledPluginInstalls, resolveManagedBundledPluginKeys, type ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import { readDistributionPluginCatalog } from "./distribution-plugin-catalog.js";
|
||||
import { createBundledPluginPackageResolver } from "./bundled-plugin-package.js";
|
||||
import { cloudConfigurationMigrationService } from "./cloud-configuration-migration.js";
|
||||
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
|
||||
import { getManagedInstanceConfig } from "./managed-config.js";
|
||||
import { pluginLoader } from "./plugin-loader.js";
|
||||
|
||||
export async function qualifyCloudConfigurationForCurrentImage(db: Db, options: {
|
||||
runtimeEnv?: CloudInstanceEnv;
|
||||
expectedPackageEvidenceDigest?: string;
|
||||
} = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
if (!isCloudManagedInstance(env)) return null;
|
||||
const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
|
||||
const catalogRoot = resolveBundledCatalogRoot(env);
|
||||
let installs: ResolvedBundledPlugin[];
|
||||
try {
|
||||
const distribution = readDistributionPluginCatalog(catalogRoot, BUNDLED_PLUGIN_CATALOG);
|
||||
installs = resolveBundledPluginInstalls(resolveManagedBundledPluginKeys(getManagedInstanceConfig(env)?.plugins.autoInstall, env) ?? [], {
|
||||
catalogRoot, env, enforceCatalogRoot: true, distributionPlugins: distribution,
|
||||
});
|
||||
} catch {
|
||||
return service.qualify({ installs: [], loadManifest: async () => null, imageCatalogUnavailable: true, expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
|
||||
}
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const loader = pluginLoader(db, {
|
||||
prebuiltOnly: true, enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
assertPackageActivation: resolver.assertActivation, resolvePackageRoot: resolver.resolve,
|
||||
});
|
||||
return service.qualify({ installs, catalogRoot, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { z } from "zod";
|
||||
import { createBundledPluginPackageResolver, type BundledPluginRegistration } from "./bundled-plugin-package.js";
|
||||
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import { distributionBundleDigest } from "./distribution-plugin-catalog.js";
|
||||
import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
|
||||
import { cloudBundledPackageDigest } from "./cloud-bundled-package-digest.js";
|
||||
|
||||
export const qualificationBlockerSchema = z.object({
|
||||
code: z.string(), pluginKey: z.string().optional(), reason: z.string(),
|
||||
registeredVersion: z.string().optional(), imageVersion: z.string().optional(),
|
||||
}).strict();
|
||||
export type QualificationBlocker = z.infer<typeof qualificationBlockerSchema>;
|
||||
export const cloudPackageEvidenceSchema = z.object({
|
||||
catalogKey: z.string(), pluginKey: z.string(), packageDigest: z.string().nullable(),
|
||||
imageManifestDigest: z.string().nullable(), approvedManifestDigest: z.string().nullable(),
|
||||
}).strict();
|
||||
export type CloudPackageEvidence = z.infer<typeof cloudPackageEvidenceSchema>;
|
||||
|
||||
export async function inspectCloudPackages(input: {
|
||||
installs: readonly ResolvedBundledPlugin[];
|
||||
registrations: readonly BundledPluginRegistration[];
|
||||
catalogRoot?: string;
|
||||
loadManifest: (packageRoot: string) => Promise<PaperclipPluginManifestV1 | null>;
|
||||
}) {
|
||||
const started = performance.now();
|
||||
const resolver = createBundledPluginPackageResolver(input.installs);
|
||||
const registrations = new Map(input.registrations.map((record) => [record.pluginKey, record]));
|
||||
const selected = new Set(input.installs.map((install) => install.pluginKey));
|
||||
const blockers: QualificationBlocker[] = input.registrations.filter((record) => !selected.has(record.pluginKey)).map((record) => ({
|
||||
code: "unsupported_plugin_registration", pluginKey: record.pluginKey, reason: "An installed plugin is not selected from the current image", registeredVersion: record.version,
|
||||
}));
|
||||
const packages: CloudPackageEvidence[] = [];
|
||||
for (const install of [...input.installs].sort((left, right) => left.pluginKey < right.pluginKey ? -1 : left.pluginKey > right.pluginKey ? 1 : 0)) {
|
||||
const registration = registrations.get(install.pluginKey);
|
||||
const evidence: CloudPackageEvidence = {
|
||||
catalogKey: install.key, pluginKey: install.pluginKey, packageDigest: null,
|
||||
imageManifestDigest: null, approvedManifestDigest: registration ? cloudConfigurationDigest(registration.manifestJson) : null,
|
||||
};
|
||||
packages.push(evidence);
|
||||
let stage = "missing_bundled_package";
|
||||
try {
|
||||
resolver.assertActivation({ packageRoot: install.localPath });
|
||||
stage = "unverified_package_integrity";
|
||||
evidence.packageDigest = install.distribution ? distributionBundleDigest(install.localPath) : cloudBundledPackageDigest(install.localPath, { catalogRoot: input.catalogRoot });
|
||||
if (install.distribution && evidence.packageDigest !== install.distribution.digest) {
|
||||
blockers.push({ code: "package_integrity_mismatch", pluginKey: install.pluginKey, reason: "Current package bytes do not match the verified distribution digest" });
|
||||
continue;
|
||||
}
|
||||
stage = "missing_manifest_evidence";
|
||||
const manifest = await input.loadManifest(install.localPath);
|
||||
if (!manifest) {
|
||||
blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image has no verified plugin manifest" });
|
||||
continue;
|
||||
}
|
||||
evidence.imageManifestDigest = cloudConfigurationDigest(manifest);
|
||||
if (!registration) {
|
||||
blockers.push({ code: "missing_plugin_registration", pluginKey: install.pluginKey, reason: "The current image package is not registered in the database", imageVersion: manifest.version });
|
||||
continue;
|
||||
}
|
||||
const inspected = resolver.inspect(registration, manifest);
|
||||
if (!inspected.compatible) blockers.push({
|
||||
code: inspected.code, pluginKey: install.pluginKey, reason: inspected.reason,
|
||||
registeredVersion: registration.version, imageVersion: manifest.version,
|
||||
});
|
||||
} catch {
|
||||
blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image package could not be verified" });
|
||||
}
|
||||
}
|
||||
const packageEvidenceDigest = cloudConfigurationDigest(packages.map(({ approvedManifestDigest: _approved, ...evidence }) => evidence));
|
||||
return { packages, blockers, packageEvidenceDigest, packageInspectionDurationMs: Math.round(performance.now() - started) };
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import {
|
||||
authConfigSchema,
|
||||
databaseBackupConfigSchema,
|
||||
llmConfigSchema,
|
||||
loggingConfigSchema,
|
||||
paperclipConfigSchema,
|
||||
secretsConfigSchema,
|
||||
secretsLocalEncryptedConfigSchema,
|
||||
serverConfigSchema,
|
||||
storageConfigSchema,
|
||||
storageLocalDiskConfigSchema,
|
||||
storageS3ConfigSchema,
|
||||
type PaperclipConfig,
|
||||
} from "@paperclipai/shared";
|
||||
import { z } from "zod";
|
||||
|
||||
const publicUrl = z.string().url().refine((value) => {
|
||||
const url = new URL(value);
|
||||
return !url.username && !url.password && !url.search && !url.hash;
|
||||
});
|
||||
|
||||
export const cloudRuntimeConfigSchema = z.object({
|
||||
server: serverConfigSchema.strict(),
|
||||
auth: authConfigSchema.extend({ publicBaseUrl: publicUrl.optional() }).strict(),
|
||||
backup: databaseBackupConfigSchema.strict(),
|
||||
storage: storageConfigSchema.extend({
|
||||
localDisk: storageLocalDiskConfigSchema.strict(),
|
||||
s3: storageS3ConfigSchema.extend({ endpoint: publicUrl.optional() }).strict(),
|
||||
}).strict(),
|
||||
secrets: secretsConfigSchema.extend({
|
||||
localEncrypted: secretsLocalEncryptedConfigSchema.strict(),
|
||||
}).strict(),
|
||||
telemetry: z.object({ enabled: z.boolean() }).strict(),
|
||||
logging: loggingConfigSchema.strict(),
|
||||
updates: z.object({ checkEnabled: z.boolean() }).strict().optional(),
|
||||
llm: llmConfigSchema.omit({ apiKey: true }).strict().optional(),
|
||||
}).strict();
|
||||
|
||||
export type CloudRuntimeConfig = z.infer<typeof cloudRuntimeConfigSchema>;
|
||||
export type StoredCloudRuntimeConfig = { version: 1; revision: number; config: CloudRuntimeConfig | null };
|
||||
|
||||
const storedSchema = z.object({
|
||||
version: z.literal(1),
|
||||
revision: z.number().int().nonnegative(),
|
||||
config: cloudRuntimeConfigSchema.nullable(),
|
||||
}).strict();
|
||||
|
||||
export function parseStoredCloudRuntimeConfig(value: unknown): StoredCloudRuntimeConfig {
|
||||
const parsed = storedSchema.safeParse(value);
|
||||
if (!parsed.success || Buffer.byteLength(JSON.stringify(parsed.data), "utf8") > 65_536) {
|
||||
throw new Error("Persisted Cloud runtime configuration is invalid");
|
||||
}
|
||||
const full = parsed.data.config === null ? { success: true } : paperclipConfigSchema.safeParse(toPaperclipConfig(parsed.data.config));
|
||||
if (!full.success) throw new Error("Persisted Cloud runtime configuration is invalid");
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
export function toPaperclipConfig(config: CloudRuntimeConfig | null): PaperclipConfig | null {
|
||||
if (config === null) return null;
|
||||
const { backup, ...settings } = structuredClone(config);
|
||||
return {
|
||||
$meta: { version: 1, updatedAt: "1970-01-01T00:00:00.000Z", source: "configure" },
|
||||
database: {
|
||||
mode: "postgres",
|
||||
embeddedPostgresDataDir: "",
|
||||
embeddedPostgresPort: 54329,
|
||||
backup,
|
||||
},
|
||||
...settings,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectCloudRuntimeConfig(source: PaperclipConfig): CloudRuntimeConfig {
|
||||
const raw = {
|
||||
server: {
|
||||
deploymentMode: source.server.deploymentMode,
|
||||
exposure: source.server.exposure,
|
||||
bind: source.server.bind,
|
||||
customBindHost: source.server.customBindHost,
|
||||
host: source.server.host,
|
||||
port: source.server.port,
|
||||
allowedHostnames: source.server.allowedHostnames,
|
||||
serveUi: source.server.serveUi,
|
||||
},
|
||||
auth: {
|
||||
baseUrlMode: source.auth.baseUrlMode,
|
||||
publicBaseUrl: source.auth.publicBaseUrl,
|
||||
disableSignUp: source.auth.disableSignUp,
|
||||
},
|
||||
backup: {
|
||||
enabled: source.database.backup.enabled,
|
||||
intervalMinutes: source.database.backup.intervalMinutes,
|
||||
retentionDays: source.database.backup.retentionDays,
|
||||
dir: source.database.backup.dir,
|
||||
},
|
||||
storage: {
|
||||
provider: source.storage.provider,
|
||||
localDisk: { baseDir: source.storage.localDisk.baseDir },
|
||||
s3: {
|
||||
bucket: source.storage.s3.bucket,
|
||||
region: source.storage.s3.region,
|
||||
endpoint: source.storage.s3.endpoint,
|
||||
prefix: source.storage.s3.prefix,
|
||||
forcePathStyle: source.storage.s3.forcePathStyle,
|
||||
},
|
||||
},
|
||||
secrets: {
|
||||
provider: source.secrets.provider,
|
||||
strictMode: source.secrets.strictMode,
|
||||
localEncrypted: { keyFilePath: source.secrets.localEncrypted.keyFilePath },
|
||||
},
|
||||
telemetry: { enabled: source.telemetry.enabled },
|
||||
logging: { mode: source.logging.mode, logDir: source.logging.logDir },
|
||||
updates: source.updates ? { checkEnabled: source.updates.checkEnabled } : undefined,
|
||||
llm: source.llm ? { provider: source.llm.provider } : undefined,
|
||||
};
|
||||
return parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config: raw }).config!;
|
||||
}
|
||||
|
||||
let snapshot: StoredCloudRuntimeConfig | null = null;
|
||||
|
||||
export function setCloudRuntimeConfigSnapshot(value: StoredCloudRuntimeConfig | null): void {
|
||||
snapshot = value ? parseStoredCloudRuntimeConfig(value) : null;
|
||||
}
|
||||
|
||||
export function getCloudRuntimeConfigSnapshot(): StoredCloudRuntimeConfig {
|
||||
if (!snapshot) throw new Error("Cloud runtime configuration has not been initialized");
|
||||
return structuredClone(snapshot);
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { constants } from "node:fs";
|
||||
import { open } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { instanceSettings, type Db } from "@paperclipai/db";
|
||||
import { paperclipConfigSchema } from "@paperclipai/shared";
|
||||
import { resolvePaperclipConfigPathForInstance } from "@paperclipai/shared/home-paths";
|
||||
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
|
||||
import {
|
||||
parseStoredCloudRuntimeConfig,
|
||||
projectCloudRuntimeConfig,
|
||||
setCloudRuntimeConfigSnapshot,
|
||||
type CloudRuntimeConfig,
|
||||
type StoredCloudRuntimeConfig,
|
||||
} from "./cloud-runtime-config-snapshot.js";
|
||||
|
||||
export const CLOUD_RUNTIME_CONFIG_KEY = "cloud-runtime-config:v1";
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type CloudRuntimeConfigDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
|
||||
|
||||
export function cloudRuntimeConfigService(db: CloudRuntimeConfigDb) {
|
||||
async function inspectStored(handle: CloudRuntimeConfigDb = db): Promise<StoredCloudRuntimeConfig | null> {
|
||||
const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).limit(1);
|
||||
return row ? parseStoredCloudRuntimeConfig(row.general) : null;
|
||||
}
|
||||
|
||||
async function seedIfAbsent(config: CloudRuntimeConfig | null, handle: CloudRuntimeConfigDb = db) {
|
||||
const next = parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config });
|
||||
const inserted = await handle.insert(instanceSettings)
|
||||
.values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: next })
|
||||
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
|
||||
const current = await inspectStored(handle);
|
||||
if (!current) throw new Error("Cloud runtime configuration seed was not persisted");
|
||||
return { seeded: inserted.length > 0, current };
|
||||
}
|
||||
|
||||
return {
|
||||
inspectStored,
|
||||
seedIfAbsent,
|
||||
async replace(config: CloudRuntimeConfig | null, expectedRevision: number, handle: CloudRuntimeConfigDb = db) {
|
||||
const validated = parseStoredCloudRuntimeConfig({ version: 1, revision: expectedRevision, config });
|
||||
return handle.transaction(async (tx) => {
|
||||
const [row] = await tx.select().from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).for("update");
|
||||
if (!row) throw new Error("Cloud runtime configuration has not been initialized");
|
||||
const current = parseStoredCloudRuntimeConfig(row.general);
|
||||
if (current.revision !== expectedRevision) throw new Error("Cloud runtime configuration revision conflict");
|
||||
const next = { ...validated, revision: current.revision + 1 };
|
||||
await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
|
||||
return next;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function legacyCloudRuntimeConfigPath(env: CloudInstanceEnv = process.env): string {
|
||||
return env.PAPERCLIP_CONFIG
|
||||
? path.resolve(env.PAPERCLIP_CONFIG)
|
||||
: resolvePaperclipConfigPathForInstance({ homeDir: env.PAPERCLIP_HOME, instanceId: env.PAPERCLIP_INSTANCE_ID });
|
||||
}
|
||||
|
||||
export async function readLegacyCloudRuntimeConfig(env: CloudInstanceEnv = process.env): Promise<CloudRuntimeConfig | null> {
|
||||
let raw: unknown;
|
||||
try {
|
||||
const file = await open(legacyCloudRuntimeConfigPath(env), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid file");
|
||||
const buffer = Buffer.alloc(65_537);
|
||||
let total = 0;
|
||||
while (total < buffer.length) {
|
||||
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
|
||||
if (bytesRead === 0) break;
|
||||
total += bytesRead;
|
||||
}
|
||||
if (total > 65_536) throw new Error("oversized file");
|
||||
raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw new Error("Legacy Cloud runtime configuration could not be read safely");
|
||||
return null;
|
||||
}
|
||||
const parsed = paperclipConfigSchema.safeParse(raw);
|
||||
if (!parsed.success) throw new Error("Legacy Cloud runtime configuration is invalid");
|
||||
if (parsed.data.llm?.provider === "openai" && parsed.data.llm.apiKey?.trim() && !env.OPENAI_API_KEY?.trim()) {
|
||||
throw new Error("Move the legacy OpenAI API key to OPENAI_API_KEY before initializing Cloud runtime configuration");
|
||||
}
|
||||
return projectCloudRuntimeConfig(parsed.data);
|
||||
}
|
||||
|
||||
export async function initializeCloudRuntimeConfig(db: CloudRuntimeConfigDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
if (!isCloudManagedInstance(env)) return null;
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
const service = cloudRuntimeConfigService(db);
|
||||
const current = await service.inspectStored();
|
||||
const result = current
|
||||
? { seeded: false, current }
|
||||
: await service.seedIfAbsent(await readLegacyCloudRuntimeConfig(env));
|
||||
setCloudRuntimeConfigSnapshot(result.current);
|
||||
return result;
|
||||
}
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
import { deriveBundleState } from "./agent-instructions.js";
|
||||
import { readInstructionBytes } from "./agent-instruction-files.js";
|
||||
import { skillPackageStore } from "./skill-package-store.js";
|
||||
import { getRunLogStore } from "./run-log-store.js";
|
||||
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
|
||||
|
||||
// This is a reviewed catalog of readers, not a proxy for existing GET routes.
|
||||
// Select credentials out of queries; never load encrypted identity records.
|
||||
@@ -311,9 +311,9 @@ export async function readCustomerSuccessResource(
|
||||
}
|
||||
if (q.operation === "runs.log") {
|
||||
const run = await ownRow(tx, schema.heartbeatRuns, q.companyId!, q.resourceId!);
|
||||
if (!run.logRef || run.logStore !== "local_file") return unavailable("run_log_unavailable");
|
||||
if (!run.logRef || !isRunLogStoreType(run.logStore)) return unavailable("run_log_unavailable");
|
||||
const result = await getRunLogStore().read(
|
||||
{ store: "local_file", logRef: String(run.logRef) },
|
||||
{ store: run.logStore, logRef: String(run.logRef) },
|
||||
{
|
||||
offset: q.range?.start ?? q.offset ?? 0,
|
||||
limitBytes: q.range ? q.range.end - q.range.start + 1 : 256000,
|
||||
|
||||
@@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
|
||||
|
||||
/** Topics for existing durable queues; register intent before their writes. */
|
||||
export const DELIVERY_QUEUES = {
|
||||
runEvidenceMigration: "run-evidence-migration",
|
||||
providerTraceRetention: "provider-trace-retention",
|
||||
nativeRunnerEvidenceRetention: "native-runner-evidence-retention",
|
||||
skillVersionRetention: "skill-version-retention",
|
||||
databaseBackups: "database-backups",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming } from "./execution-preparation-timing.js";
|
||||
import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming, withRunResponsePresentationTiming } from "./execution-preparation-timing.js";
|
||||
import { ROOT_CONTEXT, trace, SpanStatusCode, type Context } from "@opentelemetry/api";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
@@ -45,6 +45,7 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async
|
||||
try {
|
||||
await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);
|
||||
await expect(withRunWorkspaceFinalizationTiming("run", () => 42)).resolves.toBe(42);
|
||||
await expect(withRunResponsePresentationTiming("run", () => 42)).resolves.toBe(42);
|
||||
for (const phase of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission", "execution.budget_reservation",
|
||||
"skills.inventory", "skills.inventory.wait", "skills.cache.acquire", "skills.version.materialize", "skills.consume",
|
||||
"workspace_sync.seed_read_copy", "workspace_sync.seed_read_hash", "workspace_sync.seed_read_touch"] as const) {
|
||||
@@ -69,7 +70,10 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async
|
||||
}
|
||||
});
|
||||
|
||||
it("parents concurrent deferred copyback phases to their own finalization roots and preserves failure", async () => {
|
||||
it.each([
|
||||
[withRunWorkspaceFinalizationTiming, "task.workspace_finalization", "workspace_sync.restore"],
|
||||
[withRunResponsePresentationTiming, "task.response_presentation", "response.transaction"],
|
||||
] as const)("parents concurrent deferred phases to their own %s root and preserves failure", async (withTiming, rootName, phase) => {
|
||||
const spans: Array<{ name: string; attributes: Record<string, unknown>; context: Context;
|
||||
span: { end: ReturnType<typeof vi.fn>; setStatus: ReturnType<typeof vi.fn> } }> = [];
|
||||
handles.startActiveSpan.mockImplementation((name, options, context, callback) => {
|
||||
@@ -77,16 +81,16 @@ it("parents concurrent deferred copyback phases to their own finalization roots
|
||||
spans.push({ name, attributes: options.attributes, context, span });
|
||||
return callback(span);
|
||||
});
|
||||
await Promise.all(["first", "second"].map(runId => withRunWorkspaceFinalizationTiming(runId, async () => {
|
||||
await Promise.all(["first", "second"].map(runId => withTiming(runId, async () => {
|
||||
await Promise.resolve();
|
||||
await measureRunPhase("workspace_sync.restore", async () => {});
|
||||
await measureRunPhase(phase, async () => {});
|
||||
})));
|
||||
for (const runId of ["first", "second"]) {
|
||||
const hash = createHash("sha256").update(runId).digest("hex").slice(0, 12);
|
||||
const owned = spans.filter(span => span.attributes["paperclip.task.run.run_id"] === hash);
|
||||
expect(owned).toHaveLength(2);
|
||||
const root = owned.find(span => span.name === "task.workspace_finalization")!;
|
||||
const child = owned.find(span => span.name === "workspace_sync.restore")!;
|
||||
const root = owned.find(span => span.name === rootName)!;
|
||||
const child = owned.find(span => span.name === phase)!;
|
||||
expect(root.context).toBe(ROOT_CONTEXT);
|
||||
expect(trace.getSpan(child.context)).toBe(root.span);
|
||||
expect(root.span.end).toHaveBeenCalledOnce();
|
||||
@@ -94,7 +98,7 @@ it("parents concurrent deferred copyback phases to their own finalization roots
|
||||
}
|
||||
const original = new Error("private finalization failure");
|
||||
const work = vi.fn(() => { throw original; });
|
||||
await expect(withRunWorkspaceFinalizationTiming("failed", work)).rejects.toBe(original);
|
||||
await expect(withTiming("failed", work)).rejects.toBe(original);
|
||||
expect(work).toHaveBeenCalledOnce();
|
||||
expect(spans.at(-1)!.span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
|
||||
expect(spans.at(-1)!.span.end).toHaveBeenCalledOnce();
|
||||
|
||||
@@ -199,6 +199,14 @@ export function withRunWorkspaceFinalizationTiming<T>(runId: string, work: () =>
|
||||
return measure(runId, "task.workspace_finalization", work, true);
|
||||
}
|
||||
|
||||
export function withRunResponsePresentationTiming<T>(runId: string, work: () => T | Promise<T>): Promise<T> {
|
||||
const owned = ownership.getStore();
|
||||
if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) {
|
||||
return (async () => work())();
|
||||
}
|
||||
return measure(runId, "task.response_presentation", work, true);
|
||||
}
|
||||
|
||||
export async function measureRunPhase<T>(phase: ExecutionPhase, work: () => T | Promise<T>, counts?: { fileCount?: number; bytes?: number }): Promise<T> {
|
||||
const owned = ownership.getStore();
|
||||
if (!endpointEnabled() || !owned || owned.ended) return work();
|
||||
|
||||
@@ -47,7 +47,8 @@ import {
|
||||
sanitizeFeedbackValue,
|
||||
sha256Digest,
|
||||
} from "./feedback-redaction.js";
|
||||
import { getRunLogStore } from "./run-log-store.js";
|
||||
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
|
||||
import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js";
|
||||
import { getOperatorSettingDefaults } from "./setting-defaults.js";
|
||||
|
||||
const FEEDBACK_SCHEMA_VERSION = "paperclip-feedback-envelope-v2";
|
||||
@@ -318,17 +319,18 @@ async function findMatchingFile(
|
||||
return search(rootDir, 0);
|
||||
}
|
||||
|
||||
async function readFullRunLog(run: {
|
||||
async function readFullRunLog(db: Db, run: {
|
||||
logStore: string | null;
|
||||
logRef: string | null;
|
||||
}) {
|
||||
if (run.logStore !== "local_file" || !run.logRef) return null;
|
||||
if (!isRunLogStoreType(run.logStore) || !run.logRef) return null;
|
||||
const store = getRunLogStore();
|
||||
if (run.logStore === "s3_segments") await recoverSettledNativeRunLog(db, store, run.logRef).catch(() => undefined);
|
||||
let offset = 0;
|
||||
let combined = "";
|
||||
|
||||
while (true) {
|
||||
const result = await store.read({ store: "local_file", logRef: run.logRef }, {
|
||||
const result = await store.read({ store: run.logStore, logRef: run.logRef }, {
|
||||
offset,
|
||||
limitBytes: 512_000,
|
||||
}).catch(() => null);
|
||||
@@ -1526,7 +1528,7 @@ async function buildFeedbackTraceBundleFromRow(
|
||||
.from(heartbeatRunEvents)
|
||||
.where(eq(heartbeatRunEvents.runId, run.id))
|
||||
.orderBy(asc(heartbeatRunEvents.seq));
|
||||
const logText = await readFullRunLog(run);
|
||||
const logText = await readFullRunLog(db, run);
|
||||
const logEntries = parseRunLogEntries(logText);
|
||||
const stdoutText = logEntries
|
||||
.filter((entry) => entry.stream === "stdout")
|
||||
|
||||
@@ -26,7 +26,10 @@ import { createHeartbeatPreparationTiming } from "./heartbeat/preparation-timing
|
||||
import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkingCopy } from "./heartbeat/instruction-working-copy.js";
|
||||
import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js";
|
||||
import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js";
|
||||
import { createRunLogWriterAuthority } from "./heartbeat/run-log-authority.js";
|
||||
import { createHeartbeatRunOutput } from "./heartbeat/run-output.js";
|
||||
import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery.js";
|
||||
import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability";
|
||||
import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js";
|
||||
import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js";
|
||||
import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js";
|
||||
@@ -371,7 +374,7 @@ import {
|
||||
type NativeSessionBackend,
|
||||
} from "../vendor/paperclip-runner/index.js";
|
||||
|
||||
import { getRunLogStore } from "./run-log-store.js";
|
||||
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
|
||||
import {
|
||||
providerTraceStore,
|
||||
} from "./provider-trace-store.js";
|
||||
@@ -2090,6 +2093,7 @@ export function heartbeatService(
|
||||
}
|
||||
try {
|
||||
providerTraceCapture = await traceStore.prepare({
|
||||
assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner),
|
||||
runId: run.id,
|
||||
companyId: run.companyId,
|
||||
provider:
|
||||
@@ -3131,6 +3135,8 @@ export function heartbeatService(
|
||||
const runOutput = createHeartbeatRunOutput(db, {
|
||||
run, runId, issueId, runLogStore, identityRedactor, secretKeys,
|
||||
appendRunEvent, getCurrentUserRedactionOptions,
|
||||
assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner),
|
||||
runtimeLogSecretValues: observabilitySecretValues(observability),
|
||||
});
|
||||
const goalCheckpointSession: {
|
||||
current: {
|
||||
@@ -3216,7 +3222,7 @@ export function heartbeatService(
|
||||
message: "run started",
|
||||
});
|
||||
|
||||
const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent } =
|
||||
const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onPersistedNativeLog, onAdapterMeta, onAdapterEvent } =
|
||||
await timePreparation("run_output", () => runOutput.begin(currentRun));
|
||||
if (runScopedMentionedSkillKeys.length > 0) {
|
||||
await onLog(
|
||||
@@ -3450,6 +3456,7 @@ export function heartbeatService(
|
||||
identityRedactor,
|
||||
appendIdentityRedactedLog,
|
||||
onLog,
|
||||
onPersistedNativeLog,
|
||||
onAdapterEvent,
|
||||
onAdapterMeta,
|
||||
},
|
||||
@@ -4072,6 +4079,9 @@ export function heartbeatService(
|
||||
companyId: string;
|
||||
logStore: string | null;
|
||||
logRef: string | null;
|
||||
runtimeMode?: string | null;
|
||||
status?: string;
|
||||
logBytes?: number | null;
|
||||
},
|
||||
opts?: { offset?: number; limitBytes?: number; alignStartToRecord?: boolean; signal?: AbortSignal },
|
||||
) => {
|
||||
@@ -4084,12 +4094,14 @@ export function heartbeatService(
|
||||
if (!run) throw notFound("Heartbeat run not found");
|
||||
if (!run.logStore || !run.logRef) throw notFound("Run log not found");
|
||||
|
||||
const result = await runLogStore.read(
|
||||
if (!isRunLogStoreType(run.logStore)) throw notFound("Run log backend not supported");
|
||||
const result = await readRecoverableRunLog(db, runLogStore,
|
||||
{
|
||||
store: run.logStore as "local_file",
|
||||
store: run.logStore,
|
||||
logRef: run.logRef,
|
||||
},
|
||||
opts,
|
||||
run,
|
||||
);
|
||||
|
||||
return {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
|
||||
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agents, agentWakeupRequests, caseHandoffs, cases, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
@@ -158,6 +160,57 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
|
||||
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object));
|
||||
});
|
||||
|
||||
it("settles the accepted native exact rate-card price without replacing it with zero", async () => {
|
||||
const f = await fixture({ runtimeMode: "native" });
|
||||
const runnerInstanceId = randomUUID();
|
||||
const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId,
|
||||
provider: { kind: "codex", model: "gpt-6.1-sol" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
|
||||
const event = (sourceSeq: number, eventType: string, payload: Record<string, unknown>): PrpEvent => ({
|
||||
sourceKind: "runner", sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `rate-card-${sourceSeq}`,
|
||||
turnId: "price-turn", eventType, emittedAt: new Date().toISOString(), payload,
|
||||
} as PrpEvent);
|
||||
await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true,
|
||||
runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } }));
|
||||
await delivery.capture(event(2, "turn.completed", {}));
|
||||
const accepted = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id });
|
||||
expect(accepted?.costStatus).toBe("estimated");
|
||||
const exact = accepted!.costUsdExact;
|
||||
f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...accepted, usageComplete: accepted!.complete };
|
||||
await f.completion.completeRun(f.input);
|
||||
const saved = await read(f.run.id);
|
||||
expect(saved).toMatchObject({ status: "succeeded", costAccountingPending: false, costAccountedAt: expect.any(Date),
|
||||
usageJson: { costUsdExact: exact, costStatus: "estimated" } });
|
||||
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
|
||||
expect(ledger).toHaveLength(1);
|
||||
expect(ledger[0].costCents).toBe(100.2);
|
||||
});
|
||||
|
||||
it("retains newer accepted native usage when completion consumed an older receipt", async () => {
|
||||
const f = await fixture({ runtimeMode: "native", usageJson: { sessionRotated: true, configFreshness: { version: 0 } } });
|
||||
const runnerInstanceId = randomUUID();
|
||||
const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId,
|
||||
provider: { kind: "codex", model: "test-model" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
|
||||
const event = (sourceSeq: number, eventType: string, inputTokens = 100): PrpEvent => ({
|
||||
sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `racing-${sourceSeq}`, turnId: "turn-1",
|
||||
emittedAt: new Date().toISOString(), eventType, payload: eventType === "usage.reported"
|
||||
? { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens, outputTokens: 10, providerCostUsd: inputTokens / 1000 } } : {},
|
||||
} as PrpEvent);
|
||||
await delivery.capture(event(1, "usage.reported"));
|
||||
await delivery.capture(event(2, "turn.completed"));
|
||||
const acceptedA = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id });
|
||||
await delivery.capture(event(3, "usage.reported", 120));
|
||||
f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...acceptedA, usageComplete: acceptedA!.complete };
|
||||
await f.completion.completeRun(f.input);
|
||||
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", costAccountingPending: true, costAccountedAt: null,
|
||||
usageJson: { inputTokens: 120, costUsd: 0.12, accountingReceiptSequence: 3, accountingReceiptReady: false, sessionRotated: false, configFreshness: { version: 1 } } });
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0);
|
||||
await delivery.capture(event(4, "turn.completed", 120));
|
||||
expect(await (await import("../run-cost-accounting.js")).accountRunCost(db, f.run.id)).toBe(true);
|
||||
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
|
||||
expect(ledger).toHaveLength(1);
|
||||
expect(ledger[0].costCents).toBe(12);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ timedOut: true, expected: "timed_out", errorCode: "timeout" },
|
||||
{ signal: "SIGTERM", expected: "failed", errorCode: "adapter_failed" },
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
|
||||
import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js";
|
||||
import { measureRunPhase } from "../execution-preparation-timing.js";
|
||||
import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js";
|
||||
@@ -686,7 +687,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null,
|
||||
),
|
||||
|
||||
usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },
|
||||
usageJson: preserveAcceptedRunUsageProjectionSql({ ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) }),
|
||||
finishedAt:
|
||||
persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,
|
||||
updatedAt: new Date(),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyExternalConversationSettlement } from "../external-conversation-settlement-notifications.js";
|
||||
@@ -1202,6 +1203,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
.set({
|
||||
status,
|
||||
...patch,
|
||||
...(patch?.usageJson ? { usageJson: preserveAcceptedRunUsageProjectionSql(patch.usageJson) } : {}),
|
||||
executionStatusDeliveryId: randomUUID(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { legacyControllerBootId } from "../legacy-controller-lease.js";
|
||||
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
|
||||
|
||||
export function createRunLogWriterAuthority(
|
||||
db: Db,
|
||||
admitted: typeof heartbeatRuns.$inferSelect,
|
||||
nativeLeaseOwner?: string,
|
||||
) {
|
||||
return async () => {
|
||||
const [run] = await db
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
controllerBootId: heartbeatRuns.controllerBootId,
|
||||
leaseExpiresAt: heartbeatRuns.controllerLeaseExpiresAt,
|
||||
logStore: heartbeatRuns.logStore,
|
||||
logRef: heartbeatRuns.logRef,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.id, admitted.id),
|
||||
eq(heartbeatRuns.companyId, admitted.companyId),
|
||||
eq(heartbeatRuns.agentId, admitted.agentId),
|
||||
eq(heartbeatRuns.status, "running"),
|
||||
),
|
||||
);
|
||||
if (!run) throw new Error("run_log_writer_authority_lost");
|
||||
if (run.runtimeMode !== "native") {
|
||||
if (
|
||||
run.controllerBootId !== legacyControllerBootId ||
|
||||
!run.leaseExpiresAt ||
|
||||
run.leaseExpiresAt <= new Date()
|
||||
)
|
||||
throw new Error("run_log_writer_authority_lost");
|
||||
return;
|
||||
}
|
||||
const [owner] = await db
|
||||
.select({
|
||||
leaseOwner: nativeRunFinalizations.leaseOwner,
|
||||
bootId: nativeRunFinalizations.controllerBootId,
|
||||
expiresAt: nativeRunFinalizations.leaseExpiresAt,
|
||||
})
|
||||
.from(nativeRunFinalizations)
|
||||
.where(
|
||||
and(
|
||||
eq(nativeRunFinalizations.runId, admitted.id),
|
||||
eq(nativeRunFinalizations.companyId, admitted.companyId),
|
||||
),
|
||||
);
|
||||
|
||||
if (!owner && !run.logStore && !run.logRef && !admitted.logStore && !admitted.logRef) return;
|
||||
const identity = await currentNativeControllerIdentity();
|
||||
if (
|
||||
!owner ||
|
||||
owner.bootId !== identity.bootId ||
|
||||
!owner.expiresAt ||
|
||||
owner.expiresAt <= new Date() ||
|
||||
(nativeLeaseOwner !== undefined && owner.leaseOwner !== nativeLeaseOwner)
|
||||
)
|
||||
throw new Error("run_log_writer_authority_lost");
|
||||
};
|
||||
}
|
||||
@@ -7,6 +7,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createDurableRunLogStore } from "../run-log-store.js";
|
||||
import type { StorageProvider } from "../../storage/types.js";
|
||||
import { createHeartbeatRunOutput, isNewRunLog } from "./run-output.js";
|
||||
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
import { formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js";
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
|
||||
vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
|
||||
|
||||
@@ -45,15 +48,24 @@ function createRunRowDb(row: Partial<Run>) {
|
||||
return {
|
||||
update: () => ({
|
||||
set: (values: Partial<Run>) => ({
|
||||
where: async () => { Object.assign(row, values); },
|
||||
where: async () => {
|
||||
Object.assign(row, values);
|
||||
},
|
||||
}),
|
||||
}),
|
||||
} as unknown as Db;
|
||||
}
|
||||
|
||||
const newRunRow = {
|
||||
id: "run-1", companyId: "company-1", agentId: "agent-1", status: "running",
|
||||
logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0, lastOutputAt: null,
|
||||
id: "run-1",
|
||||
companyId: "company-1",
|
||||
agentId: "agent-1",
|
||||
status: "running",
|
||||
logStore: null,
|
||||
logRef: null,
|
||||
logBytes: null,
|
||||
lastOutputSeq: 0,
|
||||
lastOutputAt: null,
|
||||
} as Partial<Run>;
|
||||
|
||||
async function beginAttempt(store: ReturnType<typeof createDurableRunLogStore>, row: Partial<Run>) {
|
||||
@@ -81,11 +93,26 @@ afterEach(async () => {
|
||||
|
||||
describe("heartbeat run output log recovery", () => {
|
||||
it("treats only a run row with no recorded log or output as new", () => {
|
||||
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(true);
|
||||
expect(isNewRunLog({ logStore: "local_file", logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);
|
||||
expect(isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);
|
||||
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(false);
|
||||
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(false);
|
||||
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
isNewRunLog({
|
||||
logStore: "local_file",
|
||||
logRef: "c/a/r.ndjson",
|
||||
logBytes: null,
|
||||
lastOutputSeq: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 }),
|
||||
).toBe(false);
|
||||
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("makes no remote log recovery request for a new run", async () => {
|
||||
@@ -116,8 +143,121 @@ describe("heartbeat run output log recovery", () => {
|
||||
await second.onLog("stdout", "attempt two\n");
|
||||
await restartedStore.finalize(second.output.handle!);
|
||||
|
||||
const lines = objects.get(row.logRef!)!.toString().trim().split("\n").map((line) => JSON.parse(line));
|
||||
const lines = objects
|
||||
.get(row.logRef!)!
|
||||
.toString()
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
expect(lines.map((line) => line.chunk)).toEqual(["attempt one\n", "attempt two\n"]);
|
||||
expect(lines.map((line) => line.seq)).toEqual([1, 2]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("object-backed diagnostic recording", () => {
|
||||
it("applies identity redaction to a native key-bearing event before S3 admission", async () => {
|
||||
const privateKeyPem = generateKeyPairSync("ed25519")
|
||||
.privateKey.export({ type: "pkcs8", format: "pem" })
|
||||
.toString();
|
||||
const row = { ...newRunRow, runtimeMode: "native" } as Run;
|
||||
const db = createRunRowDb(row) as Db & { select: () => unknown };
|
||||
db.select = (() => ({
|
||||
from: () => ({ where: () => ({ orderBy: () => ({ limit: async () => [] }) }) }),
|
||||
})) as typeof db.select;
|
||||
const chunks: string[] = [];
|
||||
const output = createHeartbeatRunOutput(db, {
|
||||
run: row,
|
||||
runId: row.id,
|
||||
issueId: null,
|
||||
secretKeys: new Set(),
|
||||
identityRedactor: createAgentIdentityRedactor(privateKeyPem),
|
||||
runLogStore: {
|
||||
begin: async () => ({
|
||||
store: "s3_segments",
|
||||
logRef: "company/agent/run",
|
||||
initialSourceCursor: 0,
|
||||
}),
|
||||
append: async () => 0,
|
||||
enqueue: async (_handle, event) => {
|
||||
chunks.push(event.chunk);
|
||||
return { committed: Promise.resolve(100) };
|
||||
},
|
||||
flush: async () => undefined,
|
||||
},
|
||||
appendRunEvent: vi.fn(),
|
||||
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
|
||||
});
|
||||
const handlers = await output.begin(row);
|
||||
await handlers.onPersistedNativeLog!(
|
||||
1,
|
||||
formatNativeRunLogEvent({
|
||||
schema: "paperclip.prp.event.v1",
|
||||
schemaVersion: 1,
|
||||
runId: row.id,
|
||||
sourceEventId: "source:1",
|
||||
sourceInstanceId: "source",
|
||||
sourceKind: "runner",
|
||||
sourceSeq: 1,
|
||||
normalizedSessionId: "session",
|
||||
eventType: "harness.diagnostic",
|
||||
priority: 1,
|
||||
emittedAt: new Date().toISOString(),
|
||||
payload: { text: privateKeyPem },
|
||||
}),
|
||||
);
|
||||
expect(chunks).toHaveLength(1);
|
||||
expect(chunks[0]).not.toContain(privateKeyPem.split("\n")[1]!);
|
||||
expect(chunks[0]).toContain("REDACTED");
|
||||
});
|
||||
it("admits a burst without serial network waits, publishes after receipts, and coalesces progress", async () => {
|
||||
const { publishLiveEvent } = await import("../live-events.js");
|
||||
vi.mocked(publishLiveEvent).mockClear();
|
||||
const row = { ...newRunRow };
|
||||
const writes: Array<Partial<Run>> = [];
|
||||
const db = {
|
||||
update: () => ({
|
||||
set: (values: Partial<Run>) => ({
|
||||
where: async () => {
|
||||
writes.push(values);
|
||||
Object.assign(row, values);
|
||||
},
|
||||
}),
|
||||
}),
|
||||
} as unknown as Db;
|
||||
const commit: Array<() => void> = [];
|
||||
const recorder = createHeartbeatRunOutput(db, {
|
||||
run: row as Run,
|
||||
runId: row.id!,
|
||||
issueId: null,
|
||||
runLogStore: {
|
||||
begin: async () => ({
|
||||
store: "s3_segments",
|
||||
logRef: "company/agent/run",
|
||||
initialBytes: 0,
|
||||
initialSequence: 10,
|
||||
}),
|
||||
append: () => new Promise<number>((resolve) => commit.push(() => resolve(100))),
|
||||
},
|
||||
identityRedactor: { chunk: (_stream, chunk) => chunk, redact: (value) => value },
|
||||
secretKeys: new Set(),
|
||||
appendRunEvent: vi.fn(),
|
||||
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
|
||||
});
|
||||
const { onLog } = await recorder.begin(row as Run);
|
||||
for (let i = 0; i < 100; i++) await onLog("stdout", `line ${i}\n`);
|
||||
expect(commit).toHaveLength(100);
|
||||
expect(publishLiveEvent).not.toHaveBeenCalled();
|
||||
for (const resolve of commit) resolve();
|
||||
await vi.waitFor(() =>
|
||||
expect(
|
||||
vi
|
||||
.mocked(publishLiveEvent)
|
||||
.mock.calls.filter(([event]) => event.type === "heartbeat.run.log"),
|
||||
).toHaveLength(100),
|
||||
);
|
||||
await recorder.flushOutputProgress({ force: true });
|
||||
expect(row.lastOutputSeq).toBe(110);
|
||||
expect(row.lastOutputBytes).toBe(10000);
|
||||
expect(writes.filter((write) => write.lastOutputSeq !== undefined)).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -5,9 +5,15 @@ import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../l
|
||||
import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
import { touchHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js";
|
||||
import { publishLiveEvent } from "../live-events.js";
|
||||
import type { RunLogHandle, RunLogStore } from "../run-log-store.js";
|
||||
import { isRunLogStoreType, type RunLogHandle, type RunLogStore } from "../run-log-store.js";
|
||||
import { appendExcerpt, compactRunLogChunk } from "./run-log.js";
|
||||
import { isHeartbeatRunRuntimeStatusActive, publishHeartbeatRunRuntimeProgress, type createHeartbeatLifecycle } from "./run-lifecycle.js";
|
||||
import { createNativeRunLogExporter } from "../native-runtime/native-run-log-export.js";
|
||||
import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js";
|
||||
import {
|
||||
isHeartbeatRunRuntimeStatusActive,
|
||||
publishHeartbeatRunRuntimeProgress,
|
||||
type createHeartbeatLifecycle,
|
||||
} from "./run-lifecycle.js";
|
||||
|
||||
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
|
||||
export const ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS = 60 * 1000;
|
||||
@@ -19,11 +25,13 @@ export interface HeartbeatRunOutputInput {
|
||||
run: Run;
|
||||
runId: string;
|
||||
issueId: string | null;
|
||||
runLogStore: Pick<RunLogStore, "begin" | "append">;
|
||||
runLogStore: Pick<RunLogStore, "begin" | "append" | "enqueue" | "flush">;
|
||||
identityRedactor: Pick<ReturnType<typeof createAgentIdentityRedactor>, "chunk" | "redact">;
|
||||
secretKeys: Set<string>;
|
||||
appendRunEvent: ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];
|
||||
getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;
|
||||
assertWriterAuthority?: () => Promise<void>;
|
||||
runtimeLogSecretValues?: readonly string[];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -35,15 +43,26 @@ export interface HeartbeatRunOutputInput {
|
||||
export function isNewRunLog(
|
||||
run: Pick<Run, "logStore" | "logRef" | "logBytes" | "lastOutputSeq">,
|
||||
): boolean {
|
||||
return run.logStore == null
|
||||
&& run.logRef == null
|
||||
&& run.logBytes == null
|
||||
&& Number(run.lastOutputSeq ?? 0) === 0;
|
||||
return (
|
||||
run.logStore == null &&
|
||||
run.logRef == null &&
|
||||
run.logBytes == null &&
|
||||
Number(run.lastOutputSeq ?? 0) === 0
|
||||
);
|
||||
}
|
||||
|
||||
/** One recorder per execution attempt; the log store itself stays service-scoped. */
|
||||
export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) {
|
||||
const { run, runId, issueId, runLogStore, identityRedactor, secretKeys, appendRunEvent, getCurrentUserRedactionOptions } = input;
|
||||
const {
|
||||
run,
|
||||
runId,
|
||||
issueId,
|
||||
runLogStore,
|
||||
identityRedactor,
|
||||
secretKeys,
|
||||
appendRunEvent,
|
||||
getCurrentUserRedactionOptions,
|
||||
} = input;
|
||||
let handle: RunLogHandle | null = null;
|
||||
let stdoutExcerpt = "";
|
||||
let stderrExcerpt = "";
|
||||
@@ -59,7 +78,14 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
|
||||
} | null;
|
||||
} = { pending: null };
|
||||
let persistedLogBytes = Number(run.logBytes ?? 0);
|
||||
let progressWrite: Promise<void> | null = null;
|
||||
let recordingFailureReported = false;
|
||||
const flushOutputProgress = async (opts?: { force?: boolean }) => {
|
||||
if (progressWrite) {
|
||||
await progressWrite;
|
||||
if (opts?.force) return flushOutputProgress(opts);
|
||||
return;
|
||||
}
|
||||
const pendingOutputProgress = outputProgressState.pending;
|
||||
if (!pendingOutputProgress) return;
|
||||
const shouldFlush =
|
||||
@@ -68,7 +94,7 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
|
||||
pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >=
|
||||
ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS;
|
||||
if (!shouldFlush) return;
|
||||
await db
|
||||
progressWrite = db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
lastOutputAt: pendingOutputProgress.at,
|
||||
@@ -77,111 +103,181 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
|
||||
lastOutputBytes: pendingOutputProgress.bytes,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, run.id));
|
||||
lastOutputFlushAt = pendingOutputProgress.at;
|
||||
outputProgressState.pending = null;
|
||||
.where(eq(heartbeatRuns.id, run.id))
|
||||
.then(() => {
|
||||
lastOutputFlushAt = pendingOutputProgress.at;
|
||||
if (outputProgressState.pending === pendingOutputProgress)
|
||||
outputProgressState.pending = null;
|
||||
})
|
||||
.finally(() => {
|
||||
progressWrite = null;
|
||||
});
|
||||
await progressWrite;
|
||||
};
|
||||
// Bind handlers to the run row persisted at execution start. Progress writes
|
||||
// above use only the immutable run ID from the initial row.
|
||||
async function begin(currentRun: Run) {
|
||||
const run = currentRun;
|
||||
if (run.logStore != null && !isRunLogStoreType(run.logStore))
|
||||
throw new Error("Run log backend is not supported");
|
||||
handle = await runLogStore.begin({
|
||||
companyId: run.companyId,
|
||||
agentId: run.agentId,
|
||||
runId,
|
||||
newRun: isNewRunLog(run),
|
||||
assertWriterAuthority: input.assertWriterAuthority,
|
||||
previousStore: run.logStore ?? undefined,
|
||||
});
|
||||
|
||||
outputSeq = handle.initialSequence ?? outputSeq;
|
||||
persistedLogBytes = handle.initialBytes ?? persistedLogBytes;
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
logStore: handle.store,
|
||||
logRef: handle.logRef,
|
||||
...(handle.store === "s3_segments" ? { logBytes: null, logSha256: null } : {}),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, runId));
|
||||
|
||||
const currentUserRedactionOptions =
|
||||
await getCurrentUserRedactionOptions();
|
||||
const appendIdentityRedactedLog = async (stream: "stdout" | "stderr", chunk: string) => {
|
||||
const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
|
||||
if (
|
||||
handle.store === "s3_segments" &&
|
||||
run.runtimeMode === "native" &&
|
||||
input.runtimeLogSecretValues?.length
|
||||
) {
|
||||
const registry = createRunSecretRedactionRegistry(db);
|
||||
for (const value of new Set(input.runtimeLogSecretValues))
|
||||
await registry.register(run.companyId, runId, value);
|
||||
}
|
||||
const reportRecordingFailure = async () => {
|
||||
if (recordingFailureReported) return;
|
||||
recordingFailureReported = true;
|
||||
await appendRunEvent(currentRun, {
|
||||
eventType: "lifecycle",
|
||||
stream: "system",
|
||||
level: "warn",
|
||||
message: "Run log persistence is incomplete",
|
||||
payload: { reason: "object_log_persistence_failed" },
|
||||
}).catch(() => undefined);
|
||||
};
|
||||
const appendIdentityRedactedLog = async (
|
||||
stream: "stdout" | "stderr",
|
||||
chunk: string,
|
||||
waitForCommit = true,
|
||||
sourceCursor?: number,
|
||||
) => {
|
||||
const sanitizedChunk = compactRunLogChunk(
|
||||
redactCurrentUserText(chunk, currentUserRedactionOptions),
|
||||
);
|
||||
if (stream === "stdout")
|
||||
stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
|
||||
if (stream === "stderr")
|
||||
stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
|
||||
if (stream === "stdout") stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
|
||||
if (stream === "stderr") stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
|
||||
const ts = new Date().toISOString();
|
||||
|
||||
outputSeq += 1;
|
||||
const chunkSeq = outputSeq;
|
||||
let appendedBytes = 0;
|
||||
let committed: Promise<number> = Promise.resolve(0);
|
||||
if (handle) {
|
||||
appendedBytes = await runLogStore.append(handle, {
|
||||
const event = {
|
||||
stream,
|
||||
chunk: sanitizedChunk,
|
||||
ts,
|
||||
seq: chunkSeq,
|
||||
});
|
||||
sourceCursor,
|
||||
};
|
||||
committed = runLogStore.enqueue
|
||||
? (await runLogStore.enqueue(handle, event)).committed
|
||||
: runLogStore.append(handle, event);
|
||||
}
|
||||
const recorded = committed.then(async (appendedBytes) => {
|
||||
persistedLogBytes += appendedBytes;
|
||||
}
|
||||
outputProgressState.pending = {
|
||||
at: new Date(ts),
|
||||
seq: chunkSeq,
|
||||
stream,
|
||||
bytes: persistedLogBytes,
|
||||
};
|
||||
await flushOutputProgress();
|
||||
|
||||
// Streamed CLI output is real run activity: keep the in-memory
|
||||
// runtime status ("Working... / X ago") fresh between structured
|
||||
// events so sandbox runs with mid-run log streaming never show a
|
||||
// minutes-stale timestamp. Throttled to avoid churning the live
|
||||
// event stream on every 250ms tail chunk.
|
||||
const logActivityAt = new Date(ts);
|
||||
if (
|
||||
isHeartbeatRunRuntimeStatusActive(run.status) &&
|
||||
logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >=
|
||||
ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS
|
||||
) {
|
||||
lastLogRuntimeStatusTouchMs = logActivityAt.getTime();
|
||||
const touchedStatus = touchHeartbeatRunRuntimeStatus({
|
||||
companyId: run.companyId,
|
||||
issueId,
|
||||
agentId: run.agentId,
|
||||
runId: run.id,
|
||||
at: logActivityAt,
|
||||
});
|
||||
if (touchedStatus)
|
||||
publishHeartbeatRunRuntimeProgress(touchedStatus);
|
||||
}
|
||||
|
||||
const payloadChunk =
|
||||
sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
|
||||
? sanitizedChunk.slice(
|
||||
sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES,
|
||||
)
|
||||
: sanitizedChunk;
|
||||
|
||||
publishLiveEvent({
|
||||
companyId: run.companyId,
|
||||
type: "heartbeat.run.log",
|
||||
payload: {
|
||||
runId: run.id,
|
||||
agentId: run.agentId,
|
||||
issueId,
|
||||
ts,
|
||||
outputProgressState.pending = {
|
||||
at: new Date(ts),
|
||||
seq: chunkSeq,
|
||||
stream,
|
||||
chunk: payloadChunk,
|
||||
logRef: handle?.logRef,
|
||||
logStore: handle?.store,
|
||||
truncated: payloadChunk.length !== sanitizedChunk.length,
|
||||
},
|
||||
bytes: persistedLogBytes,
|
||||
};
|
||||
await flushOutputProgress();
|
||||
|
||||
// Streamed CLI output is real run activity: keep the in-memory
|
||||
// runtime status ("Working... / X ago") fresh between structured
|
||||
// events so sandbox runs with mid-run log streaming never show a
|
||||
// minutes-stale timestamp. Throttled to avoid churning the live
|
||||
// event stream on every 250ms tail chunk.
|
||||
const logActivityAt = new Date(ts);
|
||||
if (
|
||||
isHeartbeatRunRuntimeStatusActive(run.status) &&
|
||||
logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >=
|
||||
ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS
|
||||
) {
|
||||
lastLogRuntimeStatusTouchMs = logActivityAt.getTime();
|
||||
const touchedStatus = touchHeartbeatRunRuntimeStatus({
|
||||
companyId: run.companyId,
|
||||
issueId,
|
||||
agentId: run.agentId,
|
||||
runId: run.id,
|
||||
at: logActivityAt,
|
||||
});
|
||||
if (touchedStatus) publishHeartbeatRunRuntimeProgress(touchedStatus);
|
||||
}
|
||||
|
||||
const payloadChunk =
|
||||
sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
|
||||
? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
|
||||
: sanitizedChunk;
|
||||
|
||||
publishLiveEvent({
|
||||
companyId: run.companyId,
|
||||
type: "heartbeat.run.log",
|
||||
payload: {
|
||||
runId: run.id,
|
||||
agentId: run.agentId,
|
||||
issueId,
|
||||
ts,
|
||||
seq: chunkSeq,
|
||||
stream,
|
||||
chunk: payloadChunk,
|
||||
logRef: handle?.logRef,
|
||||
logStore: handle?.store,
|
||||
truncated: payloadChunk.length !== sanitizedChunk.length,
|
||||
},
|
||||
});
|
||||
});
|
||||
if (waitForCommit) await recorded;
|
||||
else void recorded.catch(reportRecordingFailure);
|
||||
};
|
||||
const onLog = (stream: "stdout" | "stderr", chunk: string) =>
|
||||
appendIdentityRedactedLog(stream, identityRedactor.chunk(stream, chunk));
|
||||
const onLog = (stream: "stdout" | "stderr", chunk: string) => {
|
||||
const external = handle?.store === "s3_segments";
|
||||
const recorded = appendIdentityRedactedLog(
|
||||
stream,
|
||||
identityRedactor.chunk(stream, chunk),
|
||||
!external,
|
||||
);
|
||||
return external ? recorded.catch(reportRecordingFailure) : recorded;
|
||||
};
|
||||
const nativeExporter =
|
||||
handle.store === "s3_segments" && run.runtimeMode === "native"
|
||||
? createNativeRunLogExporter({
|
||||
db,
|
||||
companyId: run.companyId,
|
||||
runId,
|
||||
afterCursor: handle.initialSourceCursor ?? 0,
|
||||
enqueue: (cursor, chunk) =>
|
||||
appendIdentityRedactedLog(
|
||||
"stdout",
|
||||
identityRedactor.chunk("stdout", chunk),
|
||||
false,
|
||||
cursor,
|
||||
),
|
||||
flush: () => runLogStore.flush!(handle!),
|
||||
})
|
||||
: undefined;
|
||||
if (nativeExporter) await nativeExporter.recover();
|
||||
const onPersistedNativeLog = nativeExporter
|
||||
? (cursor: number, chunk: string) =>
|
||||
nativeExporter.record(cursor, chunk).catch(reportRecordingFailure)
|
||||
: undefined;
|
||||
const onAdapterMeta = async (meta: AdapterInvocationMeta) => {
|
||||
meta = identityRedactor.redact(meta);
|
||||
if (meta.env && secretKeys.size > 0) {
|
||||
@@ -214,16 +310,29 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
|
||||
});
|
||||
};
|
||||
|
||||
return { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent };
|
||||
return {
|
||||
currentUserRedactionOptions,
|
||||
appendIdentityRedactedLog,
|
||||
onLog,
|
||||
onPersistedNativeLog,
|
||||
onAdapterMeta,
|
||||
onAdapterEvent,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
begin,
|
||||
get handle() { return handle; },
|
||||
get handle() {
|
||||
return handle;
|
||||
},
|
||||
outputProgressState,
|
||||
flushOutputProgress,
|
||||
// Finalization awaits I/O and can receive more output before persisting excerpts.
|
||||
get stdoutExcerpt() { return stdoutExcerpt; },
|
||||
get stderrExcerpt() { return stderrExcerpt; },
|
||||
get stdoutExcerpt() {
|
||||
return stdoutExcerpt;
|
||||
},
|
||||
get stderrExcerpt() {
|
||||
return stderrExcerpt;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -355,6 +355,9 @@ const heartbeatRunLogAccessColumns = {
|
||||
...heartbeatRunReadAccessColumns,
|
||||
logStore: heartbeatRuns.logStore,
|
||||
logRef: heartbeatRuns.logRef,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
status: heartbeatRuns.status,
|
||||
logBytes: heartbeatRuns.logBytes,
|
||||
} as const;
|
||||
|
||||
const heartbeatRunIssueControlColumns = {
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
|
||||
|
||||
import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
|
||||
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { agents, budgetPolicies, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
@@ -213,6 +216,38 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
|
||||
expectCleanup(input);
|
||||
});
|
||||
|
||||
it("returns the accepted exact native rate-card price and preserves its live budget hold", async () => {
|
||||
const { input, run } = await fixture(true);
|
||||
input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home };
|
||||
input.runtime.nativeExecution!.provider = { kind: "codex", model: "gpt-6.1-sol" };
|
||||
await db.insert(budgetPolicies).values({ companyId: run.companyId, scopeType: "agent", scopeId: input.agent.id,
|
||||
metric: "billed_cents", windowKind: "lifetime", amount: 100, notifyEnabled: false });
|
||||
let exact: string | undefined;
|
||||
vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => {
|
||||
expect(native.onUsage).toBeUndefined();
|
||||
const delivery = await createNativeUsageDelivery(db, { companyId: run.companyId, runId: run.id,
|
||||
runnerInstanceId: native.runnerInstanceId, provider: native.execution.provider,
|
||||
billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
|
||||
const event = (sourceSeq: number, eventType: string, payload: Record<string, unknown>): PrpEvent => ({
|
||||
sourceKind: "runner", sourceInstanceId: native.runnerInstanceId, runId: run.id, sourceSeq, sourceEventId: `priced-${sourceSeq}`,
|
||||
turnId: "priced-turn", eventType, emittedAt: new Date().toISOString(), payload,
|
||||
} as PrpEvent);
|
||||
await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true,
|
||||
runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } }));
|
||||
await delivery.capture(event(2, "turn.completed", {}));
|
||||
const saved = (await read(run.id)).usageJson!;
|
||||
exact = saved.costUsdExact as string;
|
||||
expect(Number(exact)).toBe(1.002);
|
||||
expect(saved.budgetExposure).toMatchObject({ costCents: "100.2000000" });
|
||||
expect((await db.select().from(agents).where(eq(agents.id, input.agent.id)))[0].lifecycleHolds).toContain("budget");
|
||||
return { exitCode: 0, signal: null, timedOut: false, usageComplete: true, costUsd: null, model: "gpt-6.1-sol" };
|
||||
});
|
||||
const result = await executeHeartbeatRuntime(db, input);
|
||||
if (!result.dispatched) throw new Error("expected native dispatch");
|
||||
expect(result.adapterResult).toMatchObject({ costUsdExact: exact, costStatus: "estimated", usageComplete: true,
|
||||
pricingProvenance: { source: "rate_card" }, usage: { inputTokens: 1000, outputTokens: 100000 } });
|
||||
});
|
||||
|
||||
it.each(["expected", "wrong"])("finalizes a plugin worktree through its owner (branch: %s)", async branch => {
|
||||
const { input } = await fixture(true);
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
|
||||
|
||||
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import { writeExecutionControl } from "../execution-control-notifications.js";
|
||||
import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js";
|
||||
@@ -34,7 +36,7 @@ import {
|
||||
NativeWorkspaceFinalizationOwnershipLostError,
|
||||
type NativeWorkspaceFinalizationOwnership,
|
||||
} from "../native-runtime/native-workspace-finalization-ownership.js";
|
||||
import { createRunUsageRecorder } from "../usage-receipts.js";
|
||||
import { createRunUsageRecorder, readAcceptedRunUsageCheckpoint } from "../usage-receipts.js";
|
||||
import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";
|
||||
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
|
||||
import { isConversation } from "../agent-conversations.js";
|
||||
@@ -173,6 +175,7 @@ export interface HeartbeatRuntimeExecutionInput {
|
||||
>;
|
||||
};
|
||||
output: Pick<AdapterExecutionContext, "onLog"> & {
|
||||
onPersistedNativeLog?: (cursor: number, chunk: string) => Promise<void>;
|
||||
onAdapterEvent: NonNullable<AdapterExecutionContext["onEvent"]>;
|
||||
onAdapterMeta: AdapterExecutionContext["onMeta"];
|
||||
identityRedactor: ReturnType<typeof createAgentIdentityRedactor>;
|
||||
@@ -496,9 +499,13 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
adapterFinalizeOutcome = status;
|
||||
};
|
||||
|
||||
const usageRecorder = await measureRunPhase("execution.usage_startup", () =>
|
||||
const producerOwnsUsage = usesNativeUsageDelivery({
|
||||
runnerd: nativeRuntimeResolution.kind === "native" && agent.adapterType === "paperclip_runner",
|
||||
remote: executionTarget?.kind === "remote", provider: nativeExecution?.provider,
|
||||
});
|
||||
const usageRecorder = producerOwnsUsage ? undefined : await measureRunPhase("execution.usage_startup", () =>
|
||||
createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }));
|
||||
input.effects.onUsageCaptureReady(usageRecorder.persistFailure);
|
||||
input.effects.onUsageCaptureReady(usageRecorder?.persistFailure ?? (async () => {}));
|
||||
let adapterResult: AdapterExecutionResult;
|
||||
const runGoalControlRequestId = readNonEmptyString(
|
||||
context.goalControlRequestId,
|
||||
@@ -630,10 +637,11 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
goalCheckpointSession.current = { params, displayId };
|
||||
},
|
||||
onLog,
|
||||
onPersistedNativeLog: input.output.onPersistedNativeLog,
|
||||
onEvent: onAdapterEvent,
|
||||
instructionWorkingCopy: nativeInstructionWorkingCopy(),
|
||||
|
||||
onUsage: async receipt => { await usageRecorder.capture(receipt); },
|
||||
onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
|
||||
preparationSpans: nativeRunnerPreparationSpans,
|
||||
// Bootstrap with executable/home discovery while keeping
|
||||
// configured provider values and the server-selected
|
||||
@@ -835,7 +843,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
onLog,
|
||||
onMeta: onAdapterMeta,
|
||||
onEvent: onAdapterEvent,
|
||||
onUsage: async receipt => { await usageRecorder.capture(receipt); },
|
||||
onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
|
||||
onExecutionPhase: executionControl.phases.enter,
|
||||
startupTraceContext: getStartupTraceContext(),
|
||||
onRuntimeProgress: async (progress) => {
|
||||
@@ -914,8 +922,14 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave };
|
||||
|
||||
if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) {
|
||||
const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult));
|
||||
adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
|
||||
if (usageRecorder) {
|
||||
const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult));
|
||||
adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
|
||||
} else {
|
||||
const accepted = await measureRunPhase("execution.usage", () => readAcceptedRunUsageCheckpoint(db, { companyId: run.companyId, runId: run.id }));
|
||||
adapterResult = { ...adapterResult, ...accepted, usageComplete: accepted?.complete ?? false,
|
||||
...(!accepted ? { costStatus: "unpriced" as const } : {}) };
|
||||
}
|
||||
} else {
|
||||
// Stop may already own the terminal result. Preserve its metadata
|
||||
// while durably recording the proof needed to release admission.
|
||||
|
||||
@@ -189,7 +189,8 @@ import {
|
||||
resolveIssueGoalId,
|
||||
resolveNextIssueGoalId,
|
||||
} from "./issue-goal-fallback.js";
|
||||
import { getRunLogStore } from "./run-log-store.js";
|
||||
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
|
||||
import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js";
|
||||
import { getDefaultCompanyGoal } from "./goals.js";
|
||||
import { assertAssignableAgent } from "./agent-assignability.js";
|
||||
import {
|
||||
@@ -6712,13 +6713,14 @@ export async function readIssueCommentRunLogText(run: {
|
||||
logStore: string | null;
|
||||
logRef: string | null;
|
||||
logBytes: number | null;
|
||||
}) {
|
||||
if (run.logStore !== "local_file" || !run.logRef) return "";
|
||||
}, db?: Db) {
|
||||
if (!isRunLogStoreType(run.logStore) || !run.logRef) return "";
|
||||
// A timed-out finalization leaves size unknown even when earlier entries
|
||||
// exist. Read those logs within the same byte budget as a known-size log.
|
||||
if (run.logBytes !== null && (!Number.isFinite(run.logBytes) || run.logBytes <= 0)) return "";
|
||||
|
||||
const logRef = run.logRef;
|
||||
const storeType = run.logStore;
|
||||
const store = getRunLogStore();
|
||||
let offset = 0;
|
||||
let content = "";
|
||||
@@ -6727,6 +6729,8 @@ export async function readIssueCommentRunLogText(run: {
|
||||
let readTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
const readChunks = async () => {
|
||||
if (db && storeType === "s3_segments") await recoverSettledNativeRunLog(db, store, logRef, undefined, controller.signal)
|
||||
.catch(() => { controller.signal.throwIfAborted(); });
|
||||
while (nextOffset !== undefined) {
|
||||
controller.signal.throwIfAborted();
|
||||
const remainingBytes =
|
||||
@@ -6734,7 +6738,7 @@ export async function readIssueCommentRunLogText(run: {
|
||||
Buffer.byteLength(content, "utf8");
|
||||
if (remainingBytes <= 0) break;
|
||||
const chunk = await store.read(
|
||||
{ store: "local_file", logRef },
|
||||
{ store: storeType, logRef },
|
||||
{
|
||||
offset,
|
||||
limitBytes: Math.min(ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES, remainingBytes),
|
||||
@@ -7259,7 +7263,7 @@ export function issueService(db: Db) {
|
||||
);
|
||||
await Promise.all(
|
||||
batch.map(async (run) => {
|
||||
logByRunId.set(run.runId, await readIssueCommentRunLogText(run));
|
||||
logByRunId.set(run.runId, await readIssueCommentRunLogText(run, db));
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -12587,6 +12591,7 @@ export function issueService(db: Db) {
|
||||
order?: "asc" | "desc";
|
||||
limit?: number | null;
|
||||
humanOnly?: boolean;
|
||||
companyId?: string;
|
||||
},
|
||||
) => {
|
||||
const order = opts?.order === "asc" ? "asc" : "desc";
|
||||
@@ -12597,6 +12602,7 @@ export function issueService(db: Db) {
|
||||
: null;
|
||||
|
||||
const conditions = [eq(issueComments.issueId, issueId)];
|
||||
if (opts?.companyId) conditions.push(eq(issueComments.companyId, opts.companyId));
|
||||
if (opts?.humanOnly) conditions.push(or(
|
||||
eq(issueComments.authorType, "user"),
|
||||
and(isNull(issueComments.authorType), isNotNull(issueComments.authorUserId)),
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { resolve } from "node:path";
|
||||
import { createDb } from "@paperclipai/db";
|
||||
import { DurablePrpControlPlane, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint } from "../../../vendor/paperclip-runner/index.js";
|
||||
import { createNativeUsageDelivery } from "../native-usage-delivery.js";
|
||||
|
||||
type Start = {
|
||||
kind: "start"; requestId: number; connectionString: string; directory: string; port?: number;
|
||||
identity: ConstructorParameters<typeof DurablePrpControlPlane>[0]["identity"];
|
||||
input: Parameters<typeof createNativeUsageDelivery>[1];
|
||||
checkpoint?: [string, string][];
|
||||
};
|
||||
let core: DurablePrpControlPlane | undefined;
|
||||
let checkpoint: DurablePrpCheckpoint | undefined;
|
||||
process.on("message", async (message: Start | { kind: "command"; requestId: number; type: string; commandId: string } | { kind: "stop"; requestId: number }) => {
|
||||
try {
|
||||
if (message.kind === "start") {
|
||||
const delivery = await createNativeUsageDelivery(createDb(message.connectionString), message.input);
|
||||
if (message.checkpoint) decodeDurablePrpCheckpoint(new Map(message.checkpoint)).restoreSessionDirectory(message.directory);
|
||||
core = new DurablePrpControlPlane({ stateDirectory: resolve(message.directory, "control-plane"), identity: message.identity,
|
||||
expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"c".repeat(64)}`,
|
||||
onCommittedEvent: async event => {
|
||||
try { await delivery.capture(event); }
|
||||
catch (error) { process.send?.({ kind: "capture-failed", cause: error instanceof Error ? error.message : String(error) }); throw error; }
|
||||
},
|
||||
});
|
||||
core.setCheckpointPersistence(async saved => { checkpoint = saved; });
|
||||
await core.start(message.port);
|
||||
const ticket = message.checkpoint ? undefined : core.issueBootstrapTicket();
|
||||
await core.flushCheckpoint();
|
||||
process.send?.({ requestId: message.requestId, connectUrl: core.connectUrl, ticket });
|
||||
} else if (message.kind === "command") {
|
||||
core!.queueCommand(message.type, {}, message.commandId, true);
|
||||
process.send?.({ requestId: message.requestId });
|
||||
} else {
|
||||
await core?.stop();
|
||||
await core?.drainPendingConnectionProcessing();
|
||||
await core?.flushCheckpoint();
|
||||
process.send?.({ requestId: message.requestId, checkpoint: [...checkpoint!.records] }, () => process.exit(0));
|
||||
}
|
||||
} catch (error) {
|
||||
process.send?.({ requestId: message.requestId, error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
});
|
||||
|
||||
process.send?.({ kind: "ready" });
|
||||
@@ -3,6 +3,7 @@ import { spawn, execFile } from "node:child_process";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises";
|
||||
import { promisify } from "node:util";
|
||||
import type { CommandManagedDuplexChannel, CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability";
|
||||
import { startAdapterExecutionTargetPaperclipBridge } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -39,7 +40,7 @@ function localDuplexChannel(command: readonly string[], cwd: string): CommandMan
|
||||
async function broker(resolveCredentials = vi.fn(async (binding: ReturnType<typeof run>) => ({
|
||||
status: "available", env: { GH_TOKEN: `fixture-${binding.runId}` },
|
||||
})), remote = false, bridgeUnavailable = false, onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>,
|
||||
duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) {
|
||||
duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean; recorder?: DuplexObservabilityRecorder }) {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "native-github-reuse-"));
|
||||
cleanups.push(() => rm(root, { recursive: true, force: true }));
|
||||
const bin = path.join(root, "real-bin");
|
||||
@@ -61,7 +62,7 @@ duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) {
|
||||
});
|
||||
const target = remote ? { kind: "remote" as const, transport: "sandbox" as const, providerKey: "test",
|
||||
remoteCwd: root, runner: { execute, ...(duplex ? { openDuplexChannel } : {}) }, streamRunLogs: false,
|
||||
...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, effectiveCapabilities: {
|
||||
...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, duplexObservabilityRecorder: duplex.recorder, effectiveCapabilities: {
|
||||
reusableLeases: true, nativeSyncIn: false, nativeSyncOut: false, persistentProcessSessions: false,
|
||||
independentControlCommands: true, incrementalSessionOutput: false, concurrentSyncOperations: false,
|
||||
duplexCommandStream: duplex.capability, runnerWebSocketIngress: false,
|
||||
@@ -122,9 +123,13 @@ it.each([false, true])("resolves managed GitHub credentials from OpenCode shell
|
||||
expect(childEnvironment.githubTokenPresent).toBe(false);
|
||||
}, 30000);
|
||||
it("uses the selected duplex transport across credential rotation without file polling", async () => {
|
||||
const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true });
|
||||
const recorder = { recordSpan: vi.fn(), incrementCounter: vi.fn(), emitEvent: vi.fn() };
|
||||
const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true, recorder });
|
||||
expect(b.bridgeModes).toEqual(["http2_v1"]);
|
||||
expect(b.openDuplexChannel).toHaveBeenCalledOnce();
|
||||
expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.channel_open", dimensions: {
|
||||
provider: "other", transport: "http2", outcome: "ok",
|
||||
} }));
|
||||
const commandsAfterStartup = b.execute.mock.calls.length;
|
||||
expect((await request(b)).status).toBe(403);
|
||||
const releaseA = b.activate(run("a"));
|
||||
@@ -135,6 +140,8 @@ it("uses the selected duplex transport across credential rotation without file p
|
||||
releaseA();
|
||||
expect(await (await request(b)).json()).toMatchObject({ env: { GH_TOKEN: "fixture-b" } });
|
||||
expect(b.resolveCredentials.mock.calls.map(([binding]) => binding.runId)).toEqual(["a", "b"]);
|
||||
expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.request" }));
|
||||
expect(recorder.emitEvent).not.toHaveBeenCalledWith(expect.objectContaining({ dimensions: expect.objectContaining({ transport: "file" }) }));
|
||||
expect(b.execute.mock.calls).toHaveLength(commandsAfterStartup);
|
||||
releaseB();
|
||||
await b.stop();
|
||||
|
||||
@@ -3,6 +3,7 @@ import { createServer } from "node:http";
|
||||
import {
|
||||
cleanupGitHubOperationLaunchers,
|
||||
adapterExecutionTargetEnablesSandboxDuplexBridge,
|
||||
adapterExecutionTargetDuplexObservabilityRecorder,
|
||||
githubOperationLauncherDirectory,
|
||||
prepareGitHubOperationLaunchers,
|
||||
startAdapterExecutionTargetPaperclipBridge,
|
||||
@@ -107,6 +108,7 @@ export async function createNativeGitHubAccess(input: {
|
||||
hostApiToken: token,
|
||||
hostApiUrl: url,
|
||||
enableSandboxDuplexBridge: duplexRequested,
|
||||
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(input.target),
|
||||
onLog: input.onLog,
|
||||
});
|
||||
if (input.target?.kind === "remote" && !bridge) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { NativeResponsePresentationScope } from "../native-response-present
|
||||
import { onCommittedChatResponseAuthorityRegistered } from "../durable-chat-wakeup.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
import { recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
|
||||
import { measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js";
|
||||
|
||||
export function registerNativeResponsePresentationWork(db: Db,
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
|
||||
@@ -20,27 +21,35 @@ export function nativeResponsePresentationWork(db: Db) {
|
||||
const query = nativeResponsePresentationCandidates(scope, after, runId).as("pending_native_response");
|
||||
return db.select().from(query).orderBy(asc(query.id));
|
||||
};
|
||||
async function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise<Entry[]> {
|
||||
function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise<Entry[]> {
|
||||
const work = () => loadWithinTrace(scope, signal, canRun);
|
||||
return scope?.runId ? withRunResponsePresentationTiming(scope.runId, work) : work();
|
||||
}
|
||||
async function loadWithinTrace(scope: NativeResponsePresentationScope | undefined, signal: AbortSignal | undefined,
|
||||
canRun: () => boolean): Promise<Entry[]> {
|
||||
const entries: Entry[] = [];
|
||||
let after: string | undefined;
|
||||
while (canRun() && !signal?.aborted) {
|
||||
const rows = await candidates(scope, after);
|
||||
const rows = await measureRunPhase("response.resolve", () => candidates(scope, after));
|
||||
for (const row of rows) entries.push({ id: row.id, companyId: row.companyId, issueId: row.issueId, agentId: row.agentId, dueAt: row.nextAttemptAt?.getTime() ?? 0 });
|
||||
if (rows.length < 100) return entries;
|
||||
after = rows[rows.length - 1].id;
|
||||
}
|
||||
throw new Error("Native response presentation discovery stopped");
|
||||
}
|
||||
async function check(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
|
||||
function check(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
|
||||
return withRunResponsePresentationTiming(entry.id, () => checkWithinTrace(entry, signal));
|
||||
}
|
||||
async function checkWithinTrace(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
|
||||
if (signal?.aborted) throw new Error("Native response presentation recovery stopped");
|
||||
const scope = { companyId: entry.companyId, issueId: entry.issueId };
|
||||
const [current] = await candidates(scope, undefined, entry.id);
|
||||
const [current] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id));
|
||||
if (!current) return null;
|
||||
if (current.nextAttemptAt && current.nextAttemptAt.getTime() > Date.now()) return { ...entry, dueAt: current.nextAttemptAt.getTime() };
|
||||
if (signal?.aborted) throw new Error("Native response presentation recovery stopped");
|
||||
const recoveryStartedAt = Date.now();
|
||||
await recoverCommittedNativeChatResponse(db, { ...scope, runId: entry.id, onlyWhenDue: true });
|
||||
const [remaining] = await candidates(scope, undefined, entry.id);
|
||||
const [remaining] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id));
|
||||
return remaining?.nextAttemptAt && remaining.nextAttemptAt.getTime() > recoveryStartedAt
|
||||
? { ...entry, dueAt: remaining.nextAttemptAt.getTime() } : null;
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { publishChatPublicationCommitSignal } from "../chat-publication-reconciliation.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { beginRunPhase, measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js";
|
||||
import {
|
||||
readNativeBoardResponseWaitOrigin,
|
||||
readNativeBoardResponseWaitSource,
|
||||
@@ -736,11 +737,20 @@ export async function repairCommittedNativeChatResponse(
|
||||
export async function recoverCommittedNativeChatResponse(
|
||||
db: Db,
|
||||
input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean },
|
||||
): Promise<boolean> {
|
||||
return withRunResponsePresentationTiming(input.runId, () => recoverCommittedNativeChatResponseInScope(db, input));
|
||||
}
|
||||
|
||||
async function recoverCommittedNativeChatResponseInScope(
|
||||
db: Db,
|
||||
input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean },
|
||||
): Promise<boolean> {
|
||||
let agentId: string | null = null;
|
||||
let materialized = false;
|
||||
const entered = beginRunPhase("response.transaction.entry");
|
||||
try {
|
||||
materialized = await db.transaction(async (transaction) => {
|
||||
materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
|
||||
entered();
|
||||
const tx = transaction as unknown as Db;
|
||||
|
||||
|
||||
@@ -1022,10 +1032,12 @@ export async function recoverCommittedNativeChatResponse(
|
||||
await projectCommittedRun({ db: tx, run, coordinator });
|
||||
agentId = run.agentId;
|
||||
return true;
|
||||
});
|
||||
}));
|
||||
} catch (error) {
|
||||
if (error instanceof CommittedChatResponseAuthorizationError) return false;
|
||||
throw error;
|
||||
} finally {
|
||||
entered();
|
||||
}
|
||||
if (materialized && agentId)
|
||||
publishChatPublicationCommitSignal({
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
import { randomUUID, createHash, generateKeyPairSync } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
createDb,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
agentIdentityKeys,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
S3Client,
|
||||
CreateBucketCommand,
|
||||
DeleteBucketCommand,
|
||||
DeleteObjectsCommand,
|
||||
ListObjectsV2Command,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { createS3StorageProvider } from "../../storage/s3-provider.js";
|
||||
import { createObjectRunLogStore } from "../object-run-log-store.js";
|
||||
import { createNativeRunLogExporter, formatNativeRunLogEvent } from "./native-run-log-export.js";
|
||||
import { recoverSettledNativeRunLog, readRecoverableRunLog } from "./native-run-log-recovery.js";
|
||||
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { createNativeRunLogTestTables } from "../../__tests__/helpers/native-run-log-database.js";
|
||||
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import type { StorageProvider } from "../../storage/types.js";
|
||||
import { appendNativeCredentialUpdate } from "./native-credential-updates.js";
|
||||
import { appendRetainedNativeCleanupEvent } from "./native-session-executor.js";
|
||||
import { idleWorkSnapshot } from "../task-admission.js";
|
||||
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { localEncryptedProvider } from "../../secrets/local-encrypted-provider.js";
|
||||
import { createHeartbeatRunOutput } from "../heartbeat/run-output.js";
|
||||
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
import { ObjectJournalConflictError } from "../object-byte-journal.js";
|
||||
|
||||
const url = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL");
|
||||
const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
|
||||
describe.skipIf(!url || !endpoint)("native run log source recovery", () => {
|
||||
const schema = `native_log_${randomUUID().replaceAll("-", "")}`,
|
||||
bucket = `pc-native-log-${randomUUID()}`;
|
||||
let db: Db,
|
||||
client: S3Client,
|
||||
bucketCreated = false;
|
||||
beforeAll(async () => {
|
||||
db = createDb(url!, { maxConnections: 1, idleTimeoutSeconds: 0 });
|
||||
await db.execute(sql.raw(`create schema ${schema}`));
|
||||
await db.execute(sql.raw(`set search_path=${schema}`));
|
||||
await createNativeRunLogTestTables(db);
|
||||
client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
|
||||
await client.send(new CreateBucketCommand({ Bucket: bucket }));
|
||||
bucketCreated = true;
|
||||
});
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
afterAll(async () => {
|
||||
if (client && bucketCreated) {
|
||||
for (;;) {
|
||||
const page = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
|
||||
if (!page.Contents?.length) break;
|
||||
await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: { Objects: page.Contents.map((item) => ({ Key: item.Key! })) },
|
||||
}),
|
||||
);
|
||||
}
|
||||
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
|
||||
client.destroy();
|
||||
}
|
||||
if (db) {
|
||||
await db.execute(sql.raw(`drop schema ${schema} cascade`));
|
||||
await db.$client.end();
|
||||
}
|
||||
});
|
||||
async function fixture() {
|
||||
const binding = {
|
||||
companyId: randomUUID(),
|
||||
runId: randomUUID(),
|
||||
agentId: randomUUID(),
|
||||
sessionId: randomUUID(),
|
||||
issueId: randomUUID(),
|
||||
sourceInstanceId: randomUUID(),
|
||||
controlPlaneSourceInstanceId: randomUUID(),
|
||||
completionContractId: randomUUID(),
|
||||
completionContractSha256: "test",
|
||||
};
|
||||
await db.execute(
|
||||
sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${binding.runId},${binding.companyId},${binding.agentId},'running')`,
|
||||
);
|
||||
const provider = createS3StorageProvider({
|
||||
endpoint,
|
||||
bucket,
|
||||
region: "us-east-1",
|
||||
forcePathStyle: true,
|
||||
});
|
||||
const store = (storage: StorageProvider = provider) =>
|
||||
createObjectRunLogStore({
|
||||
provider: storage,
|
||||
keyPrefix: "logs",
|
||||
flushMs: 30_000,
|
||||
assertWriterAuthority: async (input) => {
|
||||
await input.assertWriterAuthority?.();
|
||||
},
|
||||
});
|
||||
const begin = { companyId: binding.companyId, agentId: binding.agentId, runId: binding.runId };
|
||||
const port = new PaperclipControlPlanePort(db, binding);
|
||||
const event = (seq: number): PrpEvent => ({
|
||||
schema: "paperclip.prp.event.v1",
|
||||
schemaVersion: 1,
|
||||
sourceInstanceId: binding.sourceInstanceId,
|
||||
sourceEventId: `${binding.sourceInstanceId}:${seq}`,
|
||||
sourceSeq: seq,
|
||||
sourceKind: "runner",
|
||||
runId: binding.runId,
|
||||
normalizedSessionId: binding.sessionId,
|
||||
eventType: "item.delta",
|
||||
priority: 2,
|
||||
emittedAt: new Date().toISOString(),
|
||||
payload: { text: `step ${seq} 再開` },
|
||||
});
|
||||
return { binding, store, begin, port, event, provider };
|
||||
}
|
||||
it("replays acknowledged native records into a new writer, repairs gaps, and ignores late duplicates", async () => {
|
||||
const f = await fixture(),
|
||||
a = f.store(),
|
||||
h = await a.begin(f.begin);
|
||||
await db.execute(
|
||||
sql`update heartbeat_runs set log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
|
||||
);
|
||||
const events = [f.event(1), f.event(2), f.event(3)];
|
||||
const cursors = await Promise.all(events.slice(0, 2).map((event) => f.port.appendEvent(event)));
|
||||
const receipt = await a.enqueue!(h, {
|
||||
stream: "stdout",
|
||||
chunk: formatNativeRunLogEvent(events[0]!),
|
||||
ts: "then",
|
||||
seq: 1,
|
||||
sourceCursor: cursors[0]!.cursor,
|
||||
});
|
||||
await a.flush!(h);
|
||||
await receipt.committed;
|
||||
const b = f.store(),
|
||||
successor = await b.begin(f.begin);
|
||||
let seq = successor.initialSequence!;
|
||||
const exporter = createNativeRunLogExporter({
|
||||
db,
|
||||
companyId: f.binding.companyId,
|
||||
runId: f.binding.runId,
|
||||
afterCursor: successor.initialSourceCursor!,
|
||||
enqueue: async (sourceCursor, chunk) => {
|
||||
const queued = await b.enqueue!(successor, {
|
||||
stream: "stdout",
|
||||
chunk,
|
||||
ts: "now",
|
||||
seq: ++seq,
|
||||
sourceCursor,
|
||||
});
|
||||
void queued.committed.catch(() => undefined);
|
||||
},
|
||||
flush: () => b.flush!(successor),
|
||||
});
|
||||
const third = await f.port.appendEvent(events[2]!);
|
||||
await exporter.record(third.cursor, formatNativeRunLogEvent(events[2]!));
|
||||
await exporter.record(cursors[0]!.cursor, formatNativeRunLogEvent(events[0]!));
|
||||
await b.finalize(successor);
|
||||
const entries = (await b.read(successor)).content
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(JSON.parse(line).chunk).event);
|
||||
expect(entries.map((event) => event.sourceSeq)).toEqual([1, 2, 3]);
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(heartbeatRunEvents)
|
||||
.where(eq(heartbeatRunEvents.runId, f.binding.runId)),
|
||||
).toHaveLength(3);
|
||||
});
|
||||
it("repairs a settled run on demand after its controller lease expires, without extending source retention", async () => {
|
||||
const f = await fixture(),
|
||||
a = f.store(),
|
||||
h = await a.begin(f.begin);
|
||||
await f.port.appendEvent(f.event(1));
|
||||
await db.execute(
|
||||
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
|
||||
);
|
||||
await db.execute(
|
||||
sql`insert into native_run_finalizations values(${f.binding.runId},${f.binding.companyId},now()+interval '1 minute')`,
|
||||
);
|
||||
const b = f.store();
|
||||
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
|
||||
expect((await b.read(h)).content).toBe("");
|
||||
await db.execute(
|
||||
sql`update native_run_finalizations set lease_expires_at=now()-interval '1 second' where run_id=${f.binding.runId}`,
|
||||
);
|
||||
await Promise.all(
|
||||
[1, 2].map(() =>
|
||||
recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })),
|
||||
),
|
||||
);
|
||||
expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1);
|
||||
const [run] = await db
|
||||
.select({ bytes: heartbeatRuns.logBytes })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
expect(run!.bytes).toBeGreaterThan(0);
|
||||
});
|
||||
it.each(["hash", "payload", "identity"])(
|
||||
"rejects a corrupt canonical source %s without marking the partial archive complete",
|
||||
async (kind) => {
|
||||
const f = await fixture(),
|
||||
a = f.store(),
|
||||
h = await a.begin(f.begin);
|
||||
await f.port.appendEvent(f.event(1));
|
||||
if (kind === "hash")
|
||||
await db.execute(
|
||||
sql`update heartbeat_run_events set source_payload_sha256='wrong' where run_id=${f.binding.runId}`,
|
||||
);
|
||||
else if (kind === "payload")
|
||||
await db.execute(
|
||||
sql`update heartbeat_run_events set payload='{}'::jsonb where run_id=${f.binding.runId}`,
|
||||
);
|
||||
else
|
||||
await db.execute(
|
||||
sql`update heartbeat_run_events set source_instance_id='wrong' where run_id=${f.binding.runId}`,
|
||||
);
|
||||
await db.execute(
|
||||
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
|
||||
);
|
||||
await expect(
|
||||
recoverSettledNativeRunLog(db, f.store(), h.logRef, async () => ({ enabled: false })),
|
||||
).rejects.toThrow("integrity");
|
||||
const [run] = await db
|
||||
.select({ bytes: heartbeatRuns.logBytes })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
expect(run!.bytes).toBeNull();
|
||||
},
|
||||
);
|
||||
it("ignores credential and retained cleanup projections when replaying canonical log sources", async () => {
|
||||
const f = await fixture(),
|
||||
a = f.store(),
|
||||
h = await a.begin(f.begin);
|
||||
await f.port.appendEvent(f.event(1));
|
||||
const credential = (sourceSeq: number): PrpEvent => ({
|
||||
...f.event(sourceSeq),
|
||||
schema: "paperclip.prp.event.v3",
|
||||
schemaVersion: 3,
|
||||
priority: 0,
|
||||
eventType: "auth.credentials.updated",
|
||||
payload: {
|
||||
schema: "paperclip.auth.credentials.updated.v1",
|
||||
provider: "codex",
|
||||
bindingId: "test-auth",
|
||||
revision: 1,
|
||||
detection: "credential_store",
|
||||
},
|
||||
});
|
||||
await f.port.appendEvent(credential(2));
|
||||
await db.execute(sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef},
|
||||
native_session_id=${f.binding.sessionId}, runner_instance_id=${f.binding.sourceInstanceId} where id=${f.binding.runId}`);
|
||||
await appendNativeCredentialUpdate(db, {
|
||||
companyId: f.binding.companyId,
|
||||
agentId: f.binding.agentId,
|
||||
normalizedSessionId: f.binding.sessionId,
|
||||
runnerInstanceId: f.binding.sourceInstanceId,
|
||||
event: credential(20),
|
||||
});
|
||||
await appendRetainedNativeCleanupEvent(db, {
|
||||
companyId: f.binding.companyId,
|
||||
agentId: f.binding.agentId,
|
||||
runId: f.binding.runId,
|
||||
nativeSessionId: f.binding.sessionId,
|
||||
runnerInstanceId: f.binding.sourceInstanceId,
|
||||
requestId: "test-cleanup",
|
||||
event: f.event(21),
|
||||
});
|
||||
await appendHeartbeatRunEvent(db, {
|
||||
companyId: f.binding.companyId,
|
||||
agentId: f.binding.agentId,
|
||||
runId: f.binding.runId,
|
||||
eventType: "workspace.diff.recorded",
|
||||
payload: { complete: true },
|
||||
nativeSource: {
|
||||
sourceInstanceId: "provider-trace-reproject:test",
|
||||
sourceEventId: "provider-trace-reproject:test:change",
|
||||
sourceSeq: 1,
|
||||
protocolSchemaVersion: 1,
|
||||
canonicalPayload: { complete: true },
|
||||
},
|
||||
});
|
||||
const b = f.store();
|
||||
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
|
||||
const entries = (await b.read(h)).content.trim().split("\n");
|
||||
expect(entries).toHaveLength(2);
|
||||
expect(JSON.parse(JSON.parse(entries[0]!).chunk).event.sourceSeq).toBe(1);
|
||||
expect(JSON.parse(JSON.parse(entries[1]!).chunk).event.eventType).toBe(
|
||||
"auth.credentials.updated",
|
||||
);
|
||||
const [run] = await db
|
||||
.select({ bytes: heartbeatRuns.logBytes })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
expect(run!.bytes).toBeGreaterThan(0);
|
||||
});
|
||||
it("aborts in-flight archive writes when its last reader cancels, then resumes without loss", async () => {
|
||||
const f = await fixture(),
|
||||
a = f.store(),
|
||||
h = await a.begin(f.begin),
|
||||
baseline = idleWorkSnapshot().active;
|
||||
await f.port.appendEvent(f.event(1));
|
||||
await db.execute(
|
||||
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
|
||||
);
|
||||
let entered = false,
|
||||
aborted = false;
|
||||
const blocked: StorageProvider = {
|
||||
...f.provider,
|
||||
async putObjectConditional(input) {
|
||||
if (input.objectKey.includes("/objects/")) {
|
||||
entered = true;
|
||||
await new Promise<void>((_resolve, reject) =>
|
||||
input.signal!.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
aborted = true;
|
||||
reject(input.signal!.reason);
|
||||
},
|
||||
{ once: true },
|
||||
),
|
||||
);
|
||||
}
|
||||
return f.provider.putObjectConditional!(input);
|
||||
},
|
||||
};
|
||||
const controller = new AbortController();
|
||||
const repairing = recoverSettledNativeRunLog(
|
||||
db,
|
||||
f.store(blocked),
|
||||
h.logRef,
|
||||
async () => ({ enabled: false }),
|
||||
controller.signal,
|
||||
);
|
||||
const rejected = expect(repairing).rejects.toThrow("reader cancelled");
|
||||
const lastReader = new AbortController();
|
||||
const joined = recoverSettledNativeRunLog(
|
||||
db,
|
||||
f.store(blocked),
|
||||
h.logRef,
|
||||
async () => ({ enabled: false }),
|
||||
lastReader.signal,
|
||||
);
|
||||
const joinedRejected = expect(joined).rejects.toThrow("last reader cancelled");
|
||||
await vi.waitFor(() => expect(entered).toBe(true));
|
||||
controller.abort(new Error("reader cancelled"));
|
||||
await rejected;
|
||||
expect(aborted).toBe(false);
|
||||
lastReader.abort(new Error("last reader cancelled"));
|
||||
await joinedRejected;
|
||||
await vi.waitFor(() => {
|
||||
expect(aborted).toBe(true);
|
||||
expect(idleWorkSnapshot().active).toBe(baseline);
|
||||
});
|
||||
const b = f.store();
|
||||
await vi.waitFor(async () => {
|
||||
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
|
||||
});
|
||||
expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1);
|
||||
});
|
||||
it("redacts the existing agent identity and registered secrets before publishing recovered native output", async () => {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "e".repeat(64));
|
||||
const f = await fixture();
|
||||
const keys = generateKeyPairSync("ed25519"),
|
||||
privateKeyPem = keys.privateKey.export({ type: "pkcs8", format: "pem" }).toString();
|
||||
const publicKeyPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString();
|
||||
const keyId = `sha256:${createHash("sha256")
|
||||
.update(keys.publicKey.export({ type: "spki", format: "der" }))
|
||||
.digest("base64url")}`;
|
||||
const encrypted = await localEncryptedProvider.createSecret({ value: privateKeyPem });
|
||||
await db.insert(agentIdentityKeys).values({
|
||||
agentId: f.binding.agentId,
|
||||
companyId: f.binding.companyId,
|
||||
algorithm: "Ed25519",
|
||||
keyId,
|
||||
publicKeyPem,
|
||||
privateKeyMaterial: encrypted.material,
|
||||
});
|
||||
const secret = "registered-native-fixture-secret";
|
||||
const redactor = createAgentIdentityRedactor(privateKeyPem);
|
||||
redactor.values.push(secret);
|
||||
const run = {
|
||||
id: f.binding.runId,
|
||||
companyId: f.binding.companyId,
|
||||
agentId: f.binding.agentId,
|
||||
status: "running",
|
||||
runtimeMode: "native",
|
||||
logStore: null,
|
||||
logRef: null,
|
||||
logBytes: null,
|
||||
lastOutputSeq: 0,
|
||||
lastOutputAt: null,
|
||||
} as typeof heartbeatRuns.$inferSelect;
|
||||
const output = createHeartbeatRunOutput(db, {
|
||||
run,
|
||||
runId: run.id,
|
||||
issueId: null,
|
||||
runLogStore: f.store(),
|
||||
identityRedactor: redactor,
|
||||
secretKeys: new Set(),
|
||||
runtimeLogSecretValues: [secret],
|
||||
appendRunEvent: async () => undefined,
|
||||
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
|
||||
});
|
||||
await output.begin(run);
|
||||
const h = output.handle!;
|
||||
await f.port.appendEvent({ ...f.event(1), payload: { text: `${privateKeyPem}\n${secret}` } });
|
||||
await db.execute(
|
||||
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`,
|
||||
);
|
||||
const b = f.store();
|
||||
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
|
||||
const content = (await b.read(h)).content;
|
||||
expect(content).not.toContain(privateKeyPem.split("\n")[1]!);
|
||||
expect(content).not.toContain(secret);
|
||||
expect(content).toContain("REDACTED");
|
||||
const [snapshot] = await db
|
||||
.select({ context: heartbeatRuns.contextSnapshot })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
expect(JSON.stringify(snapshot!.context)).not.toContain(privateKeyPem.split("\n")[1]!);
|
||||
});
|
||||
it("serves the published snapshot when another writer wins a repair claim", async () => {
|
||||
const f = await fixture(),
|
||||
a = f.store(),
|
||||
h = await a.begin(f.begin);
|
||||
const event = f.event(1),
|
||||
receipt = await f.port.appendEvent(event);
|
||||
const queued = await a.enqueue!(h, {
|
||||
stream: "stdout",
|
||||
chunk: formatNativeRunLogEvent(event),
|
||||
ts: "then",
|
||||
seq: 1,
|
||||
sourceCursor: receipt.cursor,
|
||||
});
|
||||
await a.flush!(h);
|
||||
await queued.committed;
|
||||
await f.port.appendEvent(f.event(2));
|
||||
await db.execute(
|
||||
sql`update heartbeat_runs set status='succeeded',log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`,
|
||||
);
|
||||
let enter!: () => void, release!: () => void;
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
enter = resolve;
|
||||
}),
|
||||
blocked = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const store = {
|
||||
...f.store(),
|
||||
async begin() {
|
||||
enter();
|
||||
await blocked;
|
||||
throw new ObjectJournalConflictError();
|
||||
},
|
||||
};
|
||||
const repairing = recoverSettledNativeRunLog(db, store, h.logRef, async () => ({
|
||||
enabled: false,
|
||||
}));
|
||||
const rejected = expect(repairing).rejects.toBeInstanceOf(ObjectJournalConflictError);
|
||||
await entered;
|
||||
const read = readRecoverableRunLog(db, store, h);
|
||||
release();
|
||||
await rejected;
|
||||
expect((await read).content.trim().split("\n")).toHaveLength(1);
|
||||
await a.finalize(h);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,210 @@
|
||||
import { and, asc, desc, eq, gt, isNotNull, lte, not, or, sql } from "drizzle-orm";
|
||||
import { StringDecoder } from "node:string_decoder";
|
||||
import { heartbeatRunEvents, type Db } from "@paperclipai/db";
|
||||
import { validatePrpEvent, type PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
|
||||
export function formatNativeRunLogEvent(event: PrpEvent) {
|
||||
return `${JSON.stringify({ type: "paperclip.prp.event", event })}\n`;
|
||||
}
|
||||
export function nativeRunLogSourceWhere(companyId: string, runId: string) {
|
||||
return and(
|
||||
eq(heartbeatRunEvents.companyId, companyId),
|
||||
eq(heartbeatRunEvents.runId, runId),
|
||||
isNotNull(heartbeatRunEvents.sourceEventId),
|
||||
not(
|
||||
sql`coalesce((${or(
|
||||
and(
|
||||
eq(heartbeatRunEvents.eventType, "auth.credentials.updated"),
|
||||
sql`${heartbeatRunEvents.sourceEventId} like 'credentials:%'`,
|
||||
sql`${heartbeatRunEvents.sourceInstanceId} like '%:credentials'`,
|
||||
),
|
||||
and(
|
||||
eq(heartbeatRunEvents.eventType, "native.cleanup.event"),
|
||||
sql`${heartbeatRunEvents.sourceEventId} like 'cleanup:%'`,
|
||||
sql`${heartbeatRunEvents.sourceInstanceId} like '%:cleanup:%'`,
|
||||
),
|
||||
and(
|
||||
eq(heartbeatRunEvents.eventType, "workspace.diff.recorded"),
|
||||
sql`${heartbeatRunEvents.sourceEventId} like 'provider-trace-reproject:%'`,
|
||||
sql`${heartbeatRunEvents.sourceInstanceId} like 'provider-trace-reproject:%'`,
|
||||
),
|
||||
)}), false)`,
|
||||
),
|
||||
);
|
||||
}
|
||||
async function sourcePage(
|
||||
db: Db,
|
||||
companyId: string,
|
||||
runId: string,
|
||||
after: number,
|
||||
through?: number,
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
signal?.throwIfAborted();
|
||||
const rows = await db
|
||||
.select({
|
||||
seq: heartbeatRunEvents.seq,
|
||||
payload: heartbeatRunEvents.payload,
|
||||
sourceInstanceId: heartbeatRunEvents.sourceInstanceId,
|
||||
sourceEventId: heartbeatRunEvents.sourceEventId,
|
||||
sourceSeq: heartbeatRunEvents.sourceSeq,
|
||||
sha256: heartbeatRunEvents.sourcePayloadSha256,
|
||||
})
|
||||
.from(heartbeatRunEvents)
|
||||
.where(
|
||||
and(
|
||||
nativeRunLogSourceWhere(companyId, runId),
|
||||
gt(heartbeatRunEvents.seq, after),
|
||||
through === undefined ? undefined : lte(heartbeatRunEvents.seq, through),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(heartbeatRunEvents.seq))
|
||||
.limit(16);
|
||||
signal?.throwIfAborted();
|
||||
return rows.map((row) => {
|
||||
const validated = validatePrpEvent(row.payload?.prpEvent);
|
||||
if (
|
||||
!validated.ok ||
|
||||
validated.event.runId !== runId ||
|
||||
validated.event.sourceInstanceId !== row.sourceInstanceId ||
|
||||
validated.event.sourceEventId !== row.sourceEventId ||
|
||||
validated.event.sourceSeq !== row.sourceSeq ||
|
||||
nativeSha256(validated.event) !== row.sha256
|
||||
)
|
||||
throw new Error("native_run_log_source_integrity_error");
|
||||
return { cursor: row.seq, event: validated.event };
|
||||
});
|
||||
}
|
||||
|
||||
export function createNativeRunLogCoverageVerifier(db: Db, companyId: string, runId: string) {
|
||||
const decoder = new StringDecoder("utf8");
|
||||
let text = "",
|
||||
cursor = 0;
|
||||
let page: Awaited<ReturnType<typeof sourcePage>> = [];
|
||||
const unproven = () => new Error("native_log_source_coverage_unproven");
|
||||
async function next() {
|
||||
if (!page.length) page = await sourcePage(db, companyId, runId, cursor);
|
||||
return page[0];
|
||||
}
|
||||
async function line(value: string) {
|
||||
if (!value.trim()) return;
|
||||
if (Buffer.byteLength(value) > 1024 * 1024) throw unproven();
|
||||
let chunk: unknown;
|
||||
try {
|
||||
chunk = JSON.parse(value).chunk;
|
||||
} catch {
|
||||
throw unproven();
|
||||
}
|
||||
if (typeof chunk !== "string" || !chunk.startsWith('{"type":"paperclip.prp.event"')) return;
|
||||
let event: PrpEvent;
|
||||
try {
|
||||
const parsed = validatePrpEvent(JSON.parse(chunk).event);
|
||||
if (!parsed.ok) throw unproven();
|
||||
event = parsed.event;
|
||||
} catch {
|
||||
throw unproven();
|
||||
}
|
||||
const expected = await next();
|
||||
if (
|
||||
!expected ||
|
||||
event.runId !== runId ||
|
||||
event.sourceInstanceId !== expected.event.sourceInstanceId ||
|
||||
event.sourceEventId !== expected.event.sourceEventId ||
|
||||
event.sourceSeq !== expected.event.sourceSeq ||
|
||||
event.normalizedSessionId !== expected.event.normalizedSessionId ||
|
||||
event.schemaVersion !== expected.event.schemaVersion ||
|
||||
event.eventType !== expected.event.eventType
|
||||
)
|
||||
throw unproven();
|
||||
cursor = expected.cursor;
|
||||
page.shift();
|
||||
}
|
||||
return {
|
||||
async write(bytes: Buffer) {
|
||||
text += decoder.write(bytes);
|
||||
let boundary: number;
|
||||
while ((boundary = text.indexOf("\n")) !== -1) {
|
||||
await line(text.slice(0, boundary));
|
||||
text = text.slice(boundary + 1);
|
||||
}
|
||||
if (Buffer.byteLength(text) > 1024 * 1024) throw unproven();
|
||||
},
|
||||
async finish() {
|
||||
text += decoder.end();
|
||||
if (text) await line(text);
|
||||
if (await next()) throw unproven();
|
||||
return cursor;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createNativeRunLogExporter(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
runId: string;
|
||||
afterCursor: number;
|
||||
enqueue: (cursor: number, chunk: string) => Promise<void>;
|
||||
flush: () => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
let cursor = input.afterCursor;
|
||||
let queue = Promise.resolve();
|
||||
let queuedBytes = 0,
|
||||
queuedRecords = 0;
|
||||
const sourceWhere = nativeRunLogSourceWhere(input.companyId, input.runId);
|
||||
async function replay(through?: number) {
|
||||
input.signal?.throwIfAborted();
|
||||
if (through === undefined) {
|
||||
const [last] = await input.db
|
||||
.select({ seq: heartbeatRunEvents.seq })
|
||||
.from(heartbeatRunEvents)
|
||||
.where(sourceWhere)
|
||||
.orderBy(desc(heartbeatRunEvents.seq))
|
||||
.limit(1);
|
||||
through = last?.seq ?? cursor;
|
||||
}
|
||||
while (cursor < through) {
|
||||
const rows = await sourcePage(
|
||||
input.db,
|
||||
input.companyId,
|
||||
input.runId,
|
||||
cursor,
|
||||
through,
|
||||
input.signal,
|
||||
);
|
||||
if (!rows.length) break;
|
||||
for (const row of rows) {
|
||||
input.signal?.throwIfAborted();
|
||||
await input.enqueue(row.cursor, formatNativeRunLogEvent(row.event));
|
||||
cursor = row.cursor;
|
||||
}
|
||||
await input.flush();
|
||||
}
|
||||
}
|
||||
return {
|
||||
recover: () => replay(),
|
||||
async record(sourceCursor: number, chunk: string) {
|
||||
if (!Number.isSafeInteger(sourceCursor) || sourceCursor < 1)
|
||||
throw new Error("invalid_native_run_log_cursor");
|
||||
const bytes = Buffer.byteLength(chunk);
|
||||
if (queuedRecords >= 32 || queuedBytes + bytes > 8 * 1024 * 1024)
|
||||
throw new Error("native_run_log_admission_capacity_exceeded");
|
||||
queuedRecords++;
|
||||
queuedBytes += bytes;
|
||||
const work = queue
|
||||
.then(async () => {
|
||||
if (sourceCursor <= cursor) return;
|
||||
if (sourceCursor > cursor + 1) await replay(sourceCursor - 1);
|
||||
await input.enqueue(sourceCursor, chunk);
|
||||
cursor = sourceCursor;
|
||||
})
|
||||
.finally(() => {
|
||||
queuedRecords--;
|
||||
queuedBytes -= bytes;
|
||||
});
|
||||
queue = work.catch(() => undefined);
|
||||
await work;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import { and, eq, gt, inArray, isNull, sql } from "drizzle-orm";
|
||||
import {
|
||||
heartbeatRuns,
|
||||
heartbeatRunEvents,
|
||||
nativeRunFinalizations,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../log-redaction.js";
|
||||
import type { RunLogHandle, RunLogReadOptions, RunLogStore } from "../run-log-store.js";
|
||||
import { instanceSettingsService } from "../instance-settings.js";
|
||||
import { compactRunLogChunk } from "../heartbeat/run-log.js";
|
||||
import { beginIdleTrackedWork } from "../task-admission.js";
|
||||
import { agentIdentityService } from "../agent-identity.js";
|
||||
import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js";
|
||||
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
import { logger } from "../../middleware/logger.js";
|
||||
import { createNativeRunLogExporter, nativeRunLogSourceWhere } from "./native-run-log-export.js";
|
||||
|
||||
interface Recovery {
|
||||
work: Promise<void>;
|
||||
controller: AbortController;
|
||||
readers: number;
|
||||
}
|
||||
const recoveries = new WeakMap<Db, Map<string, Recovery>>();
|
||||
function awaitRecovery(recovery: Recovery, signal?: AbortSignal) {
|
||||
signal?.throwIfAborted();
|
||||
recovery.readers++;
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let finished = false;
|
||||
const finish = () => {
|
||||
if (finished) return false;
|
||||
finished = true;
|
||||
signal?.removeEventListener("abort", abort);
|
||||
recovery.readers--;
|
||||
return true;
|
||||
};
|
||||
const abort = () => {
|
||||
if (!finish()) return;
|
||||
if (!recovery.readers) recovery.controller.abort(signal!.reason);
|
||||
reject(signal!.reason);
|
||||
};
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
recovery.work.then(
|
||||
() => {
|
||||
if (finish()) resolve();
|
||||
},
|
||||
(error) => {
|
||||
if (finish()) reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export async function readRecoverableRunLog(
|
||||
db: Db,
|
||||
store: RunLogStore,
|
||||
handle: RunLogHandle,
|
||||
options?: RunLogReadOptions,
|
||||
known?: { runtimeMode?: string | null; status?: string; logBytes?: number | null },
|
||||
) {
|
||||
if (
|
||||
handle.store === "s3_segments" &&
|
||||
known?.runtimeMode !== "legacy" &&
|
||||
known?.status !== "running" &&
|
||||
(known?.logBytes === undefined || known.logBytes === null)
|
||||
)
|
||||
await recoverSettledNativeRunLog(db, store, handle.logRef, undefined, options?.signal).catch(
|
||||
() => {
|
||||
options?.signal?.throwIfAborted();
|
||||
logger.warn(
|
||||
{ runId: handle.logRef.split("/").at(-1) },
|
||||
"Native run log repair is incomplete; serving the published prefix",
|
||||
);
|
||||
},
|
||||
);
|
||||
return store.read(handle, options);
|
||||
}
|
||||
|
||||
export async function recoverSettledNativeRunLog(
|
||||
db: Db,
|
||||
store: RunLogStore,
|
||||
logRef: string,
|
||||
getRedactionOptions: () => Promise<CurrentUserRedactionOptions> = async () => ({
|
||||
enabled: (await instanceSettingsService(db).getGeneral()).censorUsernameInLogs,
|
||||
}),
|
||||
requestSignal?: AbortSignal,
|
||||
) {
|
||||
requestSignal?.throwIfAborted();
|
||||
const binding = /^([a-f0-9-]+)\/([a-f0-9-]+)\/([a-f0-9-]+)$/.exec(logRef);
|
||||
if (!binding) throw new Error("invalid_native_run_log_reference");
|
||||
let pending = recoveries.get(db);
|
||||
if (!pending) {
|
||||
pending = new Map();
|
||||
recoveries.set(db, pending);
|
||||
}
|
||||
const existing = pending.get(logRef);
|
||||
if (existing) return awaitRecovery(existing, requestSignal);
|
||||
const controller = new AbortController();
|
||||
const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(30_000)]);
|
||||
const finishWork = beginIdleTrackedWork();
|
||||
const runId = binding[3]!,
|
||||
companyId = binding[1]!;
|
||||
const eligible = () =>
|
||||
and(
|
||||
eq(heartbeatRuns.id, runId),
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
eq(heartbeatRuns.agentId, binding[2]!),
|
||||
eq(heartbeatRuns.logStore, "s3_segments"),
|
||||
eq(heartbeatRuns.logRef, logRef),
|
||||
eq(heartbeatRuns.runtimeMode, "native"),
|
||||
inArray(heartbeatRuns.status, ["succeeded", "failed", "cancelled", "timed_out"]),
|
||||
isNull(heartbeatRuns.logBytes),
|
||||
sql`not exists (select 1 from ${nativeRunFinalizations} where ${and(
|
||||
eq(nativeRunFinalizations.runId, runId),
|
||||
eq(nativeRunFinalizations.companyId, companyId),
|
||||
gt(nativeRunFinalizations.leaseExpiresAt, new Date()),
|
||||
)})`,
|
||||
);
|
||||
const recover = (async () => {
|
||||
signal.throwIfAborted();
|
||||
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eligible());
|
||||
signal.throwIfAborted();
|
||||
if (!run) return;
|
||||
const [redaction, identityRedactor, secrets] = await Promise.all([
|
||||
getRedactionOptions(),
|
||||
agentIdentityService(db).createExistingLogRedactor(companyId, binding[2]!),
|
||||
createRunSecretRedactionRegistry(db).redactionValuesForRun(companyId, runId),
|
||||
]);
|
||||
for (const value of secrets)
|
||||
identityRedactor.values.push(...createAgentIdentityRedactor(value).values);
|
||||
identityRedactor.values.sort((left, right) => right.length - left.length);
|
||||
const handle = await store.begin({
|
||||
companyId,
|
||||
agentId: binding[2]!,
|
||||
runId,
|
||||
previousStore: "s3_segments",
|
||||
signal,
|
||||
assertWriterAuthority: async () => {
|
||||
signal.throwIfAborted();
|
||||
const [current] = await db
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(eligible());
|
||||
if (!current) throw new Error("native_run_log_recovery_authority_lost");
|
||||
},
|
||||
});
|
||||
let sequence = handle.initialSequence ?? 0;
|
||||
let lastSourceCursor = handle.initialSourceCursor ?? 0;
|
||||
const receipts: Promise<number>[] = [];
|
||||
let failure: unknown;
|
||||
const exporter = createNativeRunLogExporter({
|
||||
db,
|
||||
companyId,
|
||||
runId,
|
||||
afterCursor: handle.initialSourceCursor ?? 0,
|
||||
signal,
|
||||
enqueue: async (sourceCursor, chunk) => {
|
||||
const receipt = await store.enqueue!(handle, {
|
||||
stream: "stdout",
|
||||
ts: new Date().toISOString(),
|
||||
seq: ++sequence,
|
||||
sourceCursor,
|
||||
chunk: compactRunLogChunk(
|
||||
redactCurrentUserText(identityRedactor.chunk("stdout", chunk), redaction),
|
||||
),
|
||||
});
|
||||
receipts.push(
|
||||
receipt.committed.catch((error) => {
|
||||
failure ??= error;
|
||||
return 0;
|
||||
}),
|
||||
);
|
||||
lastSourceCursor = sourceCursor;
|
||||
},
|
||||
flush: async () => {
|
||||
await store.flush!(handle);
|
||||
await Promise.all(receipts.splice(0));
|
||||
if (failure) throw failure;
|
||||
},
|
||||
});
|
||||
try {
|
||||
await exporter.recover();
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
} finally {
|
||||
const summary = await store.finalize(handle, { computeDigest: false });
|
||||
if (summary.bytes === null) failure ??= new Error("native_run_log_recovery_incomplete");
|
||||
if (!failure && summary.bytes !== null) {
|
||||
signal.throwIfAborted();
|
||||
const updated = await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
logBytes: summary.bytes,
|
||||
logSha256: summary.sha256,
|
||||
lastOutputSeq: sequence,
|
||||
lastOutputBytes: summary.bytes,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eligible(),
|
||||
sql`not exists (select 1 from ${heartbeatRunEvents} where ${and(
|
||||
nativeRunLogSourceWhere(companyId, runId),
|
||||
gt(heartbeatRunEvents.seq, lastSourceCursor),
|
||||
)})`,
|
||||
),
|
||||
)
|
||||
.returning({ id: heartbeatRuns.id });
|
||||
if (!updated.length) throw new Error("native_run_log_recovery_source_changed");
|
||||
}
|
||||
}
|
||||
if (failure) throw failure;
|
||||
})();
|
||||
const recovery: Recovery = {
|
||||
controller,
|
||||
readers: 0,
|
||||
work: recover.finally(() => {
|
||||
pending.delete(logRef);
|
||||
finishWork();
|
||||
}),
|
||||
};
|
||||
pending.set(logRef, recovery);
|
||||
await awaitRecovery(recovery, requestSignal);
|
||||
}
|
||||
@@ -111,6 +111,10 @@ vi.mock("../usage-receipts.js", async (importOriginal) => ({
|
||||
createRunUsageRecorder: usageRecorder.create,
|
||||
}));
|
||||
|
||||
vi.mock("./native-usage-delivery.js", () => ({
|
||||
createNativeUsageDelivery: vi.fn(async () => ({ capture: async () => undefined })),
|
||||
}));
|
||||
|
||||
const githubAccess = vi.hoisted(() => ({
|
||||
activate: vi.fn((_binding: { runId: string }) => vi.fn()),
|
||||
stop: vi.fn(async () => undefined),
|
||||
@@ -141,6 +145,7 @@ type BackendFactoryOptions = {
|
||||
};
|
||||
|
||||
type RunnerTransportOptions = {
|
||||
onAccountingEvent?: (event: PrpEvent) => Promise<void>;
|
||||
environment?: NodeJS.ProcessEnv;
|
||||
acpxRuntimeDirectory?: string;
|
||||
adoptExistingRunner?: { pid: number; isAlive: () => Promise<boolean> | boolean };
|
||||
@@ -250,8 +255,8 @@ vi.mock("./paperclip-control-plane-port.js", async importOriginal => {
|
||||
return { ...original, PaperclipControlPlanePort: class extends original.PaperclipControlPlanePort {
|
||||
constructor(...args: ConstructorParameters<typeof original.PaperclipControlPlanePort>) {
|
||||
super(...args);
|
||||
accountingEvents.committed = args[2]?.onCommittedEvent;
|
||||
accountingEvents.duplicate = args[2]?.onDuplicateEvent;
|
||||
accountingEvents.committed = args[2]?.onCommittedEvent ? event => args[2]!.onCommittedEvent!(event, 1) : undefined;
|
||||
accountingEvents.duplicate = args[2]?.onDuplicateEvent ? event => args[2]!.onDuplicateEvent!(event, 1) : undefined;
|
||||
}
|
||||
} };
|
||||
});
|
||||
@@ -7159,6 +7164,36 @@ describe("native terminal-turn accounting", () => {
|
||||
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true }));
|
||||
expect(usageRecorder.create).not.toHaveBeenCalled();
|
||||
});
|
||||
it("binds retained warm transport usage to the successor run authority", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "native-warm-usage-"));
|
||||
const previous = process.env.PAPERCLIP_RUNNER_STATE_DIR;
|
||||
process.env.PAPERCLIP_RUNNER_STATE_DIR = directory;
|
||||
const first = { ...execution, binding: { ...execution.binding, runId: "first-warm-usage", executionWorkspaceId: "warm-usage-workspace" },
|
||||
session: { ...execution.session, normalizedSessionId: "warm-usage-session", lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60000 } } } as NativeExecutionInputV1;
|
||||
const second = { ...first, binding: { ...first.binding, runId: "second-warm-usage" } };
|
||||
const firstCapture = vi.fn(async (_event: PrpEvent) => {}), secondCapture = vi.fn(async (_event: PrpEvent) => {});
|
||||
try {
|
||||
await createRunnerdBackend({ db: leaseDb(first), execution: first, runnerInstanceId: "warm-producer", onAccountingEvent: firstCapture });
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
const retainedHook = state.createTransport.mock.calls.at(-1)![0].onAccountingEvent!;
|
||||
const firstReport = { eventType: "usage.reported", runId: first.binding.runId, sourceInstanceId: "warm-producer" } as PrpEvent;
|
||||
await retainedHook(firstReport);
|
||||
expect(firstCapture).toHaveBeenCalledExactlyOnceWith(firstReport);
|
||||
await rm(state.createTransport.mock.calls.at(-1)![0].stateDirectory!, { recursive: true, force: true });
|
||||
await createRunnerdBackend({ db: leaseDb(second), execution: second, runnerInstanceId: "warm-producer", onAccountingEvent: secondCapture });
|
||||
await expect(retainedHook({ ...firstReport, eventType: "run.attached" })).resolves.toBeUndefined();
|
||||
expect(secondCapture).not.toHaveBeenCalled();
|
||||
const secondReport = { ...firstReport, runId: second.binding.runId };
|
||||
await retainedHook(secondReport);
|
||||
expect(secondCapture).toHaveBeenCalledExactlyOnceWith(secondReport);
|
||||
expect(firstCapture).toHaveBeenCalledTimes(1);
|
||||
await expect(retainedHook(firstReport)).rejects.toThrow("native_usage_run_binding_mismatch");
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR; else process.env.PAPERCLIP_RUNNER_STATE_DIR = previous;
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("uses the canonical recorder for partial and completed usage when no onUsage callback is supplied", async () => {
|
||||
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
|
||||
state.execute.mockReset().mockImplementationOnce(async () => {
|
||||
@@ -12367,7 +12402,14 @@ describe("runnerd provider runtime wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["github", "assigned"] as const)("cleans run-owned access after overlapping executor failure: %s", async failure => {
|
||||
it.each((["codex", "opencode"] as const).flatMap(provider =>
|
||||
(["github", "assigned"] as const).map(failure => ({ provider, failure })),
|
||||
))("cleans run-owned access after overlapping $provider executor failure: $failure", async ({ provider, failure }) => {
|
||||
const current = {
|
||||
...execution,
|
||||
provider: provider === "codex" ? execution.provider : { kind: "opencode", model: null },
|
||||
session: { ...execution.session, driverKind: provider === "codex" ? "codex_app_server" : "opencode_server" },
|
||||
} as NativeExecutionInputV1;
|
||||
const issueService = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({
|
||||
update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })),
|
||||
} as unknown as ReturnType<typeof issueServiceModule.issueService>);
|
||||
@@ -12393,7 +12435,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
state.createBackend.mockClear();
|
||||
let settled = false;
|
||||
const running = executePaperclipNativeSession({
|
||||
db: leaseDb(execution), execution, runnerInstanceId: "runner-overlap-cleanup",
|
||||
db: leaseDb(current), execution: current, runnerInstanceId: "runner-overlap-cleanup",
|
||||
useRunnerd: true, managedGitHub: true,
|
||||
runnerEnvironment: {
|
||||
PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned",
|
||||
@@ -12419,7 +12461,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
expect(state.execute).not.toHaveBeenCalled();
|
||||
expect(state.createBackend).not.toHaveBeenCalled();
|
||||
if (failure === "assigned") {
|
||||
expect(activate).toHaveBeenCalledWith(execution.binding);
|
||||
expect(activate).toHaveBeenCalledWith(current.binding);
|
||||
expect(deactivate).toHaveBeenCalledOnce();
|
||||
expect(stop).toHaveBeenCalledOnce();
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
|
||||
import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js";
|
||||
export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js";
|
||||
import { measureRunnerOperation, recordRunnerTimingReceipt } from "./runner-operation-timing.js";
|
||||
import { formatNativeRunLogEvent } from "./native-run-log-export.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeResultCommit } from "../native-result-commit-notifications.js";
|
||||
import { releaseRetainedNativeSessionCleanup } from "../retained-native-session-cleanup-notifications.js";
|
||||
@@ -107,7 +111,7 @@ import type {
|
||||
AdapterExecutionResult,
|
||||
AdapterRuntimeEvent,
|
||||
} from "../../adapters/index.js";
|
||||
import { usdToUnits, type NativeFinalizationResult } from "@paperclipai/shared";
|
||||
import type { NativeFinalizationResult } from "@paperclipai/shared";
|
||||
import type {
|
||||
DurablePrpControlPlane,
|
||||
HarnessRuntimeRequestResolution,
|
||||
@@ -1754,12 +1758,13 @@ export async function synchronizeCompletedProviderPlan(input: {
|
||||
};
|
||||
}
|
||||
|
||||
class SessionToolAuthorityEpoch {
|
||||
class SessionAuthorityEpoch {
|
||||
readonly runId: string;
|
||||
#authority: PaperclipRunnerToolAuthority;
|
||||
#revoked = false;
|
||||
|
||||
constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace) {
|
||||
constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace,
|
||||
private readonly onAccountingEvent?: (event: PrpEvent) => Promise<void>) {
|
||||
this.runId = runId;
|
||||
this.#authority = authority;
|
||||
}
|
||||
@@ -1775,6 +1780,14 @@ class SessionToolAuthorityEpoch {
|
||||
}
|
||||
}
|
||||
|
||||
async acceptAccountingEvent(event: PrpEvent) {
|
||||
if (!isNativeUsageAccountingEvent(event)) return;
|
||||
this.#assertCurrent();
|
||||
if (event.runId !== this.runId) throw new Error("native_usage_run_binding_mismatch");
|
||||
await this.onAccountingEvent?.(event);
|
||||
this.#assertCurrent();
|
||||
}
|
||||
|
||||
definitions() {
|
||||
this.#assertCurrent();
|
||||
return this.#authority.definitions();
|
||||
@@ -1795,7 +1808,7 @@ class SessionToolAuthorityEpoch {
|
||||
}
|
||||
}
|
||||
|
||||
const sessionToolAuthorityEpochs = new Map<string, SessionToolAuthorityEpoch>();
|
||||
const sessionToolAuthorityEpochs = new Map<string, SessionAuthorityEpoch>();
|
||||
const initializingSessionToolAuthorities = new Set<string>();
|
||||
const executingRunnerdSessionScopes = new Map<string, string>();
|
||||
const executingNativeOwnerScopes = new Map<string, symbol>();
|
||||
@@ -7873,6 +7886,7 @@ export async function executePaperclipNativeSession(input: {
|
||||
/** Internal, run-owned file inspection lifetime; never supplied by tool arguments. */
|
||||
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
|
||||
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
||||
onPersistedNativeLog?: (cursor: number, chunk: string) => Promise<void>;
|
||||
onEvent?: (event: AdapterRuntimeEvent) => Promise<void>;
|
||||
/** Only this run's registered private instruction entry.
|
||||
* Probe at terminal; persist only after owned shutdown. */
|
||||
@@ -8618,7 +8632,15 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent);
|
||||
}
|
||||
let fallbackUsageRecorder: ReturnType<typeof createRunUsageRecorder> | undefined;
|
||||
const producerOwnsUsage = usesNativeUsageDelivery({ runnerd: input.useRunnerd === true,
|
||||
remote: input.runnerExecutionTarget?.kind === "remote", provider: input.execution.provider });
|
||||
const producerUsage = producerOwnsUsage ? await createNativeUsageDelivery(input.db, {
|
||||
companyId: input.execution.binding.companyId, runId: input.execution.binding.runId,
|
||||
runnerInstanceId: effectiveRunnerInstanceId, provider: input.execution.provider,
|
||||
billing: resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity),
|
||||
}) : undefined;
|
||||
const persistAccountingUsage = async (usage: Record<string, unknown> | null, complete: boolean) => {
|
||||
if (producerOwnsUsage) return;
|
||||
const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" });
|
||||
const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider) : undefined;
|
||||
complete = complete && accountingUsage !== undefined;
|
||||
@@ -8721,7 +8743,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
attributes:{eventKind:event.eventType}});
|
||||
}
|
||||
},
|
||||
onCommittedEvent: async (event) => {
|
||||
onCommittedEvent: async (event, cursor) => {
|
||||
if (event.eventType === "run.result.proposed") {
|
||||
const at = Date.now();
|
||||
await trace.record({name:"completion.tool_accepted",parentName:"native.session.execute",startedAtMs:at,endedAtMs:at});
|
||||
@@ -8874,7 +8896,8 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
});
|
||||
trace.activate(taskSettleScope);
|
||||
}
|
||||
if (input.onLog)
|
||||
if (input.onPersistedNativeLog) await input.onPersistedNativeLog(cursor, formatNativeRunLogEvent(event));
|
||||
else if (input.onLog)
|
||||
await input.onLog(
|
||||
"stdout",
|
||||
`${JSON.stringify({ type: "paperclip.prp.event", event })}\n`,
|
||||
@@ -8953,12 +8976,10 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
},
|
||||
);
|
||||
},
|
||||
onDuplicateEvent: async (event) => {
|
||||
onDuplicateEvent: async (event, cursor) => {
|
||||
await input.onPersistedNativeLog?.(cursor, formatNativeRunLogEvent(event));
|
||||
if (!firstTokenRecorded) observeDeliverableText(event);
|
||||
await observeAccountingEvent(event);
|
||||
// A crash can happen after the event commit but before its callback
|
||||
// finishes. Recover only idempotent durable projections here; activity,
|
||||
// publication, logging, trace, and metric effects remain committed-only.
|
||||
await liveQuestions.observe(event);
|
||||
await projectSessionGoalEvent(event);
|
||||
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
|
||||
@@ -9230,7 +9251,8 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
return input.runnerEnvironment;
|
||||
};
|
||||
const overlapGitHubPreparation = needsCoreGitHubAccess(input) && input.useRunnerd && input.backend === undefined
|
||||
&& input.runnerExecutionTarget?.kind === "remote" && input.execution.provider.kind === "codex"
|
||||
&& input.runnerExecutionTarget?.kind === "remote"
|
||||
&& (input.execution.provider.kind === "codex" || input.execution.provider.kind === "opencode")
|
||||
&& input.execution.schema !== "paperclip.native-execution-input.v6"
|
||||
&& input.execution.schema !== "paperclip.native-execution-input.v8";
|
||||
if (!overlapGitHubPreparation) await prepareRunnerEnvironment();
|
||||
@@ -9243,6 +9265,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
input.useRunnerd && input.backend === undefined
|
||||
? await timeController("backend_setup", () => createRunnerdBackend({
|
||||
...input,
|
||||
onAccountingEvent: producerUsage?.capture,
|
||||
controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? {
|
||||
bootId: controller.bootId,
|
||||
runnerArtifact: restoredController ? { version: restoredController.runnerVersion, digest: restoredController.runnerDigest } : undefined,
|
||||
@@ -9289,7 +9312,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
persistWarmNativeCheckpoint(remoteWarmRecovery.previousExecution,
|
||||
nativeSessionConfigDigest(remoteWarmRecovery.previousExecution, "remote"), await recovered.session.snapshot());
|
||||
persistedWarmSession = loadWarmNativeCheckpoint(input.execution, warmConfigDigest!, "remote");
|
||||
runnerdBackend = await createRunnerdBackend({ ...input, runnerInstanceId: effectiveRunnerInstanceId,
|
||||
runnerdBackend = await createRunnerdBackend({ ...input, onAccountingEvent: producerUsage?.capture, runnerInstanceId: effectiveRunnerInstanceId,
|
||||
controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? { bootId: controller.bootId } : undefined,
|
||||
durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, trace, toolTrace });
|
||||
}
|
||||
@@ -10334,151 +10357,6 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
return adapterResult;
|
||||
}
|
||||
|
||||
function numericUsageField(
|
||||
usage: Record<string, unknown> | null,
|
||||
keys: string[],
|
||||
): number | undefined {
|
||||
if (!usage) return undefined;
|
||||
for (const key of keys) {
|
||||
const value = usage[key];
|
||||
if (typeof value === "number" && Number.isFinite(value) && value >= 0)
|
||||
return value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function nativeUsageMeasurement(usage: Record<string, unknown>) {
|
||||
const nestedUsage = record(usage.usage);
|
||||
// An explicit partial delta must not fall through to a full session total.
|
||||
if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta);
|
||||
if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta);
|
||||
const candidates = [
|
||||
record(usage.runDelta),
|
||||
record(nestedUsage.runDelta),
|
||||
record(usage.total),
|
||||
record(nestedUsage.total),
|
||||
record(usage.cumulative),
|
||||
record(nestedUsage.cumulative),
|
||||
nestedUsage,
|
||||
usage,
|
||||
];
|
||||
return (
|
||||
candidates.find(
|
||||
(candidate) =>
|
||||
numericUsageField(candidate, [
|
||||
"inputTokens",
|
||||
"input",
|
||||
"promptTokens",
|
||||
"outputTokens",
|
||||
"output",
|
||||
"completionTokens",
|
||||
]) !== undefined,
|
||||
) ?? usage
|
||||
);
|
||||
}
|
||||
|
||||
export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string {
|
||||
if (provider.kind === "acpx") {
|
||||
if (provider.agent === "cursor") return "cursor";
|
||||
if (provider.agent === "copilot") return "github";
|
||||
if (provider.agent === "pi") return "openrouter";
|
||||
}
|
||||
return "openai";
|
||||
}
|
||||
|
||||
function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) {
|
||||
const turns = new Map<string, { usage: Record<string, unknown> | null; terminal: boolean }>();
|
||||
const sequences = new Map<string, number>();
|
||||
let currentTurn: string | undefined;
|
||||
const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"];
|
||||
const snapshot = () => {
|
||||
if (turns.size === 0) return { usage: null, turnId: undefined, complete: false };
|
||||
let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
|
||||
let cost = 0n, costKnown = false, costIncomplete = false, complete = true;
|
||||
for (const [turnId, turn] of turns) {
|
||||
const normalized = normalizeNativeUsage(turn.usage);
|
||||
complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined;
|
||||
inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0);
|
||||
outputTokens += normalized?.outputTokens ?? 0;
|
||||
cacheReadTokens += normalized?.cachedInputTokens ?? 0;
|
||||
cacheWriteTokens += normalized?.cacheWriteTokens ?? 0;
|
||||
const price = nativeUsageCostUsd(turn.usage, provider);
|
||||
if (price === undefined) costIncomplete = true;
|
||||
else { cost += usdToUnits(price); costKnown = true; }
|
||||
}
|
||||
// A free empty attempt contributes no price evidence for later paid work.
|
||||
// Keep an incomplete zero subtotal absent so complete aggregate tokens can
|
||||
// still receive an estimate. Preserve positive known spend and true zero totals.
|
||||
const retainCost = costKnown && (!costIncomplete || cost > 0n);
|
||||
const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
|
||||
...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) };
|
||||
const current = currentTurn ? turns.get(currentTurn) : undefined;
|
||||
return { usage: { runDelta, accountingCostIncomplete: costIncomplete,
|
||||
...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) },
|
||||
turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete };
|
||||
};
|
||||
return {
|
||||
observe(event: PrpEvent) {
|
||||
if (!event) return false;
|
||||
const payload = record(event.payload);
|
||||
const terminal = terminalTypes.includes(event.eventType);
|
||||
if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false;
|
||||
// Historical replay can revisit an earlier report after restoration.
|
||||
// Re-persist its aggregate, but never rewind or count that report twice.
|
||||
if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) {
|
||||
if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true;
|
||||
sequences.set(event.sourceInstanceId, event.sourceSeq);
|
||||
}
|
||||
const key = event.turnId ?? "unattributed";
|
||||
const turn = turns.get(key) ?? { usage: null, terminal: false };
|
||||
if (event.eventType === "turn.started") {
|
||||
currentTurn = event.turnId;
|
||||
turn.usage = null;
|
||||
turn.terminal = false;
|
||||
} else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId;
|
||||
if (payload.kind === "usage") {
|
||||
const usage = record(payload.usage);
|
||||
turn.usage = Object.hasOwn(usage, "runDelta")
|
||||
? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null;
|
||||
}
|
||||
if (terminal) turn.terminal = true;
|
||||
turns.set(key, turn);
|
||||
return true;
|
||||
},
|
||||
snapshot,
|
||||
finish(turnId: string | null) {
|
||||
const turn = turnId ? turns.get(turnId) : undefined;
|
||||
if (turn) turn.terminal = true;
|
||||
return snapshot();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function nativeUsageCostUsd(
|
||||
usage: Record<string, unknown> | null,
|
||||
provider?: NativeExecutionInput["provider"],
|
||||
) {
|
||||
// The ACP normalization contract fills absent per-turn cost with zero and
|
||||
// reports actual cost cumulatively. Until it carries an authoritative run
|
||||
// delta with provenance, neither value is a candidate's billed USD receipt.
|
||||
if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined;
|
||||
if (!usage) return undefined;
|
||||
const measurement = nativeUsageMeasurement(usage);
|
||||
const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta");
|
||||
const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])
|
||||
?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]));
|
||||
// PRP currently defaults an absent providerCostUsd to zero. That zero is
|
||||
// not proof of a free request when the runtime observed token consumption.
|
||||
if (direct !== undefined && direct > 0) return direct;
|
||||
const normalized = normalizeNativeUsage(usage);
|
||||
if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0;
|
||||
const cost = record(hasRunDelta ? measurement.cost : usage.cost);
|
||||
const currency =
|
||||
typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
|
||||
if (currency !== "USD") return undefined;
|
||||
return numericUsageField(cost, ["amount", "total"]);
|
||||
}
|
||||
|
||||
/** Billing identity follows the selected runtime and resolved credentials.
|
||||
* Unknown authentication remains unknown; a provider name does not prove billing mode. */
|
||||
export function resolveNativeBilling(provider: NativeExecutionInput["provider"], env: NodeJS.ProcessEnv = {}, identity?: Pick<AdapterExecutionResult, "provider" | "biller" | "billingType">): Pick<AdapterExecutionResult, "provider" | "biller" | "billingType"> {
|
||||
@@ -10511,33 +10389,6 @@ export function resolveNativeBilling(provider: NativeExecutionInput["provider"],
|
||||
return { provider: "openai", biller: billingType === "metered_api" ? inferOpenAiCompatibleBiller(env, "openai") : "unknown", billingType };
|
||||
}
|
||||
|
||||
export function normalizeNativeUsage(usage: Record<string, unknown> | null, options: { inputIncludesCacheReads?: boolean } = {}) {
|
||||
if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined;
|
||||
const measurement = nativeUsageMeasurement(usage);
|
||||
const cache = record(measurement.cache);
|
||||
const count = (source: Record<string, unknown>, keys: string[]) => {
|
||||
for (const key of keys) {
|
||||
if (source[key] === undefined) continue;
|
||||
const value = source[key];
|
||||
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const input = count(measurement, ["inputTokens", "input", "promptTokens"]);
|
||||
const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]);
|
||||
const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]);
|
||||
const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]);
|
||||
if (input === undefined || outputTokens === undefined
|
||||
|| ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined;
|
||||
const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0)
|
||||
+ (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0);
|
||||
if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined;
|
||||
return {
|
||||
inputTokens, outputTokens,
|
||||
...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
|
||||
...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }),
|
||||
};
|
||||
}
|
||||
|
||||
function processEnvironment(
|
||||
environment: NodeJS.ProcessEnv,
|
||||
@@ -11923,6 +11774,7 @@ export async function createRunnerdBackend(input: {
|
||||
casesEnabled?: boolean;
|
||||
db: Db;
|
||||
controllerPersistence?: { bootId: string; runnerArtifact?: { version: string; digest: string } };
|
||||
onAccountingEvent?: (event: PrpEvent) => Promise<void>;
|
||||
execution: NativeExecutionInput;
|
||||
runnerInstanceId: string;
|
||||
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
|
||||
@@ -12238,13 +12090,14 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
syncIssueExternalObjects: input.syncIssueExternalObjects,
|
||||
enqueueWakeup: input.enqueueWakeup,
|
||||
});
|
||||
const authorityEpoch = new SessionToolAuthorityEpoch(
|
||||
const authorityEpoch = new SessionAuthorityEpoch(
|
||||
input.execution.binding.runId,
|
||||
authority,
|
||||
input.toolTrace,
|
||||
input.onAccountingEvent,
|
||||
);
|
||||
let dynamicTools: Awaited<
|
||||
ReturnType<SessionToolAuthorityEpoch["definitions"]>
|
||||
ReturnType<SessionAuthorityEpoch["definitions"]>
|
||||
>;
|
||||
try {
|
||||
dynamicTools = await timeController("tool_authority", () => authorityEpoch.definitions());
|
||||
@@ -12346,7 +12199,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
authority.binding.readRemoteTaskFile = file => owner.readWorkspaceFile(file.contentRef, file.byteSize);
|
||||
dynamicTools = await authorityEpoch.definitions();
|
||||
}
|
||||
const executeTool = (call: Parameters<SessionToolAuthorityEpoch["execute"]>[0]) => {
|
||||
const executeTool = (call: Parameters<SessionAuthorityEpoch["execute"]>[0]) => {
|
||||
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
|
||||
if (!current) throw new Error("native_session_tool_authority_unavailable");
|
||||
return current.execute(call);
|
||||
@@ -12441,7 +12294,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
};
|
||||
const backendOptions = {
|
||||
onSpawn: input.onSpawn, dynamicTools,
|
||||
dynamicToolHandler: async (call: Parameters<SessionToolAuthorityEpoch["execute"]>[0]) => {
|
||||
dynamicToolHandler: async (call: Parameters<SessionAuthorityEpoch["execute"]>[0]) => {
|
||||
await assertExternalNativeRunAuthority(input.db, externalExecution); return authorityEpoch.execute(call);
|
||||
},
|
||||
completionFeedback: async (result: PrpStructuredRunResult) => {
|
||||
@@ -13782,7 +13635,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
markSpawned?.();
|
||||
}
|
||||
const executeCurrentToolAuthority = (
|
||||
call: Parameters<SessionToolAuthorityEpoch["execute"]>[0],
|
||||
call: Parameters<SessionAuthorityEpoch["execute"]>[0],
|
||||
) => {
|
||||
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
|
||||
if (!current) throw new Error("native_session_tool_authority_unavailable");
|
||||
@@ -13830,6 +13683,11 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
onSpawn: input.onSpawn,
|
||||
...(input.execution.provider.kind === "codex" ? { disableCodexWebSearch: "webSearch" in input.execution.provider && input.execution.provider.webSearch === "disabled" } : {}),
|
||||
homeMode: preconfiguredHome ? "preconfigured" : "isolated",
|
||||
onAccountingEvent: input.onAccountingEvent ? async event => {
|
||||
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
|
||||
if (!current) throw new Error("native_session_authority_unavailable");
|
||||
await current.acceptAccountingEvent(event);
|
||||
} : undefined,
|
||||
onCredentialsUpdated: (event) => appendNativeCredentialUpdate(input.db, {
|
||||
companyId: input.execution.binding.companyId,
|
||||
agentId: input.execution.binding.agentId,
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createNativeTurnAccounting, nativeUsageCostUsd, normalizeNativeUsage, usesNativeUsageDelivery } from "./native-usage-accounting.js";
|
||||
|
||||
const provider = { kind: "opencode" as const, model: "test-model" };
|
||||
const event = (sourceSeq: number, eventType: string, turnId = "turn-1", payload: Record<string, unknown> = {}) => ({
|
||||
sourceInstanceId: "producer", sourceSeq, eventType, turnId, payload,
|
||||
} as PrpEvent);
|
||||
|
||||
describe("native turn accounting", () => {
|
||||
it("selects producer delivery only for supported remote Runner execution", () => {
|
||||
expect(usesNativeUsageDelivery({ runnerd: true, remote: true, provider })).toBe(true);
|
||||
expect(usesNativeUsageDelivery({ runnerd: false, remote: true, provider })).toBe(false);
|
||||
expect(usesNativeUsageDelivery({ runnerd: true, remote: false, provider })).toBe(false);
|
||||
expect(usesNativeUsageDelivery({ runnerd: true, remote: true })).toBe(false);
|
||||
expect(usesNativeUsageDelivery({ runnerd: true, remote: true, provider: { kind: "muse", model: null } as never })).toBe(false);
|
||||
});
|
||||
|
||||
it("reconstructs cumulative turn usage without counting repeated producer events twice", () => {
|
||||
const accounting = createNativeTurnAccounting(provider);
|
||||
const usage = { kind: "usage", usage: { runDeltaComplete: true, runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: 0.25 } } };
|
||||
const history = [event(1, "turn.started"), event(2, "item.completed", "turn-1", usage), event(3, "turn.completed"),
|
||||
event(4, "turn.started", "turn-2"), event(5, "item.completed", "turn-2", usage), event(6, "turn.completed", "turn-2")];
|
||||
history.forEach(value => accounting.observe(value));
|
||||
history.forEach(value => accounting.observe(value));
|
||||
const snapshot = accounting.snapshot();
|
||||
expect(snapshot.complete).toBe(true);
|
||||
expect(normalizeNativeUsage(snapshot.usage)).toEqual({ inputTokens: 240, outputTokens: 20, cachedInputTokens: 0, cacheWriteTokens: 0 });
|
||||
expect(nativeUsageCostUsd(snapshot.usage, provider)).toBe(0.5);
|
||||
});
|
||||
|
||||
it("keeps an incomplete provider report pending instead of trusting placeholder zeroes", () => {
|
||||
const accounting = createNativeTurnAccounting(provider);
|
||||
accounting.observe(event(1, "turn.started"));
|
||||
accounting.observe(event(2, "item.completed", "turn-1", { kind: "usage", usage: {
|
||||
runDeltaComplete: false, runDelta: { inputTokens: 0, outputTokens: 0, providerCostUsd: 0 },
|
||||
} }));
|
||||
accounting.observe(event(3, "turn.completed"));
|
||||
expect(accounting.snapshot().complete).toBe(false);
|
||||
expect(accounting.snapshot().usage?.accountingCostIncomplete).toBe(true);
|
||||
expect(normalizeNativeUsage({ runDeltaComplete: false, runDelta: { inputTokens: 0, outputTokens: 0 } })).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
import { usdToUnits } from "@paperclipai/shared";
|
||||
import type { NativeExecutionInput, PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
const record = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
|
||||
function numericUsageField(
|
||||
usage: Record<string, unknown> | null,
|
||||
keys: string[],
|
||||
): number | undefined {
|
||||
if (!usage) return undefined;
|
||||
for (const key of keys) {
|
||||
const value = usage[key];
|
||||
if (typeof value === "number" && Number.isFinite(value) && value >= 0)
|
||||
return value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function nativeUsageMeasurement(usage: Record<string, unknown>) {
|
||||
const nestedUsage = record(usage.usage);
|
||||
// An explicit partial delta must not fall through to a full session total.
|
||||
if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta);
|
||||
if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta);
|
||||
const candidates = [
|
||||
record(usage.runDelta),
|
||||
record(nestedUsage.runDelta),
|
||||
record(usage.total),
|
||||
record(nestedUsage.total),
|
||||
record(usage.cumulative),
|
||||
record(nestedUsage.cumulative),
|
||||
nestedUsage,
|
||||
usage,
|
||||
];
|
||||
return (
|
||||
candidates.find(
|
||||
(candidate) =>
|
||||
numericUsageField(candidate, [
|
||||
"inputTokens",
|
||||
"input",
|
||||
"promptTokens",
|
||||
"outputTokens",
|
||||
"output",
|
||||
"completionTokens",
|
||||
]) !== undefined,
|
||||
) ?? usage
|
||||
);
|
||||
}
|
||||
|
||||
export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string {
|
||||
if (provider.kind === "acpx") {
|
||||
if (provider.agent === "cursor") return "cursor";
|
||||
if (provider.agent === "copilot") return "github";
|
||||
if (provider.agent === "pi") return "openrouter";
|
||||
}
|
||||
return "openai";
|
||||
}
|
||||
|
||||
export function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) {
|
||||
const turns = new Map<string, { usage: Record<string, unknown> | null; terminal: boolean }>();
|
||||
const sequences = new Map<string, number>();
|
||||
let currentTurn: string | undefined;
|
||||
const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"];
|
||||
const snapshot = () => {
|
||||
if (turns.size === 0) return { usage: null, turnId: undefined, complete: false };
|
||||
let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
|
||||
let cost = 0n, costKnown = false, costIncomplete = false, complete = true;
|
||||
for (const [turnId, turn] of turns) {
|
||||
const normalized = normalizeNativeUsage(turn.usage);
|
||||
complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined;
|
||||
inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0);
|
||||
outputTokens += normalized?.outputTokens ?? 0;
|
||||
cacheReadTokens += normalized?.cachedInputTokens ?? 0;
|
||||
cacheWriteTokens += normalized?.cacheWriteTokens ?? 0;
|
||||
const price = nativeUsageCostUsd(turn.usage, provider);
|
||||
if (price === undefined) costIncomplete = true;
|
||||
else { cost += usdToUnits(price); costKnown = true; }
|
||||
}
|
||||
// A free empty attempt contributes no price evidence for later paid work.
|
||||
// Keep an incomplete zero subtotal absent so complete aggregate tokens can
|
||||
// still receive an estimate. Preserve positive known spend and true zero totals.
|
||||
const retainCost = costKnown && (!costIncomplete || cost > 0n);
|
||||
const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
|
||||
...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) };
|
||||
const current = currentTurn ? turns.get(currentTurn) : undefined;
|
||||
return { usage: { runDelta, accountingCostIncomplete: costIncomplete,
|
||||
...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) },
|
||||
turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete };
|
||||
};
|
||||
return {
|
||||
observe(event: PrpEvent) {
|
||||
if (!event) return false;
|
||||
const payload = record(event.payload);
|
||||
const terminal = terminalTypes.includes(event.eventType);
|
||||
if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false;
|
||||
// Historical replay can revisit an earlier report after restoration.
|
||||
// Re-persist its aggregate, but never rewind or count that report twice.
|
||||
if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) {
|
||||
if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true;
|
||||
sequences.set(event.sourceInstanceId, event.sourceSeq);
|
||||
}
|
||||
const key = event.turnId ?? "unattributed";
|
||||
const turn = turns.get(key) ?? { usage: null, terminal: false };
|
||||
if (event.eventType === "turn.started") {
|
||||
currentTurn = event.turnId;
|
||||
turn.usage = null;
|
||||
turn.terminal = false;
|
||||
} else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId;
|
||||
if (payload.kind === "usage") {
|
||||
const usage = record(payload.usage);
|
||||
turn.usage = Object.hasOwn(usage, "runDelta")
|
||||
? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null;
|
||||
}
|
||||
if (terminal) turn.terminal = true;
|
||||
turns.set(key, turn);
|
||||
return true;
|
||||
},
|
||||
snapshot,
|
||||
finish(turnId: string | null) {
|
||||
const turn = turnId ? turns.get(turnId) : undefined;
|
||||
if (turn) turn.terminal = true;
|
||||
return snapshot();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function nativeUsageCostUsd(
|
||||
usage: Record<string, unknown> | null,
|
||||
provider?: NativeExecutionInput["provider"],
|
||||
) {
|
||||
// The ACP normalization contract fills absent per-turn cost with zero and
|
||||
// reports actual cost cumulatively. Until it carries an authoritative run
|
||||
// delta with provenance, neither value is a candidate's billed USD receipt.
|
||||
if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined;
|
||||
if (!usage) return undefined;
|
||||
const measurement = nativeUsageMeasurement(usage);
|
||||
const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta");
|
||||
const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])
|
||||
?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]));
|
||||
// PRP currently defaults an absent providerCostUsd to zero. That zero is
|
||||
// not proof of a free request when the runtime observed token consumption.
|
||||
if (direct !== undefined && direct > 0) return direct;
|
||||
const normalized = normalizeNativeUsage(usage);
|
||||
if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0;
|
||||
const cost = record(hasRunDelta ? measurement.cost : usage.cost);
|
||||
const currency =
|
||||
typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
|
||||
if (currency !== "USD") return undefined;
|
||||
return numericUsageField(cost, ["amount", "total"]);
|
||||
}
|
||||
|
||||
export function normalizeNativeUsage(usage: Record<string, unknown> | null, options: { inputIncludesCacheReads?: boolean } = {}) {
|
||||
if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined;
|
||||
const measurement = nativeUsageMeasurement(usage);
|
||||
const cache = record(measurement.cache);
|
||||
const count = (source: Record<string, unknown>, keys: string[]) => {
|
||||
for (const key of keys) {
|
||||
if (source[key] === undefined) continue;
|
||||
const value = source[key];
|
||||
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const input = count(measurement, ["inputTokens", "input", "promptTokens"]);
|
||||
const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]);
|
||||
const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]);
|
||||
const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]);
|
||||
if (input === undefined || outputTokens === undefined
|
||||
|| ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined;
|
||||
const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0)
|
||||
+ (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0);
|
||||
if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined;
|
||||
return {
|
||||
inputTokens, outputTokens,
|
||||
...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
|
||||
...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }),
|
||||
};
|
||||
}
|
||||
|
||||
export function usesNativeUsageDelivery(input: {
|
||||
runnerd: boolean; remote: boolean; provider?: NativeExecutionInput["provider"] | null;
|
||||
}) {
|
||||
return input.runnerd && input.remote && input.provider !== undefined && input.provider !== null
|
||||
&& ["codex", "opencode", "acpx", "claude_managed", "aws_agentcore"].includes(input.provider.kind);
|
||||
}
|
||||
|
||||
export function isNativeUsageAccountingEvent(event: Pick<PrpEvent, "eventType">) {
|
||||
return ["usage.reported", "turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(event.eventType);
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import { compareCents, usdToUnits } from "@paperclipai/shared";
|
||||
import { spawn, execFileSync, type ChildProcess } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtempSync, readFileSync, writeFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, runUsageReceipts, type Db } from "@paperclipai/db";
|
||||
import { type PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { accountRunCost } from "../run-cost-accounting.js";
|
||||
import { usageReceiptSpoolPath } from "../usage-receipts.js";
|
||||
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
|
||||
|
||||
const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url));
|
||||
const fixtureProfile = process.env.CI ? "release" : "debug";
|
||||
const billing = { provider: "openai", biller: "openai", billingType: "metered_api" as const };
|
||||
|
||||
describe("producer-owned native usage", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
beforeAll(async () => {
|
||||
execFileSync("cargo", ["build", ...(fixtureProfile === "release" ? ["--release"] : []), "--locked",
|
||||
"--manifest-path", resolve(runnerRoot, "Cargo.toml"), "-p", "paperclip-runner-core", "--example", "recovery-fixture"],
|
||||
{ timeout: 600_000, stdio: "pipe" });
|
||||
database = await startEmbeddedPostgresTestDatabase("native-usage-delivery-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 660_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
const seed = async () => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Usage delivery", issuePrefix: randomUUID().slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Producer", adapterType: "paperclip_runner" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "running", runtimeMode: "native", costAccountingPending: true });
|
||||
return { companyId, agentId, runId, runnerInstanceId: randomUUID(), provider: { kind: "codex" as const, model: "test-model" }, billing };
|
||||
};
|
||||
const event = (input: Pick<Awaited<ReturnType<typeof seed>>, "runId" | "runnerInstanceId">, sequence: number, type: string, turnId = "turn-1", amount = 0.25): PrpEvent => ({
|
||||
schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: input.runnerInstanceId,
|
||||
sourceEventId: `usage-${sequence}`, sourceSeq: sequence, priority: 0, runId: input.runId,
|
||||
normalizedSessionId: "session-1", turnId, emittedAt: new Date().toISOString(), eventType: type,
|
||||
payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {},
|
||||
} as PrpEvent);
|
||||
|
||||
it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => {
|
||||
const input = await seed();
|
||||
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
|
||||
const root = mkdtempSync(resolve(tmpdir(), "native-usage-runner-"));
|
||||
const receiverA = resolve(root, "receiver-a"), receiverB = resolve(root, "receiver-b");
|
||||
const identity = { runnerInstanceId: input.runnerInstanceId, environmentLeaseId: randomUUID(), runId: input.runId,
|
||||
normalizedSessionId: randomUUID(), turnId: "turn-1", itemId: "item-1" };
|
||||
const expectedRunnerVersion = "0.3.0", expectedRunnerDigest = `sha256:${"c".repeat(64)}`;
|
||||
let receiver: ChildProcess | undefined, child: ChildProcess | undefined;
|
||||
let exit: Promise<unknown[]> | undefined, failed = 0, lastCaptureError = "";
|
||||
const receiptFile = resolve(root, "producer.json"), releaseFile = resolve(root, "release"), scriptFile = resolve(root, "script.json");
|
||||
writeFileSync(scriptFile, JSON.stringify({ steps: 3, waitAfter: 2, retentionLimit: 8, receiptFile, releaseFile, accounting: true }));
|
||||
const receipts = () => db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId));
|
||||
try {
|
||||
|
||||
await db.execute(sql`CREATE FUNCTION fail_native_usage_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'receipt database unavailable'; END $$`);
|
||||
await db.execute(sql`CREATE TRIGGER fail_native_usage BEFORE INSERT ON run_usage_receipts FOR EACH ROW EXECUTE FUNCTION fail_native_usage_receipt()`);
|
||||
const startReceiver = () => new Promise<ChildProcess>((resolve, reject) => {
|
||||
const process = spawn(globalThis.process.execPath, ["--import", fileURLToPath(new URL("../../../node_modules/tsx/dist/loader.mjs", import.meta.url)), fileURLToPath(new URL("./__fixtures__/usage-receiver.ts", import.meta.url))], {
|
||||
cwd: fileURLToPath(new URL("../../../../", import.meta.url)), stdio: ["ignore", "ignore", "pipe", "ipc"], env: { ...globalThis.process.env },
|
||||
});
|
||||
let diagnostics = "";
|
||||
process.stderr!.on("data", bytes => { diagnostics = (diagnostics + String(bytes)).slice(-16000); });
|
||||
const timer = setTimeout(() => { process.kill("SIGKILL"); reject(new Error(`receiver startup timeout: ${diagnostics}`)); }, 60_000);
|
||||
process.once("exit", code => { clearTimeout(timer); reject(new Error(`receiver exited ${code}: ${diagnostics}`)); });
|
||||
process.on("message", message => {
|
||||
const kind = (message as { kind?: string }).kind;
|
||||
if (kind === "capture-failed") { failed++; lastCaptureError = String((message as { cause?: string }).cause); }
|
||||
if (kind === "ready") { clearTimeout(timer); resolve(process); }
|
||||
});
|
||||
});
|
||||
let requestId = 0;
|
||||
const request = (target: ChildProcess, value: Record<string, unknown>) => new Promise<Record<string, unknown>>((resolve, reject) => {
|
||||
const id = ++requestId;
|
||||
const timer = setTimeout(() => { target.off("message", listener); reject(new Error("receiver response timeout")); }, 30_000);
|
||||
const listener = (message: unknown) => {
|
||||
const result = message as Record<string, unknown>;
|
||||
if (result.requestId !== id) return;
|
||||
clearTimeout(timer); target.off("message", listener);
|
||||
if (result.error) reject(new Error(String(result.error))); else resolve(result);
|
||||
};
|
||||
target.on("message", listener);
|
||||
target.send({ ...value, requestId: id });
|
||||
});
|
||||
receiver = await startReceiver();
|
||||
const firstReceiver = await request(receiver, { kind: "start", connectionString: database.connectionString, directory: receiverA, identity, input });
|
||||
const connectUrl = String(firstReceiver.connectUrl), port = Number(new URL(connectUrl).port), ticket = String(firstReceiver.ticket);
|
||||
child = spawn(resolve(runnerRoot, `target/${fixtureProfile}/examples/recovery-fixture`), ["--fixture-script", scriptFile, "--connect-url", connectUrl,
|
||||
"--state-dir", resolve(root, "producer"), "--runner-id", identity.runnerInstanceId, "--environment-lease-id", identity.environmentLeaseId,
|
||||
"--run-id", identity.runId, "--session-id", identity.normalizedSessionId, "--turn-id", identity.turnId, "--item-id", identity.itemId,
|
||||
"--runner-version", expectedRunnerVersion, "--runner-digest", expectedRunnerDigest, "--max-outbox-bytes", "65536", "--p0-reserve-bytes", "16384", "--reconnect-grace-ms", "120000"],
|
||||
{ env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: ticket }, stdio: ["ignore", "ignore", "pipe"] });
|
||||
exit = once(child, "exit");
|
||||
await request(receiver, { kind: "command", type: "turn.start", commandId: "start-once" });
|
||||
writeFileSync(releaseFile, "produce");
|
||||
await vi.waitFor(() => expect(failed).toBeGreaterThan(0), { timeout: 10_000 });
|
||||
expect(await receipts()).toHaveLength(0);
|
||||
const producerState = () => JSON.parse(readFileSync(resolve(root, "producer/runner-state.json"), "utf8")) as {
|
||||
outbox: Array<{ eventType: string; envelope: { payload: { sourceEventId: string } } }>;
|
||||
};
|
||||
const pendingUsage = producerState().outbox.find(report => report.eventType === "usage.reported");
|
||||
expect(pendingUsage).toBeDefined();
|
||||
const pendingSourceEventId = pendingUsage!.envelope.payload.sourceEventId;
|
||||
const stoppedReceiver = await request(receiver, { kind: "stop" });
|
||||
rmSync(receiverA, { recursive: true, force: true });
|
||||
await db.execute(sql`DROP TRIGGER fail_native_usage ON run_usage_receipts`);
|
||||
await db.execute(sql`DROP FUNCTION fail_native_usage_receipt()`);
|
||||
|
||||
receiver = await startReceiver();
|
||||
await request(receiver, { kind: "start", connectionString: database.connectionString, directory: receiverB, identity, input, port, checkpoint: stoppedReceiver.checkpoint });
|
||||
await vi.waitFor(async () => expect((await receipts()).filter(row => row.complete)).toHaveLength(1), { timeout: 15_000 });
|
||||
await vi.waitFor(() => expect(producerState().outbox).toEqual([]), { timeout: 10_000 });
|
||||
expect((await receipts()).some(row => (row.receiptJson.producerEvent as Record<string, unknown>).sourceEventId === pendingSourceEventId)).toBe(true);
|
||||
expect((await receipts()).filter(row => row.receiptJson.producerEvent && (row.receiptJson.producerEvent as Record<string, unknown>).eventType === "usage.reported")).toHaveLength(1);
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(await accountRunCost(db, input.runId)).toBe(true);
|
||||
expect(await accountRunCost(db, input.runId)).toBe(false);
|
||||
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId));
|
||||
expect(ledger).toHaveLength(1);
|
||||
expect(ledger[0].costCents).toBe(25);
|
||||
const failedBeforeLateReport = failed;
|
||||
await request(receiver, { kind: "command", type: "turn.steer", commandId: "release-late-usage" });
|
||||
await vi.waitFor(() => expect(failed).toBeGreaterThan(failedBeforeLateReport), { timeout: 10_000 });
|
||||
expect(lastCaptureError).toContain("native_usage_after_finalization_requires_reconciliation");
|
||||
const retainedLate = producerState().outbox.find(report => report.eventType === "usage.reported");
|
||||
expect(retainedLate).toBeDefined();
|
||||
expect(retainedLate!.envelope.payload).toMatchObject({ runId: input.runId, sourceInstanceId: input.runnerInstanceId });
|
||||
expect((await receipts()).filter(row => (row.receiptJson.producerEvent as Record<string, unknown>)?.eventType === "usage.reported")).toHaveLength(1);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1);
|
||||
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
|
||||
} finally {
|
||||
await db.execute(sql`DROP TRIGGER IF EXISTS fail_native_usage ON run_usage_receipts`);
|
||||
await db.execute(sql`DROP FUNCTION IF EXISTS fail_native_usage_receipt()`);
|
||||
if (receiver && receiver.exitCode === null && receiver.signalCode === null) receiver.kill("SIGKILL");
|
||||
if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
await exit;
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
}, 240_000);
|
||||
|
||||
it("preserves source takeover fencing, budget holds, and missing-usage debt", async () => {
|
||||
const input = await seed();
|
||||
const first = await createNativeUsageDelivery(db, input);
|
||||
await db.insert(budgetPolicies).values({ companyId: input.companyId, scopeType: "agent", scopeId: input.agentId,
|
||||
metric: "billed_cents", windowKind: "lifetime", amount: 10, notifyEnabled: false });
|
||||
await first.capture(event(input, 1, "usage.reported"));
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, input.agentId));
|
||||
expect(agent.lifecycleHolds).toContain("budget");
|
||||
const replacement = { ...input, runnerInstanceId: randomUUID() };
|
||||
const second = await createNativeUsageDelivery(db, replacement);
|
||||
await second.capture(event(replacement, 1, "usage.reported", "turn-2", 0.5));
|
||||
await first.capture(event(input, 2, "turn.completed"));
|
||||
const [pending] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(pending.usageJson).toMatchObject({ costUsd: 0.5, accountingReceiptReady: false });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(await accountRunCost(db, input.runId)).toBe(false);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0);
|
||||
await second.capture(event(replacement, 2, "turn.completed", "turn-2"));
|
||||
expect(await accountRunCost(db, input.runId)).toBe(true);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(50);
|
||||
});
|
||||
|
||||
it.each(["tokens", "cost", "incomplete"])("accepts equal finalized snapshots and rejects changed %s without repricing", async change => {
|
||||
const input = await seed();
|
||||
const delivery = await createNativeUsageDelivery(db, input);
|
||||
await delivery.capture(event(input, 1, "usage.reported"));
|
||||
await delivery.capture(event(input, 2, "turn.completed"));
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(await accountRunCost(db, input.runId)).toBe(true);
|
||||
await delivery.capture(event(input, 3, "usage.reported"));
|
||||
const late = event(input, 4, "usage.reported");
|
||||
(late.payload as Record<string, unknown>).runDelta = { inputTokens: change === "tokens" ? 150 : 120, outputTokens: change === "tokens" ? 15 : 10, providerCostUsd: change === "incomplete" ? 0.25 : 0.5 };
|
||||
if (change === "incomplete") (late.payload as Record<string, unknown>).runDeltaAvailable = false;
|
||||
await expect(delivery.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
await expect(delivery.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
|
||||
});
|
||||
|
||||
it("accepts equal finalized rate-card reports without requiring terminal-only derived pricing", async () => {
|
||||
const input = { ...await seed(), provider: { kind: "codex" as const, model: "gpt-6.1-sol" } };
|
||||
const delivery = await createNativeUsageDelivery(db, input);
|
||||
const report = (sequence: number) => {
|
||||
const reported = event(input, sequence, "usage.reported");
|
||||
reported.payload = { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens: 1000, outputTokens: 100000 } };
|
||||
return reported;
|
||||
};
|
||||
await delivery.capture(report(1));
|
||||
await delivery.capture(event(input, 2, "turn.completed"));
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(await accountRunCost(db, input.runId)).toBe(true);
|
||||
await delivery.capture(report(3));
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(run.usageJson).toMatchObject({ pricingProvenance: { source: "rate_card" }, accountingReceiptReady: true });
|
||||
expect(usdToUnits(run.usageJson!.costUsdExact as string)).toBe(usdToUnits("1.002"));
|
||||
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId));
|
||||
expect(ledger).toHaveLength(1);
|
||||
expect(ledger[0].costCents).toBe(100.2);
|
||||
const correction = report(4);
|
||||
(correction.payload as Record<string, unknown>).runDelta = { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 2 };
|
||||
await expect(delivery.capture(correction)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
});
|
||||
|
||||
it("keeps successor exposure pending after prior-turn accounting invalidates its live snapshot", async () => {
|
||||
const input = await seed();
|
||||
const delivery = await createNativeUsageDelivery(db, input);
|
||||
const paid = event(input, 1, "usage.reported");
|
||||
paid.payload = { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens: 50, outputTokens: 5, providerCostUsd: 0.5 } };
|
||||
await delivery.capture(paid);
|
||||
const invalidated = event(input, 2, "usage.reported");
|
||||
invalidated.payload = { runDeltaBasis: "run", runDeltaAvailable: false, runDelta: {}, reason: "accounting_reconciliation_required",
|
||||
reconciliation: { receivedTurnId: "old-provider-turn", cumulative: { inputTokens: 120, outputTokens: 12 }, priorOrigin: { runId: "old-run" } } };
|
||||
await delivery.capture(invalidated);
|
||||
await delivery.capture(event(input, 3, "turn.failed"));
|
||||
await db.update(heartbeatRuns).set({ status: "failed", errorCode: "codex_usage_accounting_integrity", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(run).toMatchObject({ costAccountingPending: true, costAccountedAt: null,
|
||||
usageJson: { accountingReceiptReady: false, costStatus: "unpriced" } });
|
||||
expect(compareCents((run.usageJson?.budgetExposure as { costCents: string }).costCents, "50")).toBe(0);
|
||||
const [required] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 2)));
|
||||
expect(required.receiptJson.producerEvent).toMatchObject({ usage: { reconciliation: { receivedTurnId: "old-provider-turn", cumulative: { inputTokens: 120 }, priorOrigin: { runId: "old-run" } } } });
|
||||
expect(await accountRunCost(db, input.runId)).toBe(false);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("retains unmarked legacy Codex reports as incomplete unpriced debt", async () => {
|
||||
const input = { ...await seed(), provider: { kind: "codex" as const, model: "gpt-6.1-sol" } };
|
||||
const delivery = await createNativeUsageDelivery(db, input);
|
||||
const legacy = event(input, 1, "usage.reported");
|
||||
delete (legacy.payload as Record<string, unknown>).runDeltaBasis;
|
||||
await delivery.capture(legacy);
|
||||
await delivery.capture(event(input, 2, "turn.completed"));
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)))[0]).toMatchObject({
|
||||
costAccountingPending: true, costAccountedAt: null, usageJson: { accountingReceiptReady: false, costStatus: "unpriced" } });
|
||||
expect(await accountRunCost(db, input.runId)).toBe(false);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("replays partial and final usage once after a lost commit response and restores turn aggregation", async () => {
|
||||
const input = { ...await seed(), provider: { kind: "opencode" as const, model: "test-model" } };
|
||||
let loseResponse = false;
|
||||
const failingDb = new Proxy(db, { get(target, property) {
|
||||
if (property === "transaction") return async (...args: Parameters<Db["transaction"]>) => {
|
||||
const result = await db.transaction(...args);
|
||||
if (loseResponse) { loseResponse = false; throw new Error("lost commit response"); }
|
||||
return result;
|
||||
};
|
||||
const value = Reflect.get(target, property);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
} }) as Db;
|
||||
let delivery = await createNativeUsageDelivery(failingDb, input);
|
||||
const first = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed");
|
||||
loseResponse = true;
|
||||
await expect(delivery.capture(first)).rejects.toThrow("lost commit response");
|
||||
await delivery.capture(first);
|
||||
await delivery.capture(done);
|
||||
delivery = await createNativeUsageDelivery(db, input);
|
||||
await delivery.capture(first);
|
||||
await delivery.capture(done);
|
||||
const nextTurn = event(input, 3, "turn.started", "turn-1");
|
||||
nextTurn.payload = { providerTurnId: "provider-turn-2" };
|
||||
await delivery.capture(nextTurn);
|
||||
const partial = event(input, 4, "usage.reported", "turn-1", 0.5);
|
||||
(partial.payload as Record<string, unknown>).runDeltaAvailable = false;
|
||||
await delivery.capture(partial);
|
||||
const final = event(input, 5, "usage.reported", "turn-1", 0.5);
|
||||
await delivery.capture(final);
|
||||
const terminal = event(input, 6, "turn.completed", "turn-1");
|
||||
terminal.payload = { providerTurnId: "provider-turn-2" };
|
||||
await delivery.capture(terminal);
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(run.usageJson).toMatchObject({ inputTokens: 240, outputTokens: 20, accountingReceiptReady: true, costUsd: 0.75 });
|
||||
expect(run.usageJson?.accountingCaptureFailed).not.toBe(true);
|
||||
expect(await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1)))).toHaveLength(1);
|
||||
await expect(delivery.capture({ ...final, payload: { ...final.payload as object, runDeltaAvailable: false } })).rejects.toThrow("native_usage_source_conflict");
|
||||
});
|
||||
});
|
||||
Loaded 100 of 322 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user