Merge master into canonical Runner D delivery

Resolve the migration sequence against current master while preserving the audited Runner D data migration. Type the native backend factory mock so the merged TypeScript checks retain its two-argument contract.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dev AgentandPaperclip committed 2026-08-19 21:17:29 +00:00
commit 83a1c438e0
227 files changed
+64329 -3014

No files matched your search

@@ -177,19 +177,22 @@ vi.mock("../services/codex-device-login-service.js", async (importOriginal) => {
function createMemoryStore(): AdapterAuthSessionStore & { rows: Map<string, AdapterAuthSessionRow> } {
const rows = new Map<string, AdapterAuthSessionRow>();
const activeSlots = new Set<string>();
const slotKey = (companyId: string, adapterType: string) => `${companyId}|${adapterType}`;
// The active slot is scoped to the company, the owner, and the adapter.
const slotKey = (companyId: string, startedByUserId: string, adapterType: string) =>
`${companyId}|${startedByUserId}|${adapterType}`;
const isActive = (status: AdapterAuthSessionRow["status"]) =>
status === "starting" || status === "waiting_for_user" || status === "promoting";
return {
rows,
async insert(input) {
const key = slotKey(input.companyId, input.adapterType);
const key = slotKey(input.companyId, input.startedByUserId, input.adapterType);
if (activeSlots.has(key)) {
throw new AdapterAuthSessionConflictError();
}
activeSlots.add(key);
rows.set(input.id, {
id: input.id,
publicSessionId: input.publicSessionId,
companyId: input.companyId,
environmentId: input.environmentId,
adapterType: input.adapterType,
@@ -213,7 +216,8 @@ function createMemoryStore(): AdapterAuthSessionStore & { rows: Map<string, Adap
if (input.failureReason !== undefined) row.failureReason = input.failureReason;
if (input.finishedAt !== undefined) row.finishedAt = input.finishedAt;
if (input.promotionExpiresAt !== undefined) row.promotionExpiresAt = input.promotionExpiresAt;
if (!isActive(input.status)) activeSlots.delete(slotKey(row.companyId, row.adapterType));
if (!isActive(input.status))
activeSlots.delete(slotKey(row.companyId, row.startedByUserId, row.adapterType));
},
async compareAndSetStatus(input) {
const row = rows.get(input.sessionId);
@@ -222,14 +226,25 @@ function createMemoryStore(): AdapterAuthSessionStore & { rows: Map<string, Adap
if (input.failureReason !== undefined) row.failureReason = input.failureReason;
if (input.finishedAt !== undefined) row.finishedAt = input.finishedAt;
if (input.promotionExpiresAt !== undefined) row.promotionExpiresAt = input.promotionExpiresAt;
if (!isActive(input.status)) activeSlots.delete(slotKey(row.companyId, row.adapterType));
if (!isActive(input.status))
activeSlots.delete(slotKey(row.companyId, row.startedByUserId, row.adapterType));
return true;
},
async get(sessionId) {
const row = rows.get(sessionId);
return row ? { ...row } : null;
},
async withCompanyAdapterPromotionLock(_companyId, _adapterType, fn) {
async getByPublicId(publicSessionId, companyId) {
// Scope the read to the company and the public session id, so a
// foreign-company lookup reads nothing and the internal id never matches.
for (const row of rows.values()) {
if (row.publicSessionId === publicSessionId && row.companyId === companyId) {
return { ...row };
}
}
return null;
},
async withCompanyAdapterPromotionLock(_companyId, _startedByUserId, _adapterType, fn) {
// The route test runs on a single event loop, so it needs no real lock. The
// pass-through keeps the promotion contract satisfied.
return fn();
@@ -370,13 +385,21 @@ describe("adapter device-login routes", () => {
adapterType: "codex_local",
startedByUserId: OWNER_A,
});
// The row persists the immutable owner from the actor.
// The row persists the immutable owner from the actor. The response carries
// the public session id, so read the row by the public id, not the internal id.
const store = harness.store as ReturnType<typeof createMemoryStore>;
const row = store.rows.get(res.body.sessionId);
const row = await store.getByPublicId(res.body.sessionId, COMPANY_1);
expect(row?.startedByUserId).toBe(OWNER_A);
// The public session id is never the internal row id.
expect(row?.id).not.toBe(res.body.sessionId);
// No row is keyed by the public session id in the internal-id map.
expect(store.rows.get(res.body.sessionId)).toBeUndefined();
});
it("rejects a non-codex adapter", async () => {
it("rejects an adapter whose login capability drives a different transport", async () => {
// The Claude adapter declares a pseudo-terminal login, not a streamed-exec
// device login. The guard reads the capability transport, so it rejects the
// adapter with a fixed 400 before any lease.
const app = await createApp();
const res = await request(app)
@@ -387,6 +410,58 @@ describe("adapter device-login routes", () => {
expect(harness.acquisitions).toHaveLength(0);
});
it("starts a device login for a third adapter that declares the streamed-exec capability", async () => {
// A third adapter, not the Codex adapter, declares a streamed-exec login
// capability. The guard reads the registry capability, not the adapter name,
// so the adapter passes the guard and starts a session. This proves no
// adapter-name branch remains in the guard path. The test overrides an
// existing adapter type so the strict request schema accepts it.
const app = await createApp();
const { registerServerAdapter, unregisterServerAdapter } = await import("../adapters/index.js");
registerServerAdapter({
type: "gemini_local",
execute: async () => {
throw new Error("not used");
},
testEnvironment: async () => {
throw new Error("not used");
},
loginCapability: {
panelMode: "displayed_code",
sandboxTransport: "streamed_exec",
timeoutPolicy: "caller_bounded",
getCommand: () => "vendor login",
parsePrompt: () => null,
},
});
try {
const res = await request(app)
.post(loginPath(COMPANY_1, "gemini_local"))
.send({ environmentId: SANDBOX_ENV_1 });
expect(res.status, JSON.stringify(res.body)).toBe(201);
expect(res.body).toMatchObject({ environmentId: SANDBOX_ENV_1, status: "starting" });
expect(harness.acquisitions).toHaveLength(1);
expect(harness.acquisitions[0]).toMatchObject({ adapterType: "gemini_local" });
} finally {
unregisterServerAdapter("gemini_local");
}
});
it("rejects a malformed start body with the strict schema before any side effect", async () => {
const app = await createApp();
// The strict start schema rejects an unknown field. The old lax parse
// accepted an extra field and started a session; the shared spine now fails
// the request with a fixed 400 before it acquires a lease.
const res = await request(app)
.post(loginPath(COMPANY_1))
.send({ environmentId: SANDBOX_ENV_1, unexpectedField: "x" });
expect(res.status, JSON.stringify(res.body)).toBe(400);
expect(harness.acquisitions).toHaveLength(0);
});
it("rejects a local environment", async () => {
mockEnvironmentService.getById.mockResolvedValueOnce({
id: SANDBOX_ENV_1,
@@ -522,7 +597,7 @@ describe("adapter device-login routes", () => {
expect(restart.body.sessionId).not.toBe(sessionId);
});
it("returns 409 for a second active start by a different owner", async () => {
it("lets a second owner start an active login in the same company", async () => {
const app = await createApp();
const first = await request(app)
@@ -531,13 +606,16 @@ describe("adapter device-login routes", () => {
expect(first.status, JSON.stringify(first.body)).toBe(201);
// A different owner starts a second login for the same company and adapter.
// The active slot is scoped to the company, the owner, and the adapter, so
// the second owner holds an independent slot and the start succeeds.
currentActor = boardActor(OWNER_B);
const second = await request(app)
.post(loginPath(COMPANY_1))
.send({ environmentId: SANDBOX_ENV_1 });
expect(second.status, JSON.stringify(second.body)).toBe(409);
// The second start never acquires a lease.
expect(harness.acquisitions).toHaveLength(1);
expect(second.status, JSON.stringify(second.body)).toBe(201);
expect(second.body.sessionId).not.toBe(first.body.sessionId);
// Each owner's start acquires its own lease.
expect(harness.acquisitions).toHaveLength(2);
});
it("returns 409 for a second active start in a different environment", async () => {
@@ -576,7 +654,10 @@ describe("adapter device-login routes", () => {
expect(status.body.failure?.reason).toBe("promotion_failed");
});
const row = (harness.store as ReturnType<typeof createMemoryStore>).rows.get(sessionId);
const row = await (harness.store as ReturnType<typeof createMemoryStore>).getByPublicId(
sessionId,
COMPANY_1,
);
expect(row?.status).toBe("failed");
expect(mockDeviceLoginPromotion).toHaveBeenCalledTimes(1);
});
@@ -602,7 +683,10 @@ describe("adapter device-login routes", () => {
expect(status.body.failure?.reason).toBe("promotion_failed");
});
const row = (harness.store as ReturnType<typeof createMemoryStore>).rows.get(sessionId);
const row = await (harness.store as ReturnType<typeof createMemoryStore>).getByPublicId(
sessionId,
COMPANY_1,
);
expect(row?.status).toBe("failed");
expect(mockDeviceLoginPromotion).toHaveBeenCalledTimes(1);
});
@@ -6,8 +6,8 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import {
activityLog,
adapterAuthSessions,
agents,
claudeSetupTokenSessions,
companies,
companySecretBindings,
companySecretProviderConfigs,
@@ -227,7 +227,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
await db.delete(companySecretProviderConfigs);
await db.delete(userSecretDeclarations);
await db.delete(userSecretDefinitions);
await db.delete(claudeSetupTokenSessions);
await db.delete(adapterAuthSessions);
await db.delete(agents);
await db.delete(environments);
await db.delete(companies);
@@ -303,8 +303,8 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
async function readClaim(sessionId: string) {
const rows = await db
.select()
.from(claudeSetupTokenSessions)
.where(eq(claudeSetupTokenSessions.sessionId, sessionId));
.from(adapterAuthSessions)
.where(eq(adapterAuthSessions.publicSessionId, sessionId));
return rows[0];
}
@@ -507,10 +507,10 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
});
const lockHeld = lockDb.transaction(async (tx) => {
await tx.execute(
sql`SELECT bound_at FROM claude_setup_token_sessions WHERE session_id = ${sessionId} FOR UPDATE`,
sql`SELECT bound_at FROM adapter_auth_sessions WHERE public_session_id = ${sessionId} FOR UPDATE`,
);
await tx.execute(
sql`UPDATE claude_setup_token_sessions SET deadline_at = clock_timestamp() + interval '300 milliseconds' WHERE session_id = ${sessionId}`,
sql`UPDATE adapter_auth_sessions SET expires_at = clock_timestamp() + interval '300 milliseconds' WHERE public_session_id = ${sessionId}`,
);
signalLocked();
await gate;
@@ -639,7 +639,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
expect(JSON.stringify(created)).not.toContain("sk-owner-token");
expect(await countAgents(scope.companyId)).toBe(1);
// The path consumes no setup-token claim, so it needs no login round trip.
const claims = await db.select().from(claudeSetupTokenSessions);
const claims = await db.select().from(adapterAuthSessions);
expect(claims).toHaveLength(0);
});
@@ -803,7 +803,6 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
return {
companyId: scope.companyId,
ownerUserId: scope.ownerUserId,
targetAgentId: null,
adapterType: "claude_local",
environmentId: scope.environmentId,
};
@@ -818,7 +817,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// The durable row transitioned to `stored` and stays unconsumed.
const claim = await readClaim(sessionId);
expect(claim?.state).toBe("stored");
expect(claim?.status).toBe("stored");
expect(claim?.boundAt).toBeNull();
// The owner value committed in the same transaction.
const status = await secretService(db).readClaudeOAuthUserSecretStatus(
@@ -848,7 +847,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
).rejects.toThrow();
const claim = await readClaim(sessionId);
expect(claim?.state).toBe("submitting");
expect(claim?.status).toBe("submitting");
expect(claim?.boundAt).toBeNull();
expect(
await secretService(db).readClaudeOAuthUserSecretStatus(scope.companyId, scope.ownerUserId),
@@ -865,7 +864,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
writer({ scope: claimScope(scope), sessionId, token: "sk-ant-oat01-EXPIRED" }),
).rejects.toThrow();
expect((await readClaim(sessionId))?.state).toBe("submitting");
expect((await readClaim(sessionId))?.status).toBe("submitting");
expect(
await secretService(db).readClaudeOAuthUserSecretStatus(scope.companyId, scope.ownerUserId),
).toBeNull();
@@ -918,7 +917,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
});
// The claim transitioned and the stored value rotated to a new version.
expect((await readClaim(sessionId))?.state).toBe("stored");
expect((await readClaim(sessionId))?.status).toBe("stored");
const after = await secretService(db).readClaudeOAuthUserSecretStatus(
scope.companyId,
scope.ownerUserId,
@@ -86,7 +86,7 @@ function createMemoryReaperStore(): AdapterAuthReaperStore & {
const row = rows.get(sessionId);
return row ? { ...row } : null;
},
async withCompanyAdapterPromotionLock(_companyId, _adapterType, fn) {
async withCompanyAdapterPromotionLock(_companyId, _startedByUserId, _adapterType, fn) {
// The in-memory store runs on a single event loop, so it needs no real
// lock. The pass-through keeps the reaper contract satisfied.
return fn();
@@ -66,12 +66,15 @@ const OWNER_B = "user-b";
// the status through a serialized tail, so a test waits for the write to land
// before it drives the next step.
async function waitForStatus(
store: { get(sessionId: string): Promise<AdapterAuthSessionRow | null> },
sessionId: string,
store: {
getByPublicId(publicSessionId: string, companyId: string): Promise<AdapterAuthSessionRow | null>;
},
publicSessionId: string,
companyId: string,
status: AdapterAuthSessionRow["status"],
): Promise<void> {
for (let attempt = 0; attempt < 1000; attempt += 1) {
const row = await store.get(sessionId);
const row = await store.getByPublicId(publicSessionId, companyId);
if (row?.status === status) return;
await new Promise((resolve) => setImmediate(resolve));
}
@@ -153,17 +156,21 @@ function createMemoryStore(): AdapterAuthSessionStore & {
} {
const rows = new Map<string, AdapterAuthSessionRow>();
const activeSlots = new Set<string>();
const slotKey = (companyId: string, adapterType: string) => `${companyId}|${adapterType}`;
// The active company credential slot is scoped to the company, the owner, and
// the adapter, so two owners in one company hold independent slots.
const slotKey = (companyId: string, startedByUserId: string, adapterType: string) =>
`${companyId}|${startedByUserId}|${adapterType}`;
const isActive = (status: AdapterAuthSessionRow["status"]) =>
status === "starting" || status === "waiting_for_user" || status === "promoting";
return {
rows,
async insert(input) {
const key = slotKey(input.companyId, input.adapterType);
const key = slotKey(input.companyId, input.startedByUserId, input.adapterType);
if (activeSlots.has(key)) throw new AdapterAuthSessionConflictError();
activeSlots.add(key);
rows.set(input.id, {
id: input.id,
publicSessionId: input.publicSessionId,
companyId: input.companyId,
environmentId: input.environmentId,
adapterType: input.adapterType,
@@ -187,7 +194,8 @@ function createMemoryStore(): AdapterAuthSessionStore & {
if (input.failureReason !== undefined) row.failureReason = input.failureReason;
if (input.finishedAt !== undefined) row.finishedAt = input.finishedAt;
if (input.promotionExpiresAt !== undefined) row.promotionExpiresAt = input.promotionExpiresAt;
if (!isActive(input.status)) activeSlots.delete(slotKey(row.companyId, row.adapterType));
if (!isActive(input.status))
activeSlots.delete(slotKey(row.companyId, row.startedByUserId, row.adapterType));
},
async compareAndSetStatus(input) {
const row = rows.get(input.sessionId);
@@ -196,14 +204,25 @@ function createMemoryStore(): AdapterAuthSessionStore & {
if (input.failureReason !== undefined) row.failureReason = input.failureReason;
if (input.finishedAt !== undefined) row.finishedAt = input.finishedAt;
if (input.promotionExpiresAt !== undefined) row.promotionExpiresAt = input.promotionExpiresAt;
if (!isActive(input.status)) activeSlots.delete(slotKey(row.companyId, row.adapterType));
if (!isActive(input.status))
activeSlots.delete(slotKey(row.companyId, row.startedByUserId, row.adapterType));
return true;
},
async get(sessionId) {
const row = rows.get(sessionId);
return row ? { ...row } : null;
},
async withCompanyAdapterPromotionLock(_companyId, _adapterType, fn) {
async getByPublicId(publicSessionId, companyId) {
// Scope the read to the company and the public session id, so a
// foreign-company lookup reads nothing and the internal id never matches.
for (const row of rows.values()) {
if (row.publicSessionId === publicSessionId && row.companyId === companyId) {
return { ...row };
}
}
return null;
},
async withCompanyAdapterPromotionLock(_companyId, _startedByUserId, _adapterType, fn) {
// The in-memory store runs on a single event loop, so it needs no real
// lock. The pass-through keeps the store contract satisfied.
return fn();
@@ -275,15 +294,17 @@ describe("codex device login service", () => {
// The prompt is surfaced only after the conditional move to
// `waiting_for_user` wins, so wait for that surface, then drain the microtask
// that retains the prompt before the owner reads it.
await waitForStatus(store, session.sessionId, "waiting_for_user");
await waitForStatus(store, session.sessionId, companyId, "waiting_for_user");
await new Promise((resolve) => setImmediate(resolve));
// The owner reads the one-time prompt through the owner read path.
const owner = await service.readOwnerSession(session.sessionId, OWNER_A);
const owner = await service.readOwnerSession(session.sessionId, companyId, OWNER_A);
expect(owner?.prompt).toEqual({ url: DEVICE_LOGIN_URL, code: PROMPT_CODE });
// The read returns the public session id, never the internal row id.
expect(owner?.sessionId).toBe(session.sessionId);
// A non-owner never reads the prompt.
const other = await service.readOwnerSession(session.sessionId, OWNER_B);
const other = await service.readOwnerSession(session.sessionId, companyId, OWNER_B);
expect(other?.prompt).toBeNull();
const outcome = await completed;
@@ -292,15 +313,18 @@ describe("codex device login service", () => {
expect(outcome.sandboxDeleteObserved).toBe(true);
expect(deleteCalls).toHaveLength(1);
expect(promoted).toHaveLength(1);
// The promotion runs with the session and company context, so it resolves the
// company scope and the sole-active-owner check for this exact session.
expect(promotionContexts).toEqual([{ sessionId: session.sessionId, companyId }]);
// The promotion runs with the internal session id and the company context, so
// it resolves the company scope and the sole-active-owner check for this exact
// session. The internal id is not the public session id.
const internalRow = await store.getByPublicId(session.sessionId, companyId);
expect(promotionContexts).toEqual([{ sessionId: internalRow?.id, companyId }]);
expect(internalRow?.id).not.toBe(session.sessionId);
// The prompt is one-time: a second owner read returns null.
const secondRead = await service.readOwnerSession(session.sessionId, OWNER_A);
const secondRead = await service.readOwnerSession(session.sessionId, companyId, OWNER_A);
expect(secondRead?.prompt).toBeNull();
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("authenticated");
expect(row?.finishedAt).not.toBeNull();
@@ -338,8 +362,9 @@ describe("codex device login service", () => {
},
},
});
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -350,9 +375,9 @@ describe("codex device login service", () => {
expect(outcome.status).toBe("failed");
expect(promoteCalls).toBe(0);
expect(deleteCalls).toHaveLength(1);
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("failed");
const failed = await service.readOwnerSession(session.sessionId, OWNER_A);
const failed = await service.readOwnerSession(session.sessionId, companyId, OWNER_A);
expect(failed?.failure?.reason).toBe("promotion_failed");
});
@@ -360,8 +385,9 @@ describe("codex device login service", () => {
const store = createMemoryStore();
const { runtime, deleteCalls } = createFakeRuntime({ exec: execFailure });
const service = makeService({ store, runtime });
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -370,7 +396,7 @@ describe("codex device login service", () => {
expect(outcome.status).toBe("failed");
expect(outcome.sandboxDeleteObserved).toBe(true);
expect(deleteCalls).toHaveLength(1);
const failed = await service.readOwnerSession(session.sessionId, OWNER_A);
const failed = await service.readOwnerSession(session.sessionId, companyId, OWNER_A);
expect(failed?.status).toBe("failed");
expect(failed?.failure?.reason).toBe("login_command_failed");
});
@@ -395,8 +421,9 @@ describe("codex device login service", () => {
const { runtime, deleteCalls } = createFakeRuntime({ exec: execHang });
const service = makeService({ store, runtime });
const controller = new AbortController();
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -407,7 +434,7 @@ describe("codex device login service", () => {
expect(outcome.status).toBe("cancelled");
expect(outcome.sandboxDeleteObserved).toBe(true);
expect(deleteCalls).toHaveLength(1);
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("cancelled");
});
@@ -462,8 +489,9 @@ describe("codex device login service", () => {
});
const service = makeService({ store, runtime });
const controller = new AbortController();
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -475,10 +503,10 @@ describe("codex device login service", () => {
expect(outcome.cleanupPending).toBe(true);
expect(outcome.sandboxDeleteObserved).toBe(true);
expect(deleteCalls).toHaveLength(1);
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("cleanup_pending");
// The public read resolves the retained terminal, never a false-clean.
const publicRead = await service.readOwnerSession(session.sessionId, OWNER_A);
const publicRead = await service.readOwnerSession(session.sessionId, companyId, OWNER_A);
expect(publicRead?.status).toBe(terminal);
},
);
@@ -499,8 +527,9 @@ describe("codex device login service", () => {
},
},
});
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -512,9 +541,9 @@ describe("codex device login service", () => {
expect(outcome.cleanupPending).toBe(true);
expect(outcome.sandboxDeleteObserved).toBe(true);
expect(deleteCalls).toHaveLength(1);
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("cleanup_pending");
const publicRead = await service.readOwnerSession(session.sessionId, OWNER_A);
const publicRead = await service.readOwnerSession(session.sessionId, companyId, OWNER_A);
expect(publicRead?.status).toBe("failed");
expect(publicRead?.failure?.reason).toBe("promotion_failed");
});
@@ -527,8 +556,9 @@ describe("codex device login service", () => {
delete: async () => ({ outcome: "not_found" }),
});
const service = makeService({ store, runtime });
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -537,7 +567,7 @@ describe("codex device login service", () => {
expect(outcome.status).toBe("authenticated");
expect(outcome.cleanupPending).toBe(false);
expect(outcome.sandboxDeleteObserved).toBe(true);
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("authenticated");
});
@@ -560,19 +590,22 @@ describe("codex device login service", () => {
});
const promote = vi.fn(() => {});
const service = makeService({ store, runtime, promotion: { promote } });
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
});
// The prompt moves the row to the active `waiting_for_user` state.
await waitForStatus(store, session.sessionId, "waiting_for_user");
await waitForStatus(store, session.sessionId, companyId, "waiting_for_user");
// The reaper wins the slot: it terminates the row before the promotion claim.
// The status write keys on the internal id, not the public session id.
const seededRow = await store.getByPublicId(session.sessionId, companyId);
await store.setStatus({
sessionId: session.sessionId,
sessionId: seededRow!.id,
status: "timed_out",
at: new Date(),
finishedAt: new Date(),
@@ -586,7 +619,7 @@ describe("codex device login service", () => {
// The lost claim writes no credential and never authenticates.
expect(promote).not.toHaveBeenCalled();
expect(outcome.status).not.toBe("authenticated");
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("timed_out");
// The service still deletes its own sandbox on the lost-claim path.
expect(deleteCalls).toHaveLength(1);
@@ -616,18 +649,18 @@ describe("codex device login service", () => {
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
});
await waitForStatus(store, session.sessionId, "waiting_for_user");
await waitForStatus(store, session.sessionId, companyId, "waiting_for_user");
// A non-owner cannot cancel the session.
expect(await service.cancelOwnerSession(session.sessionId, OWNER_B)).toBeNull();
expect(await service.cancelOwnerSession(session.sessionId, companyId, OWNER_B)).toBeNull();
// The owner cancel resolves the public terminal status at once.
const cancelled = await service.cancelOwnerSession(session.sessionId, OWNER_A);
const cancelled = await service.cancelOwnerSession(session.sessionId, companyId, OWNER_A);
expect(cancelled?.status).toBe("cancelled");
// The row holds the internal cleanup_pending state that encodes the
// cancelled terminal, so the reaper deletes the sandbox and finalizes it.
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("cleanup_pending");
expect(row?.finishedAt).not.toBeNull();
@@ -662,19 +695,20 @@ describe("codex device login service", () => {
});
const { runtime } = createFakeRuntime({ exec: execSuccess, authBytes: Buffer.from("{}") });
const service = makeService({ store, runtime, promotion: { promote } });
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
});
await waitForStatus(store, session.sessionId, "promoting");
await waitForStatus(store, session.sessionId, companyId, "promoting");
// The cancel skips the promoting row, so the credential write finishes. The
// public projection of a promoting row is `waiting_for_user`.
const result = await service.cancelOwnerSession(session.sessionId, OWNER_A);
const result = await service.cancelOwnerSession(session.sessionId, companyId, OWNER_A);
expect(result?.status).toBe("waiting_for_user");
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("promoting");
// Release the promotion, so the run reaches its own terminal.
@@ -691,8 +725,9 @@ describe("codex device login service", () => {
const store = createMemoryStore();
const { runtime, deleteCalls } = createFakeRuntime({ exec: execHang });
const service = makeService({ store, runtime });
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -706,7 +741,7 @@ describe("codex device login service", () => {
// claim.
await vi.advanceTimersByTimeAsync(CODEX_DEVICE_LOGIN_TIMEOUT_MS - 1);
expect(settled).toBe(false);
const midRow = await store.get(session.sessionId);
const midRow = await store.getByPublicId(session.sessionId, companyId);
expect(["starting", "waiting_for_user"]).toContain(midRow?.status);
// The last millisecond fires the timeout.
@@ -715,7 +750,7 @@ describe("codex device login service", () => {
expect(outcome.status).toBe("timed_out");
expect(outcome.sandboxDeleteObserved).toBe(true);
expect(deleteCalls).toHaveLength(1);
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("timed_out");
} finally {
vi.useRealTimers();
@@ -733,8 +768,9 @@ describe("codex device login service", () => {
},
});
const service = makeService({ store, runtime });
const companyId = randomUUID();
const { session, completed } = await service.start({
companyId: randomUUID(),
companyId,
environmentId: randomUUID(),
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
@@ -745,7 +781,7 @@ describe("codex device login service", () => {
expect(outcome.cleanupPending).toBe(true);
expect(outcome.sandboxDeleteObserved).toBe(true);
expect(deleteCalls).toHaveLength(1);
const row = await store.get(session.sessionId);
const row = await store.getByPublicId(session.sessionId, companyId);
expect(row?.status).toBe("cleanup_pending");
} finally {
vi.useRealTimers();
@@ -887,73 +923,112 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
return environmentId;
}
it.each([
{ name: "two owners, same environment" },
{ name: "one owner, two environments" },
])(
"returns one session, one lease, and one 409 for concurrent starts ($name)",
async ({ name }) => {
const { companyId, environmentId: environmentA } = await seedCompanyEnvironment();
const twoOwners = name.startsWith("two owners");
const environmentB = twoOwners ? environmentA : await seedEnvironment(companyId);
const ownerB = twoOwners ? OWNER_B : OWNER_A;
it("lets two owners in one company each hold an active session", async () => {
// The active slot is scoped to the company, the owner, and the adapter. Two
// owners in one company hold independent slots, so both concurrent starts
// succeed and each acquires one lease.
const { companyId, environmentId } = await seedCompanyEnvironment();
const store = createDbAdapterAuthSessionStore(db);
const { runtime, acquisitions } = createFakeRuntime({ exec: execHang });
const service = makeService({ store, runtime });
const controller = new AbortController();
const store = createDbAdapterAuthSessionStore(db);
const { runtime, acquisitions } = createFakeRuntime({ exec: execHang });
const service = makeService({ store, runtime });
const controller = new AbortController();
const results = await Promise.allSettled([
service.start({
companyId,
environmentId,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
signal: controller.signal,
}),
service.start({
companyId,
environmentId,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_B,
signal: controller.signal,
}),
]);
const results = await Promise.allSettled([
service.start({
companyId,
environmentId: environmentA,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
signal: controller.signal,
}),
service.start({
companyId,
environmentId: environmentB,
adapterType: ADAPTER_TYPE,
startedByUserId: ownerB,
signal: controller.signal,
}),
]);
const fulfilled = results.filter(
(result): result is PromiseFulfilledResult<Awaited<ReturnType<typeof service.start>>> =>
result.status === "fulfilled",
);
expect(fulfilled).toHaveLength(2);
expect(acquisitions).toHaveLength(2);
const rows = await db
.select()
.from(adapterAuthSessions)
.where(eq(adapterAuthSessions.companyId, companyId));
expect(rows).toHaveLength(2);
const fulfilled = results.filter(
(result): result is PromiseFulfilledResult<Awaited<ReturnType<typeof service.start>>> =>
result.status === "fulfilled",
);
const rejected = results.filter(
(result): result is PromiseRejectedResult => result.status === "rejected",
);
expect(fulfilled).toHaveLength(1);
expect(rejected).toHaveLength(1);
const conflict = rejected[0]!.reason;
expect(conflict).toBeInstanceOf(AdapterAuthSessionConflictError);
expect(conflict.statusCode).toBe(409);
// Release the surviving runs so the hanging login commands end.
controller.abort();
await Promise.all(fulfilled.map((result) => result.value.completed));
});
// Exactly one start acquired a lease; the losing start never acquired.
expect(acquisitions).toHaveLength(1);
const rows = await db
.select()
.from(adapterAuthSessions)
.where(eq(adapterAuthSessions.companyId, companyId));
expect(rows).toHaveLength(1);
it("returns one session, one lease, and one 409 when one owner starts twice", async () => {
// One owner starting twice in two environments conflicts on the active slot.
// The slot dropped the environment term, so the second environment does not
// grant a second active session.
const { companyId, environmentId: environmentA } = await seedCompanyEnvironment();
const environmentB = await seedEnvironment(companyId);
// Release the surviving run so the hanging login command ends.
controller.abort();
await fulfilled[0]!.value.completed;
},
);
const store = createDbAdapterAuthSessionStore(db);
const { runtime, acquisitions } = createFakeRuntime({ exec: execHang });
const service = makeService({ store, runtime });
const controller = new AbortController();
it("rejects a second start with a 409 while the first login holds the promotion claim", async () => {
const results = await Promise.allSettled([
service.start({
companyId,
environmentId: environmentA,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
signal: controller.signal,
}),
service.start({
companyId,
environmentId: environmentB,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
signal: controller.signal,
}),
]);
const fulfilled = results.filter(
(result): result is PromiseFulfilledResult<Awaited<ReturnType<typeof service.start>>> =>
result.status === "fulfilled",
);
const rejected = results.filter(
(result): result is PromiseRejectedResult => result.status === "rejected",
);
expect(fulfilled).toHaveLength(1);
expect(rejected).toHaveLength(1);
const conflict = rejected[0]!.reason;
expect(conflict).toBeInstanceOf(AdapterAuthSessionConflictError);
expect(conflict.statusCode).toBe(409);
// Exactly one start acquired a lease; the losing start never acquired.
expect(acquisitions).toHaveLength(1);
const rows = await db
.select()
.from(adapterAuthSessions)
.where(eq(adapterAuthSessions.companyId, companyId));
expect(rows).toHaveLength(1);
// Release the surviving run so the hanging login command ends.
controller.abort();
await fulfilled[0]!.value.completed;
});
it("rejects the same owner's second start with a 409 while the first holds the promotion claim", async () => {
const { companyId, environmentId } = await seedCompanyEnvironment();
const store = createDbAdapterAuthSessionStore(db);
const { runtime } = createFakeRuntime({ exec: execSuccess, authBytes: Buffer.from("{}") });
// A promotion that never resolves. The first login stays in `promoting`, so
// it holds the active company slot through the claim window.
// it holds the active slot through the claim window.
let releasePromotion!: () => void;
const promotionGate = new Promise<void>((resolve) => {
releasePromotion = resolve;
@@ -972,16 +1047,16 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
});
// The first login reaches the `promoting` claim and holds it.
await waitForStatus(store, first.session.sessionId, "promoting");
await waitForStatus(store, first.session.sessionId, companyId, "promoting");
// A second start for the same company and adapter conflicts on the active
// slot, even though the first login is mid-promotion.
// A second start for the same company, owner, and adapter conflicts on the
// active slot, even though the first login is mid-promotion.
await expect(
service.start({
companyId,
environmentId,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_B,
startedByUserId: OWNER_A,
}),
).rejects.toBeInstanceOf(AdapterAuthSessionConflictError);
@@ -1003,6 +1078,7 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
environmentId,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
publicSessionId: randomUUID(),
providerLeaseId: `lease-${sessionId}`,
status: "promoting",
expiresAt: past,
@@ -1033,7 +1109,7 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
const sectionActive = new Promise<void>((resolve) => {
markSectionActive = resolve;
});
const promotion = store.withCompanyAdapterPromotionLock(companyId, ADAPTER_TYPE, async () => {
const promotion = store.withCompanyAdapterPromotionLock(companyId, OWNER_A, ADAPTER_TYPE, async () => {
markSectionActive();
await sectionGate;
// Decision H: the write proceeds only while the session still owns the slot.
@@ -1078,7 +1154,7 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
// A delayed promotion now runs its section. The ownership check reads the
// reclaimed row, so it writes no credential and the terminal claim fails.
let wroteCredential = false;
await store.withCompanyAdapterPromotionLock(companyId, ADAPTER_TYPE, async () => {
await store.withCompanyAdapterPromotionLock(companyId, OWNER_A, ADAPTER_TYPE, async () => {
const row = await store.get(sessionId);
if (row?.status === "promoting" && row.companyId === companyId) {
wroteCredential = true;
@@ -1122,8 +1198,8 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
startedByUserId: OWNER_A,
signal: controller.signal,
});
const sessionId = started.session.sessionId;
await waitForStatus(store, sessionId, "starting");
const publicSessionId = started.session.sessionId;
await waitForStatus(store, publicSessionId, companyId, "starting");
// The reaper reclaims the expired row and times it out.
const { runtime: reaperRuntime } = createReaperRuntime();
@@ -1134,7 +1210,7 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
});
const sweep = await reaper.sweep();
expect(sweep.expiredTimedOut).toBe(1);
expect((await store.get(sessionId))?.status).toBe("timed_out");
expect((await store.getByPublicId(publicSessionId, companyId))?.status).toBe("timed_out");
// The delayed prompt arrives now. The conditional transition must not revive
// the reaped row, and the owner must not read a prompt for it.
@@ -1142,12 +1218,94 @@ describeEmbeddedPostgres("codex device login service concurrency (embedded postg
await new Promise((resolve) => setImmediate(resolve));
await new Promise((resolve) => setImmediate(resolve));
const owner = await service.readOwnerSession(sessionId, OWNER_A);
expect((await store.get(sessionId))?.status).toBe("timed_out");
const owner = await service.readOwnerSession(publicSessionId, companyId, OWNER_A);
expect((await store.getByPublicId(publicSessionId, companyId))?.status).toBe("timed_out");
expect(owner?.prompt ?? null).toBeNull();
// End the run so no timer survives.
controller.abort();
await started.completed;
});
describe("public session id lookup", () => {
// Start one session and read its rows. The service returns the public session
// id, and the store keeps the internal id private.
async function startOneSession(companyId: string, environmentId: string) {
const store = createDbAdapterAuthSessionStore(db);
const controller = new AbortController();
const { runtime } = createFakeRuntime({ exec: execHang });
const service = makeService({ store, runtime });
const started = await service.start({
companyId,
environmentId,
adapterType: ADAPTER_TYPE,
startedByUserId: OWNER_A,
signal: controller.signal,
});
const publicSessionId = started.session.sessionId;
const row = await store.getByPublicId(publicSessionId, companyId);
return { store, service, controller, started, publicSessionId, row };
}
it("reads a session by its public id in the same company", async () => {
const { companyId, environmentId } = await seedCompanyEnvironment();
const { store, controller, started, publicSessionId, row } = await startOneSession(
companyId,
environmentId,
);
// The lookup by the public id succeeds in the same company.
expect(row).not.toBeNull();
expect(row?.publicSessionId).toBe(publicSessionId);
// The internal primary-key id is a distinct value, never the public id.
expect(row?.id).toBeTruthy();
expect(row?.id).not.toBe(publicSessionId);
// The API-facing row read stays scoped: the same public id resolves the
// same row on a second read.
const again = await store.getByPublicId(publicSessionId, companyId);
expect(again?.id).toBe(row?.id);
controller.abort();
await started.completed;
});
it("does not read a session for a foreign company", async () => {
const { companyId, environmentId } = await seedCompanyEnvironment();
const { store, controller, started, publicSessionId } = await startOneSession(
companyId,
environmentId,
);
// A different company id never resolves the row, even with the exact public
// session id. The predicate carries the company id, so a query never keys on
// the public session id alone.
const foreignCompanyId = randomUUID();
expect(await store.getByPublicId(publicSessionId, foreignCompanyId)).toBeNull();
controller.abort();
await started.completed;
});
it("never accepts the internal row id as an API identifier", async () => {
const { companyId, environmentId } = await seedCompanyEnvironment();
const { store, service, controller, started, publicSessionId, row } =
await startOneSession(companyId, environmentId);
const internalId = row!.id;
// The store never resolves a row by the internal id through the public-id
// path, even within the same company.
expect(await store.getByPublicId(internalId, companyId)).toBeNull();
// The owner read returns the public session id, never the internal id.
const owner = await service.readOwnerSession(publicSessionId, companyId, OWNER_A);
expect(owner?.sessionId).toBe(publicSessionId);
expect(owner?.sessionId).not.toBe(internalId);
// The owner read rejects the internal id as an API identifier.
expect(await service.readOwnerSession(internalId, companyId, OWNER_A)).toBeNull();
controller.abort();
await started.completed;
});
});
});
@@ -36,6 +36,36 @@ describe("dev-runner worktree env bootstrap", () => {
expect(isWorktreeSeedPending(root)).toBe(false);
});
it("guards every manifest state except a complete verified manifest", () => {
const root = createTempRoot("paperclip-dev-runner-seed-manifest-");
const manifestPath = path.join(root, ".paperclip", "seed-manifest.json");
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "failed" }), "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "verified" }), "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
fs.writeFileSync(manifestPath, JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: "/source/config.json" },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "minimal",
migrationRevision: "0001",
targetInstanceId: "target",
phase: "complete",
state: "verified",
attemptId: "attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}), "utf8");
expect(isWorktreeSeedPending(root)).toBe(false);
fs.writeFileSync(manifestPath, "not-json", "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
});
it("detects linked git worktrees from .git files", () => {
const root = createTempRoot("paperclip-dev-runner-worktree-");
fs.writeFileSync(path.join(root, ".git"), "gitdir: /tmp/paperclip/.git/worktrees/feature\n", "utf8");
@@ -0,0 +1,61 @@
import { EventEmitter } from "node:events";
import { describe, expect, it, vi } from "vitest";
import { createEmbeddedPostgresSupervisor, type SupervisedEmbeddedPostgres } from "../embedded-postgres-supervisor.js";
function createInstance(startError?: Error) {
const process = new EventEmitter();
const instance: SupervisedEmbeddedPostgres = {
process,
start: vi.fn(async () => { if (startError) throw startError; }),
stop: vi.fn(async () => undefined),
};
return { instance, process };
}
describe("embedded PostgreSQL supervisor", () => {
it("restarts PostgreSQL after its managed child exits unexpectedly", async () => {
const initial = createInstance();
const replacement = createInstance();
const onRestarted = vi.fn();
const supervisor = createEmbeddedPostgresSupervisor({
initialInstance: initial.instance,
createInstance: () => replacement.instance,
restartDelaysMs: [0],
onRestarted,
});
initial.process.emit("exit", 137, "SIGKILL");
await supervisor.waitForRecovery();
expect(replacement.instance.start).toHaveBeenCalledOnce();
expect(supervisor.current()).toBe(replacement.instance);
expect(onRestarted).toHaveBeenCalledWith(1);
});
it("does not restart PostgreSQL during orderly shutdown", async () => {
const initial = createInstance();
const createReplacement = vi.fn(() => createInstance().instance);
const supervisor = createEmbeddedPostgresSupervisor({
initialInstance: initial.instance,
createInstance: createReplacement,
restartDelaysMs: [0],
});
await supervisor.shutdown();
expect(initial.instance.stop).toHaveBeenCalledOnce();
expect(createReplacement).not.toHaveBeenCalled();
});
it("bounds recovery attempts and reports the final failure", async () => {
const initial = createInstance();
const failures = [new Error("first"), new Error("second"), new Error("third")];
const onRecoveryExhausted = vi.fn();
const supervisor = createEmbeddedPostgresSupervisor({
initialInstance: initial.instance,
createInstance: () => createInstance(failures.shift()).instance,
restartDelaysMs: [0, 0, 0],
onRecoveryExhausted,
});
initial.process.emit("exit", 1, null);
await supervisor.waitForRecovery();
expect(onRecoveryExhausted).toHaveBeenCalledOnce();
expect(onRecoveryExhausted).toHaveBeenCalledWith(expect.objectContaining({ message: "third" }));
});
});
@@ -2,6 +2,7 @@ import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { environmentRoutes } from "../routes/environments.js";
import { HttpError } from "../errors.js";
import {
environmentCustomImageTerminalConnectionRegistry,
environmentCustomImageTerminalSessionStore,
@@ -40,6 +41,7 @@ const mockEnvironmentCustomImageService = vi.hoisted(() => ({
finishSetupSession: vi.fn(),
cancelSetupSession: vi.fn(),
rollbackTemplate: vi.fn(),
relinkActiveTemplate: vi.fn(),
disableTemplate: vi.fn(),
cleanupExpiredSetupSessions: vi.fn(),
}));
@@ -292,6 +294,10 @@ describe("environment customImage setup routes", () => {
templateRef: "disabled-template-secret",
status: "revoked",
}));
mockEnvironmentCustomImageService.relinkActiveTemplate.mockResolvedValue({
template: createTemplate({ id: "template-1", templateRef: "relinked-template-secret" }),
classification: "knob_only",
});
});
it("starts a setup session, returns the live payload, and logs redacted details", async () => {
@@ -641,4 +647,78 @@ describe("environment customImage setup routes", () => {
expect(activity).not.toContain("old-template-secret");
expect(activity).not.toContain("disabled-template-secret");
});
it("relinks the active template through the company-scoped route", async () => {
const res = await request(createApp(boardActor()))
.post("/api/environments/env-1/custom-image-template/relink?companyId=company-1")
.send({ confirmBootSourceDrift: true });
expect(res.status).toBe(200);
expect(res.body.classification).toBe("knob_only");
expect(mockEnvironmentCustomImageService.relinkActiveTemplate).toHaveBeenCalledWith({
environmentId: "env-1",
confirmBootSourceDrift: true,
actor: {
actorType: "user",
actorId: "user-1",
agentId: null,
runId: null,
agentApiKeyId: null,
},
companyId: "company-1",
});
});
it("defaults the confirmation flag to false and rejects unknown body keys", async () => {
const app = createApp(boardActor());
const withoutFlag = await request(app)
.post("/api/environments/env-1/custom-image-template/relink?companyId=company-1")
.send({});
expect(withoutFlag.status).toBe(200);
expect(mockEnvironmentCustomImageService.relinkActiveTemplate).toHaveBeenCalledExactlyOnceWith(
expect.objectContaining({ confirmBootSourceDrift: false }),
);
// The strict schema rejects unknown keys before the handler runs, so the
// relink service is never reached for the malformed body.
const unknownKey = await request(app)
.post("/api/environments/env-1/custom-image-template/relink?companyId=company-1")
.send({ confirmBootSourceDrift: false, unexpected: true });
expect(unknownKey.status).not.toBe(200);
expect(mockEnvironmentCustomImageService.relinkActiveTemplate).toHaveBeenCalledTimes(1);
});
it("propagates the drift conflict from the relink service", async () => {
mockEnvironmentCustomImageService.relinkActiveTemplate.mockImplementationOnce(async () => {
throw new HttpError(409, "Confirm the relink to keep the captured snapshot.", {
classification: "boot_source_drift",
driftedPaths: [{ path: "image", from: "fake:base", to: "fake:other" }],
});
});
const res = await request(createApp(boardActor()))
.post("/api/environments/env-1/custom-image-template/relink?companyId=company-1")
.send({});
expect(res.status).toBe(409);
});
it("denies agent API key actors before the relink service is called", async () => {
const res = await request(createApp(agentActor()))
.post("/api/environments/env-1/custom-image-template/relink?companyId=company-1")
.send({});
expect(res.status).toBe(403);
expect(mockEnvironmentCustomImageService.relinkActiveTemplate).not.toHaveBeenCalled();
});
it("denies non-admin board users before the relink service is called", async () => {
const res = await request(createApp(boardActor({
companyIds: ["company-2"],
isInstanceAdmin: false,
})))
.post("/api/environments/env-1/custom-image-template/relink?companyId=company-1")
.send({});
expect(res.status).toBe(403);
expect(mockEnvironmentCustomImageService.relinkActiveTemplate).not.toHaveBeenCalled();
});
});
@@ -1,7 +1,8 @@
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { desc, eq } from "drizzle-orm";
import {
activityLog,
companies,
createDb,
environmentCustomImageSetupSessions,
@@ -866,3 +867,501 @@ describeEmbeddedPostgres("environmentCustomImageService reconciliation", () => {
expect(outOfSync.activeTemplateMatchesConfig).toBe(false);
});
});
function secretPluginManifest() {
return {
id: "paperclip.fake-secret-sandbox-provider",
apiVersion: 1,
version: "0.1.0",
displayName: "Fake Secret Sandbox Provider",
categories: ["automation"],
capabilities: ["environment.drivers.register"],
entrypoints: { worker: "./dist/worker.js" },
environmentDrivers: [
{
driverKey: "fake-secret-plugin",
kind: "sandbox_provider",
displayName: "Fake Secret Sandbox Provider",
supportsInteractiveSetup: true,
interactiveSetupConnectionTypes: ["ssh"],
supportsTemplateCapture: true,
templateRefKind: "snapshot",
templateConfigBinding: { field: "customTemplate", unsetFields: ["image"] },
// apiUrl overlaps a secret exactly; auth.token is a child of secret
// `auth`; credentials is a parent of secret `credentials.secret`.
templateIdentityPaths: ["apiUrl", "auth.token", "credentials"],
supportsTemplateDelete: true,
configSchema: {
type: "object",
properties: {
apiUrl: { type: "string", format: "secret-ref" },
auth: { type: "string", format: "secret-ref" },
credentials: {
type: "object",
properties: { secret: { type: "string", format: "secret-ref" } },
},
},
},
},
],
} as const;
}
function invalidIdentityPluginManifest() {
return {
id: "paperclip.fake-invalid-sandbox-provider",
apiVersion: 1,
version: "0.1.0",
displayName: "Fake Invalid Identity Sandbox Provider",
categories: ["automation"],
capabilities: ["environment.drivers.register"],
entrypoints: { worker: "./dist/worker.js" },
environmentDrivers: [
{
driverKey: "fake-invalid-plugin",
kind: "sandbox_provider",
displayName: "Fake Invalid Identity Sandbox Provider",
supportsInteractiveSetup: true,
interactiveSetupConnectionTypes: ["ssh"],
supportsTemplateCapture: true,
templateRefKind: "snapshot",
templateConfigBinding: { field: "customTemplate", unsetFields: ["image"] },
// A valid path plus two that must fail canonicalization.
templateIdentityPaths: ["apiUrl", "bad path!", "a..b"],
supportsTemplateDelete: true,
configSchema: { type: "object" },
},
],
} as const;
}
function prototypeKeyIdentityPluginManifest() {
return {
id: "paperclip.fake-prototype-key-sandbox-provider",
apiVersion: 1,
version: "0.1.0",
displayName: "Fake Prototype Key Sandbox Provider",
categories: ["automation"],
capabilities: ["environment.drivers.register"],
entrypoints: { worker: "./dist/worker.js" },
environmentDrivers: [
{
driverKey: "fake-prototype-key-plugin",
kind: "sandbox_provider",
displayName: "Fake Prototype Key Sandbox Provider",
supportsInteractiveSetup: true,
interactiveSetupConnectionTypes: ["ssh"],
supportsTemplateCapture: true,
templateRefKind: "snapshot",
templateConfigBinding: { field: "customTemplate", unsetFields: ["image"] },
// Every prototype key has an identifier shape but names no own field.
// Each must fail canonicalization and record only the marker.
templateIdentityPaths: ["__proto__", "constructor", "prototype", "nested.__proto__"],
supportsTemplateDelete: true,
configSchema: { type: "object" },
},
],
} as const;
}
describeEmbeddedPostgres("environmentCustomImageService relink", () => {
let stopDb: (() => Promise<void>) | null = null;
let db!: ReturnType<typeof createDb>;
beforeAll(async () => {
const started = await startEmbeddedPostgresTestDatabase("environment-custom-images-relink");
stopDb = started.stop;
db = createDb(started.connectionString);
});
afterEach(async () => {
await db.delete(activityLog);
await db.delete(environmentCustomImageSetupSessions);
await db.delete(environmentCustomImageTemplates);
await db.delete(plugins);
await db.delete(environments);
await db.delete(companies);
});
afterAll(async () => {
await stopDb?.();
});
async function seed(opts?: {
manifest?: ReturnType<typeof pluginManifest> | ReturnType<typeof secretPluginManifest> | ReturnType<typeof invalidIdentityPluginManifest> | ReturnType<typeof prototypeKeyIdentityPluginManifest>;
config?: Record<string, unknown>;
}) {
const manifest = opts?.manifest ?? pluginManifest();
const provider = manifest.environmentDrivers[0]!.driverKey;
const companyId = randomUUID();
const environmentId = randomUUID();
await db.insert(companies).values(
{ id: companyId, name: "Acme", issuePrefix: `A${companyId.slice(0, 4)}` },
);
await db.insert(environments).values({
id: environmentId,
name: `Fake ${environmentId.slice(0, 8)}`,
driver: "sandbox",
status: "active",
config: opts?.config ?? { provider, image: "fake:base", reuseLease: false },
envVars: {},
});
await db.insert(plugins).values({
pluginKey: manifest.id,
packageName: `paperclip-plugin-${provider}`,
version: "0.1.0",
apiVersion: 1,
categories: ["automation"],
manifestJson: manifest,
status: "ready",
});
return { companyId, environmentId, provider };
}
const relinkActor = {
actorType: "user" as const,
actorId: "user-1",
agentId: null,
runId: null,
agentApiKeyId: null,
};
async function activeTemplateRow(environmentId: string) {
return db
.select()
.from(environmentCustomImageTemplates)
.where(eq(environmentCustomImageTemplates.environmentId, environmentId))
.orderBy(desc(environmentCustomImageTemplates.createdAt))
.then((rows) => rows[0]!);
}
async function relinkActivityRows(environmentId: string) {
return db
.select()
.from(activityLog)
.where(eq(activityLog.entityId, environmentId));
}
it("keeps secret-ref values out of metadata, the response, and activity, and relinks with the flag", async () => {
const config = {
provider: "fake-secret-plugin",
image: "fake:base",
apiUrl: "https://secret-endpoint.example",
auth: "auth-secret-value",
credentials: { secret: "cred-secret-value", region: "eu" },
reuseLease: false,
};
const { companyId, environmentId } = await seed({ manifest: secretPluginManifest(), config });
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
const started = await service.startSetupSession({ environmentId, actor: { userId: "user-1" } });
const promoted = await service.finishSetupSession({ sessionId: started.session.id });
// The public template response never carries the server-internal snapshot.
expect((promoted.template.metadata as Record<string, unknown>).bootRelevantConfig).toBeUndefined();
// The persisted snapshot lives in the row. Every secret-ref overlap is
// excluded with no value (exact, child, parent).
const persistedRow = await activeTemplateRow(environmentId);
const boot = (persistedRow.metadata as Record<string, unknown>).bootRelevantConfig as {
values: Record<string, unknown>;
excludedPaths: string[];
};
expect(boot.excludedPaths).toEqual(expect.arrayContaining(["apiUrl", "auth.token", "credentials"]));
expect(Object.keys(boot.values)).not.toContain("apiUrl");
expect(Object.keys(boot.values)).not.toContain("auth.token");
expect(Object.keys(boot.values)).not.toContain("credentials");
// Neither the persisted snapshot nor the response leaks a secret value.
for (const metadataJson of [
JSON.stringify(persistedRow.metadata),
JSON.stringify(promoted.template.metadata),
]) {
expect(metadataJson).not.toContain("secret-endpoint.example");
expect(metadataJson).not.toContain("auth-secret-value");
expect(metadataJson).not.toContain("cred-secret-value");
}
// An excluded path forces the fail-closed unclassified result: the flag is required.
await expect(service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId }))
.rejects.toMatchObject({ status: 409 });
const relinked = await service.relinkActiveTemplate({
environmentId,
confirmBootSourceDrift: true,
actor: relinkActor,
companyId,
});
expect(relinked.classification).toBe("unclassified");
const responseJson = JSON.stringify(relinked);
expect(responseJson).not.toContain("secret-endpoint.example");
expect(responseJson).not.toContain("auth-secret-value");
expect(responseJson).not.toContain("cred-secret-value");
const activities = await relinkActivityRows(environmentId);
const relinkActivity = activities.find((row) => row.action === "environment.custom_image_template.relinked");
expect(relinkActivity).toBeDefined();
const activityJson = JSON.stringify(relinkActivity!.details);
expect(activityJson).not.toContain("secret-endpoint.example");
expect(activityJson).not.toContain("auth-secret-value");
expect(activityJson).not.toContain("cred-secret-value");
// Drift detail carries path names only, never a fingerprint value.
expect(activityJson).not.toContain(promoted.template.sourceEnvironmentConfigFingerprint);
});
it("fails closed for legacy templates without a boot-relevant snapshot", async () => {
const { companyId, environmentId } = await seed();
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
await db.insert(environmentCustomImageTemplates).values({
environmentId,
provider: "fake-plugin",
templateKind: "snapshot",
templateRef: "snapshot-legacy",
sourceEnvironmentConfigFingerprint: "stale-fingerprint",
status: "active",
metadata: { runtimeConfigBinding: { field: "customTemplate", unsetFields: ["image"] } },
});
await expect(service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId }))
.rejects.toMatchObject({ status: 409 });
const beforeRow = await activeTemplateRow(environmentId);
expect(beforeRow.sourceEnvironmentConfigFingerprint).toBe("stale-fingerprint");
const relinked = await service.relinkActiveTemplate({
environmentId,
confirmBootSourceDrift: true,
actor: relinkActor,
companyId,
});
expect(relinked.classification).toBe("unclassified");
const afterRow = await activeTemplateRow(environmentId);
expect(afterRow.sourceEnvironmentConfigFingerprint).not.toBe("stale-fingerprint");
});
it("relinks a knob-only change without a flag and keeps the fail-closed runtime gate", async () => {
const { companyId, environmentId } = await seed();
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
const started = await service.startSetupSession({ environmentId, actor: { userId: "user-1" } });
const promoted = await service.finishSetupSession({ sessionId: started.session.id });
// A non-boot field changes the fingerprint and detaches the template.
await db.update(environments)
.set({ config: { provider: "fake-plugin", image: "fake:base", reuseLease: false, region: "eu" } })
.where(eq(environments.id, environmentId));
const detached = await service.getOverview({ environmentId });
expect(detached.activeTemplateMatchesConfig).toBe(false);
const relinked = await service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId });
expect(relinked.classification).toBe("knob_only");
const afterRelink = await service.getOverview({ environmentId });
expect(afterRelink.activeTemplateMatchesConfig).toBe(true);
const resolved = await resolveEnvironmentDriverConfigForRuntime(db, companyId, {
id: environmentId,
driver: "sandbox",
config: { provider: "fake-plugin", image: "fake:base", reuseLease: false, region: "eu" },
}, { heartbeatRunId: randomUUID() });
expect(resolved.config).toMatchObject({ customTemplate: promoted.template.templateRef });
// The gate stays fail-closed: a base image change detaches again.
await db.update(environments)
.set({ config: { provider: "fake-plugin", image: "fake:other", reuseLease: false, region: "eu" } })
.where(eq(environments.id, environmentId));
const afterImageChange = await service.getOverview({ environmentId });
expect(afterImageChange.activeTemplateMatchesConfig).toBe(false);
});
it("requires the flag for boot-source drift and re-stamps only after confirmation", async () => {
const { companyId, environmentId } = await seed();
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
const started = await service.startSetupSession({ environmentId, actor: { userId: "user-1" } });
await service.finishSetupSession({ sessionId: started.session.id });
await db.update(environments)
.set({ config: { provider: "fake-plugin", image: "fake:other", reuseLease: false } })
.where(eq(environments.id, environmentId));
await expect(service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId }))
.rejects.toMatchObject({
status: 409,
details: {
classification: "boot_source_drift",
driftedPaths: expect.arrayContaining([
expect.objectContaining({ path: "image", from: "fake:base", to: "fake:other" }),
]),
},
});
const relinked = await service.relinkActiveTemplate({
environmentId,
confirmBootSourceDrift: true,
actor: relinkActor,
companyId,
});
expect(relinked.classification).toBe("boot_source_drift");
const afterRelink = await service.getOverview({ environmentId });
expect(afterRelink.activeTemplateMatchesConfig).toBe(true);
});
it("re-stamps only the active template and never a superseded one", async () => {
const { companyId, environmentId } = await seed();
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
const first = await service.finishSetupSession({
sessionId: (await service.startSetupSession({ environmentId, actor: { userId: "user-1" } })).session.id,
});
const second = await service.finishSetupSession({
sessionId: (await service.startSetupSession({ environmentId, actor: { userId: "user-1" } })).session.id,
});
// The first template is now superseded; the second is active.
const supersededBefore = await db
.select()
.from(environmentCustomImageTemplates)
.where(eq(environmentCustomImageTemplates.id, first.template.id))
.then((rows) => rows[0]!);
await db.update(environments)
.set({ config: { provider: "fake-plugin", image: "fake:base", reuseLease: false, region: "eu" } })
.where(eq(environments.id, environmentId));
const relinked = await service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId });
expect(relinked.template.id).toBe(second.template.id);
const supersededAfter = await db
.select()
.from(environmentCustomImageTemplates)
.where(eq(environmentCustomImageTemplates.id, first.template.id))
.then((rows) => rows[0]!);
expect(supersededAfter.status).toBe("superseded");
expect(supersededAfter.sourceEnvironmentConfigFingerprint)
.toBe(supersededBefore.sourceEnvironmentConfigFingerprint);
});
it("aborts with a conflict and writes no active template when none exists", async () => {
const { companyId, environmentId } = await seed();
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
await expect(service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId }))
.rejects.toMatchObject({ status: 404 });
const activities = await relinkActivityRows(environmentId);
expect(activities.filter((row) => row.action === "environment.custom_image_template.relinked")).toHaveLength(0);
});
it("classifies an invalid driver identity path as unclassified without treating it as value-bearing", async () => {
const { companyId, environmentId } = await seed({ manifest: invalidIdentityPluginManifest(), config: {
provider: "fake-invalid-plugin",
image: "fake:base",
reuseLease: false,
} });
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
const started = await service.startSetupSession({ environmentId, actor: { userId: "user-1" } });
const promoted = await service.finishSetupSession({ sessionId: started.session.id });
expect((promoted.template.metadata as Record<string, unknown>).bootRelevantConfig).toBeUndefined();
const persistedRow = await activeTemplateRow(environmentId);
const boot = (persistedRow.metadata as Record<string, unknown>).bootRelevantConfig as {
values: Record<string, unknown>;
excludedPaths: string[];
};
expect(boot.excludedPaths).toContain("[unresolved-identity-path]");
// The raw invalid paths are never persisted and never value-bearing.
const metadataJson = JSON.stringify(persistedRow.metadata);
expect(metadataJson).not.toContain("bad path!");
expect(metadataJson).not.toContain("a..b");
expect(Object.keys(boot.values)).not.toContain("bad path!");
expect(Object.keys(boot.values)).not.toContain("a..b");
await expect(service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId }))
.rejects.toMatchObject({ status: 409 });
const relinked = await service.relinkActiveTemplate({
environmentId,
confirmBootSourceDrift: true,
actor: relinkActor,
companyId,
});
expect(relinked.classification).toBe("unclassified");
});
it("fails closed for prototype-key driver identity paths and never persists them", async () => {
const { companyId, environmentId } = await seed({ manifest: prototypeKeyIdentityPluginManifest(), config: {
provider: "fake-prototype-key-plugin",
image: "fake:base",
reuseLease: false,
} });
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
const started = await service.startSetupSession({ environmentId, actor: { userId: "user-1" } });
const promoted = await service.finishSetupSession({ sessionId: started.session.id });
expect((promoted.template.metadata as Record<string, unknown>).bootRelevantConfig).toBeUndefined();
const persistedRow = await activeTemplateRow(environmentId);
const boot = (persistedRow.metadata as Record<string, unknown>).bootRelevantConfig as {
values: Record<string, unknown>;
excludedPaths: string[];
};
// Every prototype key fails canonicalization and records only the marker.
expect(boot.excludedPaths).toContain("[unresolved-identity-path]");
// No raw prototype-key path and no value for one persists in the snapshot.
expect(boot.excludedPaths).not.toContain("__proto__");
expect(boot.excludedPaths).not.toContain("constructor");
expect(boot.excludedPaths).not.toContain("prototype");
expect(Object.keys(boot.values)).not.toContain("__proto__");
expect(Object.keys(boot.values)).not.toContain("constructor");
expect(Object.keys(boot.values)).not.toContain("prototype");
// The prototype of the values map stays clean; no key leaked onto it.
expect(Object.getPrototypeOf(boot.values)).toBe(Object.prototype);
const metadataJson = JSON.stringify(persistedRow.metadata);
expect(metadataJson).not.toContain("__proto__");
expect(metadataJson).not.toContain("nested.__proto__");
// An unflagged relink fails closed with 409 and the unclassified class.
await expect(service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId }))
.rejects.toMatchObject({ status: 409, details: { classification: "unclassified" } });
// A confirmed relink succeeds and writes the relinked activity entry.
const relinked = await service.relinkActiveTemplate({
environmentId,
confirmBootSourceDrift: true,
actor: relinkActor,
companyId,
});
expect(relinked.classification).toBe("unclassified");
const activities = await relinkActivityRows(environmentId);
const relinkActivity = activities.find((row) => row.action === "environment.custom_image_template.relinked");
expect(relinkActivity).toBeDefined();
});
it("fails closed when the provider adds a boot-relevant identity path after capture", async () => {
const { companyId, environmentId } = await seed({ config: {
provider: "fake-plugin",
image: "fake:base",
region: "us",
reuseLease: false,
} });
const service = environmentCustomImageService(db, { pluginWorkerManager: createWorkerManager() });
const started = await service.startSetupSession({ environmentId, actor: { userId: "user-1" } });
await service.finishSetupSession({ sessionId: started.session.id });
// A knob-only region change detaches the template. On its own it would
// relink without confirmation.
await db.update(environments)
.set({ config: { provider: "fake-plugin", image: "fake:base", region: "eu", reuseLease: false } })
.where(eq(environments.id, environmentId));
// The provider now declares `region` a boot-relevant identity path. The
// capture snapshot never covered it, so the server cannot verify the boot
// source and must not classify the drift as knob-only.
const manifest = pluginManifest();
const nextManifest = {
...manifest,
environmentDrivers: [
{ ...manifest.environmentDrivers[0], templateIdentityPaths: ["apiUrl", "region"] },
],
};
await db.update(plugins)
.set({ manifestJson: nextManifest })
.where(eq(plugins.pluginKey, manifest.id));
await expect(service.relinkActiveTemplate({ environmentId, actor: relinkActor, companyId }))
.rejects.toMatchObject({ status: 409, details: { classification: "unclassified" } });
const relinked = await service.relinkActiveTemplate({
environmentId,
confirmBootSourceDrift: true,
actor: relinkActor,
companyId,
});
expect(relinked.classification).toBe("unclassified");
});
});
@@ -19,6 +19,10 @@ const SNAPSHOT: EffectiveSandboxCapabilities = {
persistentProcessSessions: true,
independentControlCommands: false,
incrementalSessionOutput: false,
// Concurrent sync operations need BOTH sync verbs; this snapshot verified only
// inbound sync, so the opt-in stays off.
concurrentSyncOperations: false,
duplexCommandStream: false,
};
// A snapshot that grants every capability. A test overrides one flag to prove
@@ -30,6 +34,8 @@ const FULL_GRANT: EffectiveSandboxCapabilities = {
persistentProcessSessions: true,
independentControlCommands: true,
incrementalSessionOutput: true,
concurrentSyncOperations: true,
duplexCommandStream: true,
};
// Build a sandbox execution target with a fixed snapshot and a fixed
@@ -185,6 +191,42 @@ describe("effective snapshot gates the sync decision", () => {
});
});
// The execution target carries the concurrent-sync opt-in on both the effective
// snapshot and the runner. The runner Boolean feeds the sync client, which then
// tells the orchestrator whether it may run sync operations concurrently.
describe("effective snapshot carries the concurrent-sync capability", () => {
beforeEach(() => {
mockResolveEnvironmentDriverConfigForRuntime.mockReset();
});
it("carries the concurrent-sync opt-in on the snapshot and the runner", async () => {
const { target } = await buildSandboxTarget({ snapshot: FULL_GRANT, supportsSync: true });
expect(target.effectiveCapabilities?.concurrentSyncOperations).toBe(true);
expect(target.runner?.allowConcurrentSyncOperations).toBe(true);
});
it("keeps the concurrent-sync opt-in off when the snapshot removes it", async () => {
const { target } = await buildSandboxTarget({
snapshot: { ...FULL_GRANT, concurrentSyncOperations: false },
supportsSync: true,
});
expect(target.effectiveCapabilities?.concurrentSyncOperations).toBe(false);
expect(target.runner?.allowConcurrentSyncOperations).toBe(false);
});
it("keeps the concurrent-sync opt-in off when the resolution rejects", async () => {
// A rejected resolution carries no snapshot; it must not read as an open
// grant. The runner keeps the opt-in off.
const { target } = await buildSandboxTarget({
snapshot: null,
supportsSync: true,
rejectResolution: true,
});
expect(target.effectiveCapabilities).toBeUndefined();
expect(target.runner?.allowConcurrentSyncOperations).toBe(false);
});
});
// Session-output streaming is decided downstream from the carried snapshot
// alone (see `streamAgentSessionOutput` in `acpx-engine/execute.ts`): the bridge
// streams only when the snapshot grants `incrementalSessionOutput`. That key is
@@ -0,0 +1,379 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const { mockResolveEnvironmentDriverConfigForRuntime, mockResolvePluginSandboxProviderDriverById } =
vi.hoisted(() => ({
mockResolveEnvironmentDriverConfigForRuntime: vi.fn(),
mockResolvePluginSandboxProviderDriverById: vi.fn(),
}));
vi.mock("../services/environment-config.js", () => ({
resolveEnvironmentDriverConfigForRuntime: mockResolveEnvironmentDriverConfigForRuntime,
}));
// The centralized duplex authorization gate resolves the exact lease capability
// snapshot before it calls the driver. The plugin branch of the snapshot reads
// the pinned plugin's declaration from the database. These service tests carry no
// database, so replace only the by-id declaration resolver. The test sets the
// resolved declaration per case; every other export stays real.
vi.mock("../services/plugin-environment-driver.js", async (importActual) => ({
...(await importActual<typeof import("../services/plugin-environment-driver.js")>()),
resolvePluginSandboxProviderDriverById: mockResolvePluginSandboxProviderDriverById,
}));
import type { EffectiveSandboxCapabilities } from "@paperclipai/adapter-utils/execution-target";
import { createSshCommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/ssh";
import type { Environment, EnvironmentLease } from "@paperclipai/shared";
import { resolveEnvironmentExecutionTarget } from "../services/environment-execution-target.js";
import {
buildSandboxCapabilityNarrowing,
environmentRuntimeService,
} from "../services/environment-runtime.js";
import type {
EnvironmentRuntimeDriver,
EnvironmentRuntimeService,
} from "../services/environment-runtime.js";
import type {
DuplexChannelHostSession,
PluginWorkerManager,
} from "../services/plugin-worker-manager.js";
// A snapshot that grants the opt-in duplex capability, plus the rest true so the
// gate reads only the duplex flag.
const DUPLEX_GRANT: EffectiveSandboxCapabilities = {
reusableLeases: true,
nativeSyncIn: true,
nativeSyncOut: true,
persistentProcessSessions: true,
independentControlCommands: true,
incrementalSessionOutput: true,
duplexCommandStream: true,
};
const DUPLEX_ABSENT: EffectiveSandboxCapabilities = {
...DUPLEX_GRANT,
duplexCommandStream: false,
};
function flushMicrotasks(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 0));
}
// A fake worker manager whose one worker returns a controllable host session.
// The session records every host→worker call and lets the test settle the exit.
function makeFakeWorkerManager() {
let settleWait: (value: { exitCode: number | null }) => void = () => {};
const waitPromise = new Promise<{ exitCode: number | null }>((resolve) => {
settleWait = resolve;
});
const hostSession = {
onData: vi.fn(),
write: vi.fn(),
wait: vi.fn(() => waitPromise),
kill: vi.fn(),
close: vi.fn(async () => {}),
} satisfies DuplexChannelHostSession;
const openDuplexChannel = vi.fn(async () => hostSession);
const worker = { openDuplexChannel, supportedMethods: ["duplexChannelOpen"] };
const manager = {
getWorker: vi.fn(() => worker),
} as unknown as PluginWorkerManager;
return { manager, worker, hostSession, openDuplexChannel, settleWait };
}
const PLUGIN_LEASE: EnvironmentLease = {
id: "lease-1",
companyId: "company-1",
providerLeaseId: "provider-lease-1",
metadata: {
sandboxProviderPlugin: true,
pluginId: "test.plugin",
provider: "daytona",
},
} as unknown as EnvironmentLease;
const SANDBOX_ENVIRONMENT: Environment = {
id: "env-1",
driver: "sandbox",
} as unknown as Environment;
// A plugin driver declaration that grants the opt-in duplex capability. The
// centralized gate reads it through the by-id resolver and pairs it with the
// worker's verified `duplexChannelOpen` verb to grant the capability.
const DUPLEX_DECLARED_DRIVER = {
plugin: {},
driver: { sandboxCapabilities: { duplexCommandStream: true } },
};
// The fixed refusal the centralized gate throws when the lease does not grant the
// opt-in duplex capability.
const DUPLEX_DENIED = /does not grant the duplex command stream capability/;
beforeEach(() => {
// The default resolves a clean sandbox config and a declaration that grants the
// duplex capability, so the plugin lease reaches the worker. Each refusal test
// overrides one input to deny the capability.
mockResolveEnvironmentDriverConfigForRuntime.mockReset();
mockResolveEnvironmentDriverConfigForRuntime.mockResolvedValue({
driver: "sandbox",
config: { provider: "daytona", timeoutMs: 30_000 },
});
mockResolvePluginSandboxProviderDriverById.mockReset();
mockResolvePluginSandboxProviderDriverById.mockResolvedValue(DUPLEX_DECLARED_DRIVER);
});
describe("sandbox driver duplex channel wiring", () => {
it("reaches the worker manager route with the lease scope and delegates the channel members", async () => {
const { manager, worker, hostSession, settleWait } = makeFakeWorkerManager();
const service = environmentRuntimeService({} as never, { pluginWorkerManager: manager });
const channel = await service.openDuplexChannel({
environment: SANDBOX_ENVIRONMENT,
lease: PLUGIN_LEASE,
command: "bridge-callback",
});
// The driver resolves the worker by the pinned plugin id and passes the same
// lease scope the sandbox execute path uses.
expect(manager.getWorker).toHaveBeenCalledWith("test.plugin");
expect(worker.openDuplexChannel).toHaveBeenCalledWith({
driverKey: "daytona",
companyId: "company-1",
environmentId: "env-1",
providerLeaseId: "provider-lease-1",
command: "bridge-callback",
});
// write / stop / close map to write / kill / close on the host session.
channel.write("input-bytes");
expect(hostSession.write).toHaveBeenCalledWith("input-bytes");
channel.stop();
expect(hostSession.kill).toHaveBeenCalledTimes(1);
await channel.close();
expect(hostSession.close).toHaveBeenCalledTimes(1);
// onData maps one to one.
const dataListener = vi.fn();
channel.onData(dataListener);
expect(hostSession.onData).toHaveBeenCalledWith(dataListener);
// onExit bridges the host session's one-time wait() to the exit listener.
const exitListener = vi.fn();
channel.onExit(exitListener);
expect(exitListener).not.toHaveBeenCalled();
settleWait({ exitCode: 7 });
await flushMicrotasks();
expect(exitListener).toHaveBeenCalledWith({ exitCode: 7 });
});
it("refuses a lease that is not a plugin-backed sandbox lease before it reaches the worker", async () => {
const { manager, worker } = makeFakeWorkerManager();
const service = environmentRuntimeService({} as never, { pluginWorkerManager: manager });
const nonPluginLease = {
id: "lease-2",
companyId: "company-1",
providerLeaseId: "provider-lease-2",
metadata: { pluginId: "test.plugin", provider: "daytona" },
} as unknown as EnvironmentLease;
// The lease is not a plugin-backed sandbox lease, so the effective snapshot
// never grants the opt-in duplex capability. The centralized gate refuses the
// open before the driver, so the worker `duplexChannelOpen` RPC never runs.
await expect(
service.openDuplexChannel({
environment: SANDBOX_ENVIRONMENT,
lease: nonPluginLease,
command: "bridge-callback",
}),
).rejects.toThrow(DUPLEX_DENIED);
expect(worker.openDuplexChannel).not.toHaveBeenCalled();
});
});
// Direct regressions for the centralized authorization gate on
// EnvironmentRuntimeService.openDuplexChannel. The gate resolves the exact lease
// capability snapshot and refuses unless it grants the opt-in duplex capability.
// Every refusal must happen before the driver, so the worker `duplexChannelOpen`
// RPC never runs.
describe("EnvironmentRuntimeService.openDuplexChannel capability gate", () => {
it("refuses a lease whose provider does not declare the duplex capability", async () => {
const { manager, worker } = makeFakeWorkerManager();
// The worker verifies the duplex verb, but the declaration omits the opt-in
// capability. An opt-in capability needs an explicit declaration, so the
// effective snapshot denies it.
mockResolvePluginSandboxProviderDriverById.mockResolvedValue({ plugin: {}, driver: {} });
const service = environmentRuntimeService({} as never, { pluginWorkerManager: manager });
await expect(
service.openDuplexChannel({
environment: SANDBOX_ENVIRONMENT,
lease: PLUGIN_LEASE,
command: "bridge-callback",
}),
).rejects.toThrow(DUPLEX_DENIED);
expect(worker.openDuplexChannel).not.toHaveBeenCalled();
});
it("refuses a lease whose worker does not verify the duplex verb", async () => {
const { manager, worker } = makeFakeWorkerManager();
// The declaration grants the capability, but the worker does not advertise
// the `duplexChannelOpen` verb, so the runtime never verified it. A
// declaration never grants an unverified capability.
worker.supportedMethods = [];
const service = environmentRuntimeService({} as never, { pluginWorkerManager: manager });
await expect(
service.openDuplexChannel({
environment: SANDBOX_ENVIRONMENT,
lease: PLUGIN_LEASE,
command: "bridge-callback",
}),
).rejects.toThrow(DUPLEX_DENIED);
expect(worker.openDuplexChannel).not.toHaveBeenCalled();
});
it("refuses a lease narrowed away from the duplex capability", async () => {
// A fake driver returns a snapshot that grants every capability except the
// duplex one, which models a lease narrowed away from it. The gate keys on
// the exact duplex flag, so it refuses even though the other capabilities are
// granted, and never calls the driver open.
const openDuplexChannel = vi.fn();
const narrowedDriver = {
driver: "sandbox",
acquireRunLease: vi.fn(),
releaseRunLease: vi.fn(),
effectiveSandboxCapabilities: vi.fn(async () => ({ ...DUPLEX_ABSENT })),
openDuplexChannel,
} as unknown as EnvironmentRuntimeDriver;
const service = environmentRuntimeService({} as never, { drivers: [narrowedDriver] });
await expect(
service.openDuplexChannel({
environment: SANDBOX_ENVIRONMENT,
lease: { ...PLUGIN_LEASE, metadata: { ...PLUGIN_LEASE.metadata, driver: "sandbox" } },
command: "bridge-callback",
}),
).rejects.toThrow(DUPLEX_DENIED);
expect(openDuplexChannel).not.toHaveBeenCalled();
});
it("refuses a lease whose provider config fails to resolve", async () => {
const { manager, worker } = makeFakeWorkerManager();
// The declaration grants the capability and the worker verifies the verb, but
// the provider config cannot be resolved. An untrusted provider fails closed,
// so the config-resolution failure narrows the duplex capability away.
mockResolveEnvironmentDriverConfigForRuntime.mockRejectedValue(new Error("config unresolved"));
const service = environmentRuntimeService({} as never, { pluginWorkerManager: manager });
await expect(
service.openDuplexChannel({
environment: SANDBOX_ENVIRONMENT,
lease: PLUGIN_LEASE,
command: "bridge-callback",
}),
).rejects.toThrow(DUPLEX_DENIED);
expect(worker.openDuplexChannel).not.toHaveBeenCalled();
});
it("narrows the duplex capability away when the provider config resolution fails", () => {
// Unit cover for the narrowing rule the config-failure case relies on: a
// failed config resolution fails closed on the opt-in duplex capability.
const narrowing = buildSandboxCapabilityNarrowing({
leasePolicy: "reuse_by_environment",
leaseMetadata: {},
configResolutionFailed: true,
});
expect(narrowing.duplexCommandStream).toBe(false);
const granted = buildSandboxCapabilityNarrowing({
leasePolicy: "reuse_by_environment",
leaseMetadata: {},
configResolutionFailed: false,
});
expect(granted.duplexCommandStream).toBeUndefined();
});
});
// Build a sandbox execution target with a fixed capability snapshot and a fake
// environment runtime whose openDuplexChannel is a spy. The helper returns the
// runner and the spy so a test reads the capability-gated member.
async function buildSandboxRunner(input: {
snapshot: EffectiveSandboxCapabilities | null;
}) {
mockResolveEnvironmentDriverConfigForRuntime.mockResolvedValue({
driver: "sandbox",
config: { provider: "daytona", timeoutMs: 30_000 },
});
const openDuplexChannel = vi.fn(async () => ({
write: vi.fn(),
onData: vi.fn(),
onExit: vi.fn(),
stop: vi.fn(),
close: vi.fn(async () => {}),
}));
const environmentRuntime = {
execute: vi.fn().mockResolvedValue({
exitCode: 0,
signal: null,
timedOut: false,
stdout: "ok",
stderr: "",
metadata: {},
}),
supportsSync: () => false,
syncIn: vi.fn(),
syncOut: vi.fn(),
openDuplexChannel,
effectiveSandboxCapabilities: vi.fn(async () =>
input.snapshot ? Object.freeze({ ...input.snapshot }) : null,
),
} as unknown as EnvironmentRuntimeService;
const target = await resolveEnvironmentExecutionTarget({
db: {} as never,
companyId: "company-1",
adapterType: "codex_local",
environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } },
leaseId: "lease-1",
leaseMetadata: { remoteCwd: "/work" },
lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never,
environmentRuntime,
});
if (!target || target.kind !== "remote" || target.transport !== "sandbox") {
throw new Error("expected a sandbox execution target");
}
return { runner: target.runner, openDuplexChannel };
}
describe("inline sandbox runner duplex capability gate", () => {
it("exposes openDuplexChannel that delegates to the runtime when the capability is granted", async () => {
const { runner, openDuplexChannel } = await buildSandboxRunner({ snapshot: DUPLEX_GRANT });
expect(runner?.openDuplexChannel).toBeDefined();
await runner!.openDuplexChannel!({ command: "bridge-callback" });
expect(openDuplexChannel).toHaveBeenCalledWith({
environment: expect.objectContaining({ id: "env-1" }),
lease: expect.objectContaining({ id: "lease-1" }),
command: "bridge-callback",
});
});
it("omits openDuplexChannel when the capability is absent", async () => {
const { runner } = await buildSandboxRunner({ snapshot: DUPLEX_ABSENT });
expect(runner?.openDuplexChannel).toBeUndefined();
});
it("omits openDuplexChannel when the capability snapshot is null", async () => {
const { runner } = await buildSandboxRunner({ snapshot: null });
expect(runner?.openDuplexChannel).toBeUndefined();
});
});
describe("ssh runner factory", () => {
it("omits openDuplexChannel", () => {
const runner = createSshCommandManagedRuntimeRunner({
spec: { remoteCwd: "/work" } as never,
});
expect(runner.openDuplexChannel).toBeUndefined();
});
});
@@ -10,6 +10,7 @@ const mockBuildWorkspaceRealizationRequest = vi.hoisted(() => vi.fn());
const mockUpdateLeaseMetadata = vi.hoisted(() => vi.fn());
const mockUpdateExecutionWorkspace = vi.hoisted(() => vi.fn());
const mockLogActivity = vi.hoisted(() => vi.fn());
const mockLoggerInfo = vi.hoisted(() => vi.fn());
vi.mock("../services/environment-execution-target.js", () => ({
resolveEnvironmentExecutionTarget: mockResolveEnvironmentExecutionTarget,
@@ -44,6 +45,15 @@ vi.mock("../services/activity-log.js", () => ({
logActivity: mockLogActivity,
}));
vi.mock("../middleware/logger.js", () => ({
logger: {
info: mockLoggerInfo,
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
},
}));
// ---------------------------------------------------------------------------
// Imports after mocks
// ---------------------------------------------------------------------------
@@ -407,6 +417,84 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
});
it("runs a remote provision command after workspace realization when configured", async () => {
mockBuildWorkspaceRealizationRequest.mockReturnValue({
version: 1,
adapterType: "claude_local",
companyId: "company-1",
environmentId: "env-1",
executionWorkspaceId: null,
issueId: null,
heartbeatRunId: "run-1",
requestedMode: null,
source: {
kind: "project_primary",
localPath: "/workspace/project",
projectId: null,
projectWorkspaceId: null,
repoUrl: null,
repoRef: null,
strategy: "project_primary",
branchName: null,
worktreePath: null,
},
runtimeOverlay: {
provisionCommand: "npm install -g @anthropic-ai/claude-code",
},
});
mockResolveEnvironmentExecutionTarget.mockResolvedValue({
kind: "remote",
transport: "ssh",
remoteCwd: "/remote/workspace",
environmentId: "env-1",
leaseId: "lease-1",
spec: {
host: "ssh.example.test",
port: 22,
username: "ssh-user",
remoteCwd: "/remote/workspace",
remoteWorkspacePath: "/remote/workspace",
privateKey: null,
knownHosts: null,
strictHostKeyChecking: true,
},
});
const runtime = makeMockRuntime({
realizeWorkspace: vi.fn().mockResolvedValue({
cwd: "/remote/workspace",
metadata: {
workspaceRealization: {
version: 1,
transport: "ssh",
remote: { path: "/remote/workspace" },
},
},
}),
});
const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
await orchestrator.realizeForRun(makeRealizeInput({
environment: makeEnvironment("ssh"),
}));
// The `ssh` driver runs the command on the remote host that shares the
// workspace path, so the configured provision command still runs there.
expect(runtime.execute).toHaveBeenCalledOnce();
expect(runtime.execute).toHaveBeenCalledWith(expect.objectContaining({
environment: expect.objectContaining({ driver: "ssh" }),
lease: expect.objectContaining({ id: "lease-1" }),
command: "bash",
args: ["-lc", "npm install -g @anthropic-ai/claude-code"],
cwd: "/remote/workspace",
env: {
SHELL: "/bin/bash",
},
}));
// The sandbox skip log is specific to the sandbox driver; ssh stays quiet.
expect(mockLoggerInfo).not.toHaveBeenCalled();
});
it("skips the host provision command for a sandbox environment and logs the skip", async () => {
mockBuildWorkspaceRealizationRequest.mockReturnValue({
version: 1,
adapterType: "claude_local",
@@ -458,17 +546,59 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
environment: makeEnvironment("sandbox"),
}));
expect(runtime.execute).toHaveBeenCalledOnce();
expect(runtime.execute).toHaveBeenCalledWith(expect.objectContaining({
environment: expect.objectContaining({ driver: "sandbox" }),
lease: expect.objectContaining({ id: "lease-1" }),
command: "bash",
args: ["-lc", "npm install -g @anthropic-ai/claude-code"],
cwd: "/remote/workspace",
env: {
SHELL: "/bin/bash",
// The sandbox receives the provisioned tree through the adapter stage.sync
// step, so the orchestrator must not run the host command in the sandbox.
expect(runtime.execute).not.toHaveBeenCalled();
// The skip is observable: exactly one log line records it with the driver.
expect(mockLoggerInfo).toHaveBeenCalledOnce();
expect(mockLoggerInfo).toHaveBeenCalledWith(
expect.objectContaining({ driver: "sandbox", environmentId: "env-1" }),
expect.stringContaining("Skip host provisionCommand"),
);
});
it("does not rerun the provision command during local environment realization", async () => {
mockBuildWorkspaceRealizationRequest.mockReturnValue({
version: 1,
adapterType: "claude_local",
companyId: "company-1",
environmentId: "env-1",
executionWorkspaceId: null,
issueId: null,
heartbeatRunId: "run-1",
requestedMode: null,
source: {
kind: "project_primary",
localPath: "/workspace/project",
projectId: null,
projectWorkspaceId: null,
repoUrl: null,
repoRef: null,
strategy: "project_primary",
branchName: null,
worktreePath: null,
},
runtimeOverlay: {
provisionCommand: "npm install -g @anthropic-ai/claude-code",
},
});
mockResolveEnvironmentExecutionTarget.mockResolvedValue({
kind: "local",
environmentId: "env-1",
leaseId: "lease-1",
});
const runtime = makeMockRuntime();
const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
await orchestrator.realizeForRun(makeRealizeInput({
environment: makeEnvironment("local"),
}));
// Local workspace provisioning already ran the command before realizeForRun.
expect(runtime.execute).not.toHaveBeenCalled();
// The sandbox skip log is specific to the sandbox driver; local stays quiet.
expect(mockLoggerInfo).not.toHaveBeenCalled();
});
it("runs project-level provision commands for ssh environments", async () => {
@@ -588,7 +718,7 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
await expect(orchestrator.realizeForRun(makeRealizeInput({
environment: makeEnvironment("sandbox"),
environment: makeEnvironment("ssh"),
}))).rejects.toSatisfy(
(err: unknown) =>
err instanceof EnvironmentRunError &&
@@ -1,3 +1,8 @@
import { EventEmitter } from "node:events";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { PassThrough } from "node:stream";
import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -24,11 +29,18 @@ const mockStartRuntimeServices = vi.hoisted(() => vi.fn());
const mockStopRuntimeServicesForExecutionWorkspace = vi.hoisted(() => vi.fn());
const mockEnsurePersistedExecutionWorkspaceAvailable = vi.hoisted(() => vi.fn());
const mockBuildWorkspaceRuntimeDesiredStatePatch = vi.hoisted(() => vi.fn());
const mockListConfiguredRuntimeServiceEntries = vi.hoisted(() => vi.fn());
// The integrated control path also takes the durable runtime-control lease (PAP-17205). This
// suite covers the in-flight guard and failed-start reconciliation, so the lease always grants;
// `execution-workspace-runtime-lease-route.test.ts` exercises the lease itself against a real db.
const mockClaimRuntimeLease = vi.hoisted(() => vi.fn(async () => null));
const mockReleaseRuntimeLease = vi.hoisted(() => vi.fn(async () => undefined));
const mockSpawn = vi.hoisted(() => vi.fn());
vi.mock("node:child_process", async () => ({
...(await vi.importActual<typeof import("node:child_process")>("node:child_process")),
spawn: mockSpawn,
}));
vi.mock("../telemetry.js", () => ({ getTelemetryClient: mockGetTelemetryClient }));
@@ -45,14 +57,14 @@ vi.mock("../services/index.js", () => ({
claim: mockClaimRuntimeLease,
release: mockReleaseRuntimeLease,
}),
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/workspace-runtime.js", () => ({
buildWorkspaceRuntimeDesiredStatePatch: mockBuildWorkspaceRuntimeDesiredStatePatch,
cleanupExecutionWorkspaceArtifacts: vi.fn(),
ensurePersistedExecutionWorkspaceAvailable: mockEnsurePersistedExecutionWorkspaceAvailable,
listConfiguredRuntimeServiceEntries: vi.fn(() => []),
listConfiguredRuntimeServiceEntries: mockListConfiguredRuntimeServiceEntries,
runWorkspaceJobForControl: vi.fn(),
startRuntimeServicesForWorkspaceControl: mockStartRuntimeServices,
stopRuntimeServicesForExecutionWorkspace: mockStopRuntimeServicesForExecutionWorkspace,
@@ -65,6 +77,9 @@ vi.mock("../routes/workspace-runtime-service-authz.js", () => ({
}));
const executionWorkspaceId = "33333333-3333-4333-8333-333333333333";
const projectId = "44444444-4444-4444-8444-444444444444";
const projectWorkspaceId = "55555555-5555-4555-8555-555555555555";
let registeredProjectWorkspace: { id: string; cwd: string; metadata: null } | null = null;
function buildExecutionWorkspace(overrides: Record<string, unknown> = {}) {
return {
@@ -122,11 +137,100 @@ async function createApp() {
};
next();
});
app.use("/api", executionWorkspaceRoutes({} as any));
const db = {
select: (selection: Record<string, unknown>) => ({
from: () => ({
where: () => Promise.resolve(
Object.prototype.hasOwnProperty.call(selection, "cwd")
? registeredProjectWorkspace ? [registeredProjectWorkspace] : []
: [{ executionWorkspacePolicy: null }],
),
}),
}),
};
app.use("/api", executionWorkspaceRoutes(db as any));
app.use(errorHandler);
return app;
}
function createRegisteredRepairFixture(
prefix: string,
options: { targetInstanceId?: string; withCli?: boolean } = {},
) {
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
const baseCwd = fs.mkdtempSync(path.join(os.tmpdir(), `${prefix}base-`));
const configDir = path.join(workspaceCwd, ".paperclip");
const sourceConfigPath = path.join(baseCwd, ".paperclip", "config.json");
const targetInstanceId = options.targetInstanceId ?? "repair-target";
const cliRunner = path.join(baseCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(baseCwd, "cli", "src", "index.ts");
fs.mkdirSync(configDir, { recursive: true });
fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true });
fs.writeFileSync(sourceConfigPath, "{}\n");
fs.writeFileSync(path.join(path.dirname(sourceConfigPath), ".env"), "PAPERCLIP_INSTANCE_ID=repair-source\n");
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
fs.writeFileSync(path.join(configDir, ".env"), `PAPERCLIP_INSTANCE_ID=${targetInstanceId}\n`);
if (options.withCli !== false) {
fs.mkdirSync(path.dirname(cliRunner), { recursive: true });
fs.mkdirSync(path.dirname(cliEntry), { recursive: true });
fs.writeFileSync(cliRunner, "// test runner\n");
fs.writeFileSync(cliEntry, "// test entry\n");
}
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "repair-source", configPath: sourceConfigPath },
targetInstanceId,
state: "failed",
attemptId: "previous",
}));
registeredProjectWorkspace = { id: projectWorkspaceId, cwd: baseCwd, metadata: null };
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({
cwd: workspaceCwd,
projectId,
projectWorkspaceId,
}));
return { workspaceCwd, baseCwd, configDir, sourceConfigPath, targetInstanceId };
}
function removeRegisteredRepairFixture(fixture: { workspaceCwd: string; baseCwd: string }) {
fs.rmSync(fixture.workspaceCwd, { recursive: true, force: true });
fs.rmSync(fixture.baseCwd, { recursive: true, force: true });
}
function mockVerifiedReseed(
fixture: ReturnType<typeof createRegisteredRepairFixture>,
targetInstanceId = fixture.targetInstanceId,
) {
mockSpawn.mockImplementation(() => {
const child = new EventEmitter() as EventEmitter & {
stdout: PassThrough;
stderr: PassThrough;
kill: ReturnType<typeof vi.fn>;
};
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = vi.fn();
queueMicrotask(() => {
fs.writeFileSync(path.join(fixture.configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "repair-source", configPath: fixture.sourceConfigPath },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "full",
migrationRevision: "0142_test.sql",
targetInstanceId,
phase: "complete",
state: "verified",
attemptId: "repair-attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}));
child.emit("exit", 0);
});
return child;
});
}
/**
* Route-level guarantees for PAP-17249: the competing lane in the PAP-17207 report must get a
* stable 409 while a control is genuinely live, and a start that fails must leave the workspace
@@ -135,6 +239,8 @@ async function createApp() {
describe.sequential("execution workspace runtime control conflict and failure reconciliation", () => {
beforeEach(() => {
vi.clearAllMocks();
mockSpawn.mockReset();
registeredProjectWorkspace = null;
mockAccessService.decide.mockResolvedValue({
allowed: true,
action: "runtime:manage",
@@ -159,13 +265,14 @@ describe.sequential("execution workspace runtime control conflict and failure re
repoRef: "main",
});
mockStopRuntimeServicesForExecutionWorkspace.mockResolvedValue(undefined);
mockListConfiguredRuntimeServiceEntries.mockReturnValue([{ name: "app" }]);
mockWorkspaceOperationService.createRecorder.mockReturnValue({
attachExecutionWorkspaceId: vi.fn(),
recordOperation: async (input: any) => {
// Mirror the real recorder: a throwing `run` becomes a terminal failed operation and
// the error still propagates to the caller.
try {
await input.run();
await input.run(async () => undefined);
} catch (error) {
(error as any).recordedOperationStatus = "failed";
throw error;
@@ -247,6 +354,320 @@ describe.sequential("execution workspace runtime control conflict and failure re
);
});
it("returns 422 with a stable reason when the repair seed manifest is malformed", async () => {
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-malformed-"));
try {
const configDir = path.join(workspaceCwd, ".paperclip");
fs.mkdirSync(configDir, { recursive: true });
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), "{ definitely-not-json\n");
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
const res = await request(await createApp())
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(422);
expect(res.body).toMatchObject({
code: "workspace_repair_precondition_failed",
reason: "seed_manifest_malformed",
repairPhase: "precondition_validation",
});
expect(mockSpawn).not.toHaveBeenCalled();
} finally {
fs.rmSync(workspaceCwd, { recursive: true, force: true });
}
});
it.each(["sibling", "foreign_instance", "symlink", "instance_mismatch"] as const)(
"rejects a %s repair manifest source before spawn or runtime mutation",
async (variant) => {
const fixture = createRegisteredRepairFixture(`paperclip-route-repair-${variant}-`);
const attackerDir = path.join(path.dirname(fixture.workspaceCwd), `${path.basename(fixture.workspaceCwd)}-attacker`);
try {
fs.mkdirSync(attackerDir, { recursive: true });
const attackerConfig = path.join(attackerDir, "config.json");
fs.writeFileSync(attackerConfig, "{}\n");
fs.writeFileSync(
path.join(attackerDir, ".env"),
`PAPERCLIP_INSTANCE_ID=${variant === "foreign_instance" ? "foreign-source" : "repair-source"}\n`,
);
const diagnosticPath = variant === "instance_mismatch"
? fixture.sourceConfigPath
: variant === "symlink"
? path.join(fixture.baseCwd, "source-alias.json")
: attackerConfig;
if (variant === "symlink") fs.symlinkSync(fixture.sourceConfigPath, diagnosticPath);
fs.writeFileSync(path.join(fixture.configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: {
instanceId: variant === "foreign_instance" || variant === "instance_mismatch"
? "foreign-source"
: "repair-source",
configPath: diagnosticPath,
},
targetInstanceId: fixture.targetInstanceId,
state: "failed",
attemptId: "attacker-controlled",
}));
const res = await request(await createApp())
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(422);
expect(res.body).toMatchObject({
code: "workspace_repair_precondition_failed",
reason: "source_registration_invalid",
repairPhase: "precondition_validation",
});
expect(mockSpawn).not.toHaveBeenCalled();
expect(mockStopRuntimeServicesForExecutionWorkspace).not.toHaveBeenCalled();
expect(mockWorkspaceOperationService.createRecorder).not.toHaveBeenCalled();
expect(mockWorkspaceOperationService.assertRuntimeControlAvailable).not.toHaveBeenCalled();
} finally {
fs.rmSync(attackerDir, { recursive: true, force: true });
removeRegisteredRepairFixture(fixture);
}
},
);
it("rejects repair when the execution workspace has no registered base workspace", async () => {
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-no-source-"));
try {
const configDir = path.join(workspaceCwd, ".paperclip");
fs.mkdirSync(configDir, { recursive: true });
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: path.join(workspaceCwd, "missing", "config.json") },
state: "failed",
attemptId: "previous",
}));
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
const res = await request(await createApp())
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(422);
expect(res.body).toMatchObject({
code: "workspace_repair_precondition_failed",
reason: "source_registration_invalid",
repairPhase: "precondition_validation",
});
expect(mockSpawn).not.toHaveBeenCalled();
expect(mockStopRuntimeServicesForExecutionWorkspace).not.toHaveBeenCalled();
expect(mockWorkspaceOperationService.assertRuntimeControlAvailable).not.toHaveBeenCalled();
} finally {
fs.rmSync(workspaceCwd, { recursive: true, force: true });
}
});
it("rejects repair when the registered base workspace has no runnable Paperclip CLI", async () => {
const fixture = createRegisteredRepairFixture("paperclip-route-repair-no-cli-", { withCli: false });
try {
const res = await request(await createApp())
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(422);
expect(res.body).toMatchObject({
code: "workspace_repair_precondition_failed",
reason: "source_registration_invalid",
repairPhase: "precondition_validation",
});
expect(mockSpawn).not.toHaveBeenCalled();
expect(mockStopRuntimeServicesForExecutionWorkspace).not.toHaveBeenCalled();
} finally {
removeRegisteredRepairFixture(fixture);
}
});
it("returns 422 with a stable reason when the verified manifest names another instance", async () => {
const fixture = createRegisteredRepairFixture("paperclip-route-repair-wrong-instance-");
try {
mockVerifiedReseed(fixture, "another-workspace-instance");
const res = await request(await createApp())
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(422);
expect(res.body).toMatchObject({
code: "workspace_repair_precondition_failed",
reason: "seed_manifest_instance_mismatch",
repairPhase: "full_reseed",
});
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
} finally {
removeRegisteredRepairFixture(fixture);
}
});
it("uses the recorded instance pointer consistently for readiness, handoff, and repair", async () => {
const recordedInstanceId = "recorded-repair-instance";
const fixture = createRegisteredRepairFixture("paperclip-route-repair-success-", {
targetInstanceId: recordedInstanceId,
});
try {
const { deriveWorktreeInstanceId } = await import("../services/workspace-instance-cleanup.js");
const {
resolveManagedWorkspaceIdentity,
resolveManagedWorkspaceInstanceId,
} = await import("../services/managed-workspace-identity.js");
expect(deriveWorktreeInstanceId(fixture.workspaceCwd)).not.toBe(recordedInstanceId);
expect(resolveManagedWorkspaceInstanceId(fixture.workspaceCwd)).toBe(recordedInstanceId);
expect(resolveManagedWorkspaceIdentity({
workspaceCwd: fixture.workspaceCwd,
executionWorkspaceId,
companyId: "company-1",
env: { PAPERCLIP_WORKSPACE_HANDOFF_SECRET: "test-root-secret" },
})?.instanceId).toBe(recordedInstanceId);
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: fixture.workspaceCwd });
mockStartRuntimeServices.mockResolvedValue([{
id: "service-1",
status: "running",
healthStatus: "healthy",
}]);
mockSpawn.mockImplementation((_command: string, args: string[], options: { env?: NodeJS.ProcessEnv }) => {
expect(args).toEqual(expect.arrayContaining([
"worktree", "reseed", "--from-config", fixture.sourceConfigPath,
"--seed-mode", "full", "--yes", "--backup-target",
]));
expect(options.env).toMatchObject({
PAPERCLIP_SEED_EXPECTED_COMPANY_ID: "company-1",
PAPERCLIP_WORKSPACE_BASE_CWD: fixture.baseCwd,
PAPERCLIP_PROJECT_WORKSPACE_ID: projectWorkspaceId,
});
const child = new EventEmitter() as EventEmitter & {
stdout: PassThrough;
stderr: PassThrough;
kill: ReturnType<typeof vi.fn>;
};
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = vi.fn();
queueMicrotask(() => {
fs.writeFileSync(path.join(fixture.configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "repair-source", configPath: fixture.sourceConfigPath },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "full",
migrationRevision: "0142_test.sql",
targetInstanceId: recordedInstanceId,
phase: "complete",
state: "verified",
attemptId: "repair-attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}));
child.emit("exit", 0);
});
return child;
});
const app = await createApp();
const res = await request(app)
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(200);
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledWith(
expect.objectContaining({ executionWorkspaceId, workspaceCwd: fixture.workspaceCwd }),
);
expect(mockStartRuntimeServices).toHaveBeenCalledTimes(1);
expect(mockLogActivity).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ action: "execution_workspace.runtime_repair" }),
);
} finally {
removeRegisteredRepairFixture(fixture);
}
});
it("completes database-only repair when no managed runtime service is configured", async () => {
const fixture = createRegisteredRepairFixture("paperclip-route-repair-db-only-");
try {
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({
cwd: fixture.workspaceCwd,
projectId,
projectWorkspaceId,
config: { desiredState: "stopped" },
}));
mockListConfiguredRuntimeServiceEntries.mockReturnValue([]);
mockVerifiedReseed(fixture);
const app = await createApp();
const res = await request(app)
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(200);
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledTimes(1);
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
expect(mockBuildWorkspaceRuntimeDesiredStatePatch).toHaveBeenCalledWith(
expect.objectContaining({ action: "stop" }),
);
} finally {
removeRegisteredRepairFixture(fixture);
}
});
it("fails repair at the exact seed phase and leaves managed services stopped", async () => {
const fixture = createRegisteredRepairFixture("paperclip-route-repair-failure-");
try {
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: fixture.workspaceCwd });
mockSpawn.mockImplementation(() => {
const child = new EventEmitter() as EventEmitter & {
stdout: PassThrough;
stderr: PassThrough;
kill: ReturnType<typeof vi.fn>;
};
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = vi.fn();
queueMicrotask(() => {
fs.writeFileSync(path.join(fixture.configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "repair-source", configPath: fixture.sourceConfigPath },
targetInstanceId: fixture.targetInstanceId,
state: "failed",
phase: "restore",
attemptId: "repair-attempt",
}));
child.emit("exit", 4);
});
return child;
});
const progress: Array<Record<string, unknown>> = [];
mockWorkspaceOperationService.createRecorder.mockReturnValue({
attachExecutionWorkspaceId: vi.fn(),
recordOperation: async (input: any) => {
await input.run(async (entry: Record<string, unknown>) => {
progress.push(entry);
});
},
});
const app = await createApp();
const res = await request(app)
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(500);
expect(res.body).toEqual({ error: "Internal server error" });
expect(progress).toEqual(expect.arrayContaining([
expect.objectContaining({ metadata: { seedFailurePhase: "restore" } }),
]));
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledTimes(2);
} finally {
removeRegisteredRepairFixture(fixture);
}
});
it("reconciles once when the operation's own time budget fails a start that never settles", async () => {
const { WorkspaceOperationTimeoutError } = await import("../services/workspace-operations.js");
// The recorder's ceiling fires outside `run`, so only the outer handler can reconcile.
@@ -19,6 +19,7 @@ const mockExecutionWorkspaceService = vi.hoisted(() => ({
const mockWorkspaceOperationService = vi.hoisted(() => ({
listForExecutionWorkspace: vi.fn(),
createRecorder: vi.fn(),
assertRuntimeControlAvailable: vi.fn(async () => undefined),
}));
const mockWorkspaceRuntimeLeaseService = vi.hoisted(() => ({
@@ -47,7 +48,7 @@ vi.mock("../services/index.js", () => ({
logActivity: mockLogActivity,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/environment-runtime.js", () => ({
@@ -259,6 +259,10 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
pullRequestDetailsByKey.get(`${companyId}:${reference.number}`)
?? { state: "unknown", headRef: null, headSha: null }
),
// Disable the reaper cooldown for the delivery, terminal, race, and
// cleanup tests. They assert immediate reaping. The cooldown gets its own
// tests further down.
workspaceReaperCooldownDays: 0,
});
}, 20_000);
@@ -687,6 +691,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
// workspace first.
const service = executionWorkspaceService(db, {
resolvePullRequestDetails: async () => ({ state: "unknown", headRef: null, headSha: null }),
workspaceReaperCooldownDays: 0,
});
const firstSweep = await service.sweepTerminalWorkspaces(1);
@@ -727,6 +732,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
const service = executionWorkspaceService(db, {
resolvePullRequestDetails: async () => ({ state: "unknown", headRef: null, headSha: null }),
now: () => new Date(clockMs),
workspaceReaperCooldownDays: 0,
});
// An eligible ancestry workspace with an old updatedAt. Its source issue
@@ -829,6 +835,105 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
expect(finalState?.status).toBe("archived");
}, 30_000);
describe("reaper cooldown", () => {
const DAY_MS = 24 * 60 * 60 * 1000;
const nowMs = Date.UTC(2026, 5, 1);
function cooldownService(cooldownDays: number) {
return executionWorkspaceService(db, {
resolvePullRequestDetails: async (companyId, reference) =>
pullRequestDetailsByKey.get(`${companyId}:${reference.number}`)
?? { state: "unknown", headRef: null, headSha: null },
now: () => new Date(nowMs),
workspaceReaperCooldownDays: cooldownDays,
});
}
async function statusOf(executionWorkspaceId: string) {
const [row] = await db
.select({ status: executionWorkspaces.status })
.from(executionWorkspaces)
.where(eq(executionWorkspaces.id, executionWorkspaceId));
return row?.status ?? null;
}
it("skips a terminal tree that is younger than the cooldown", async () => {
const seeded = await seedTerminalWorkspace({ mergedPr: true });
// Keep the workspace inside the sweep boundary that the fixed clock sets.
await db
.update(executionWorkspaces)
.set({ updatedAt: new Date(nowMs - DAY_MS) })
.where(eq(executionWorkspaces.id, seeded.executionWorkspaceId));
// The issue became terminal one day ago. A cooldown of seven days is not
// over yet.
await db
.update(issues)
.set({ completedAt: new Date(nowMs - DAY_MS) })
.where(eq(issues.id, seeded.sourceIssueId));
const sweep = await cooldownService(7).sweepTerminalWorkspaces();
expect(sweep).toMatchObject({ archived: 0, skippedCooldown: 1 });
expect(await statusOf(seeded.executionWorkspaceId)).toBe("active");
}, 20_000);
it("archives a terminal tree that is older than the cooldown", async () => {
const seeded = await seedTerminalWorkspace({ mergedPr: true });
await db
.update(executionWorkspaces)
.set({ updatedAt: new Date(nowMs - DAY_MS) })
.where(eq(executionWorkspaces.id, seeded.executionWorkspaceId));
// The issue became terminal ten days ago. The seven-day cooldown is over.
await db
.update(issues)
.set({ completedAt: new Date(nowMs - 10 * DAY_MS) })
.where(eq(issues.id, seeded.sourceIssueId));
const sweep = await cooldownService(7).sweepTerminalWorkspaces();
expect(sweep).toMatchObject({ archived: 1, skippedCooldown: 0 });
expect(await statusOf(seeded.executionWorkspaceId)).toBe("archived");
}, 20_000);
it("archives immediately when the cooldown is zero", async () => {
const seeded = await seedTerminalWorkspace({ mergedPr: true });
await db
.update(executionWorkspaces)
.set({ updatedAt: new Date(nowMs - DAY_MS) })
.where(eq(executionWorkspaces.id, seeded.executionWorkspaceId));
// The issue became terminal now. A cooldown of zero disables the wait.
await db
.update(issues)
.set({ completedAt: new Date(nowMs) })
.where(eq(issues.id, seeded.sourceIssueId));
const sweep = await cooldownService(0).sweepTerminalWorkspaces();
expect(sweep).toMatchObject({ archived: 1, skippedCooldown: 0 });
expect(await statusOf(seeded.executionWorkspaceId)).toBe("archived");
}, 20_000);
it("falls back to updatedAt when completedAt is null and gates the archive", async () => {
const seeded = await seedTerminalWorkspace({ mergedPr: true });
// The done issue has no completedAt. The anchor falls back to updatedAt.
// Set updatedAt two days ago, inside the seven-day cooldown, so the sweep
// must skip.
await db
.update(executionWorkspaces)
.set({ updatedAt: new Date(nowMs - 2 * DAY_MS) })
.where(eq(executionWorkspaces.id, seeded.executionWorkspaceId));
await db
.update(issues)
.set({ completedAt: null, updatedAt: new Date(nowMs - 2 * DAY_MS) })
.where(eq(issues.id, seeded.sourceIssueId));
const sweep = await cooldownService(7).sweepTerminalWorkspaces();
expect(sweep).toMatchObject({ archived: 0, skippedCooldown: 1 });
expect(await statusOf(seeded.executionWorkspaceId)).toBe("active");
}, 20_000);
});
it("does not treat an unrelated inbound issue mention as delivery evidence", async () => {
const seeded = await seedTerminalWorkspace();
const unrelatedIssueId = randomUUID();
@@ -965,6 +1070,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
const racingService = executionWorkspaceService(db, {
resolvePullRequestDetails: async (_companyId, reference) =>
pullRequestDetailsByKey.get(`${seeded.companyId}:${reference.number}`) ?? { state: "unknown" },
workspaceReaperCooldownDays: 0,
beforeTerminalWorkspaceCleanup: async () => {
await fs.writeFile(path.join(seeded.worktreePath, "late-work.txt"), "not delivered\n", "utf8");
},
@@ -994,6 +1100,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
const racingService = executionWorkspaceService(db, {
resolvePullRequestDetails: async (_companyId, reference) =>
pullRequestDetailsByKey.get(`${seeded.companyId}:${reference.number}`) ?? { state: "unknown" },
workspaceReaperCooldownDays: 0,
beforeTerminalWorkspaceCleanup: async (workspace) => {
// Stand in for a reopen and a fresh archive that ran after this sweep
// captured the generation. Raise the generation past the captured value,
@@ -1597,6 +1704,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
const racingService = executionWorkspaceService(db, {
resolvePullRequestDetails: async (_companyId, reference) =>
pullRequestDetailsByKey.get(`${failSeed.companyId}:${reference.number}`) ?? { state: "unknown" },
workspaceReaperCooldownDays: 0,
beforeTerminalWorkspaceCleanup: async (workspace) => {
await db
.update(executionWorkspaces)
@@ -1639,6 +1747,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
const lockingService = executionWorkspaceService(db, {
resolvePullRequestDetails: async (_companyId, reference) =>
pullRequestDetailsByKey.get(`${seeded.companyId}:${reference.number}`) ?? { state: "unknown" },
workspaceReaperCooldownDays: 0,
beforeTerminalWorkspaceCleanup: async () => {
try {
await runGit(seeded.worktreePath, ["commit", "--allow-empty", "-m", "Late commit"]);
@@ -0,0 +1,174 @@
// Test worker fixture for the host-owned duplex channel route state machine. The
// fixture drives the manager route state machine through the four typed methods
// (open, write, stop, close) and the data and exit notifications.
//
// The duplex channel is generic. It carries no command allowlist. The test
// encodes a JSON directive in the forwarded `providerLeaseId`, so one fixture
// serves every route case:
// - `mode`: "normal" | "malformed-open" | "no-open-reply" | "duplicate-open-reply" |
// "no-write-reply"
// - `workerSessionId`: the worker session id the open reply returns (default "ws-1")
// - `data`: an array of `{ chunk, sid? }`. The fixture emits each as a data
// notification after the open reply. `sid` defaults to the real worker session
// id; a test sets a wrong `sid` to prove the host drops a mismatched
// notification and counts a protocol error.
// - `exitCode`: when set, the fixture emits an exit notification after the data.
// - `echoInput`: when true, the fixture echoes each `duplexChannelWrite` back as
// one data notification for the bound session.
// - `closeMode`: "ack" | "bad-ack" | "no-ack" (default "ack"). It controls the
// close reply, so a test proves the host retires the worker on an unconfirmed
// close.
const readline = require("node:readline");
function send(message) {
process.stdout.write(`${JSON.stringify(message)}\n`);
}
// The registered channels, keyed by the host route id. Each entry records the
// bound worker session id and the close directive.
const routes = new Map();
function parseDirective(raw) {
try {
const parsed = JSON.parse(raw);
return parsed && typeof parsed === "object" ? parsed : {};
} catch {
return {};
}
}
const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity });
rl.on("line", (line) => {
if (!line.trim()) return;
const message = JSON.parse(line);
const method = message && typeof message.method === "string" ? message.method : null;
const params = message.params ?? {};
if (method === "initialize") {
send({
jsonrpc: "2.0",
id: message.id,
result: {
ok: true,
supportedMethods: [
"duplexChannelOpen",
"duplexChannelWrite",
"duplexChannelStop",
"duplexChannelClose",
],
},
});
return;
}
if (method === "duplexChannelOpen") {
const directive = parseDirective(params.providerLeaseId);
const mode = directive.mode ?? "normal";
const workerSessionId = directive.workerSessionId ?? "ws-1";
const closeMode = directive.closeMode ?? "ack";
routes.set(params.hostRouteId, {
workerSessionId,
closeMode,
echoInput: directive.echoInput === true,
noWriteReply: mode === "no-write-reply",
});
if (mode === "no-open-reply") {
// Never reply, so the host open call times out.
return;
}
if (mode === "malformed-open") {
// Reply with no worker session id, so the host terminalizes the route.
send({ jsonrpc: "2.0", id: message.id, result: {} });
return;
}
const reply = () =>
send({ jsonrpc: "2.0", id: message.id, result: { workerSessionId } });
reply();
if (mode === "duplicate-open-reply") {
// Send a second open reply for the same request id. The host drops it.
reply();
}
// Emit the scripted data and the exit after the open reply, so the host
// binds the route first.
setImmediate(() => {
const data = Array.isArray(directive.data) ? directive.data : [];
for (const entry of data) {
send({
jsonrpc: "2.0",
method: "duplexChannel.data",
params: {
workerSessionId: entry.sid ?? workerSessionId,
chunk: entry.chunk,
},
});
}
if (typeof directive.exitCode === "number") {
send({
jsonrpc: "2.0",
method: "duplexChannel.exit",
params: { workerSessionId, exitCode: directive.exitCode },
});
}
});
return;
}
if (method === "duplexChannelWrite") {
const entry = [...routes.values()].find(
(route) => route.workerSessionId === params.workerSessionId,
);
if (entry && entry.noWriteReply) {
// Never reply, so the host write call stays pending. The test proves the
// host ends the route on the pending-request bound.
return;
}
if (entry && entry.echoInput) {
// Echo the input back as one data notification for the bound session, so a
// test proves the input reaches the worker and the output routes back.
send({
jsonrpc: "2.0",
method: "duplexChannel.data",
params: { workerSessionId: entry.workerSessionId, chunk: `echo:${params.data}` },
});
}
send({ jsonrpc: "2.0", id: message.id, result: null });
return;
}
if (method === "duplexChannelStop") {
send({ jsonrpc: "2.0", id: message.id, result: null });
return;
}
if (method === "duplexChannelClose") {
const entry = routes.get(params.hostRouteId);
routes.delete(params.hostRouteId);
const closeMode = entry ? entry.closeMode : "ack";
if (closeMode === "no-ack") {
// Never reply, so the host close call times out and the host retires us.
return;
}
if (closeMode === "bad-ack") {
send({ jsonrpc: "2.0", id: message.id, result: { hostRouteId: "mismatched-route" } });
return;
}
send({ jsonrpc: "2.0", id: message.id, result: { hostRouteId: params.hostRouteId } });
return;
}
if (method === "shutdown") {
send({ jsonrpc: "2.0", id: message.id, result: {} });
setImmediate(() => process.exit(0));
return;
}
send({
jsonrpc: "2.0",
id: message.id,
error: { code: -32601, message: `Unhandled method: ${method}` },
});
});
@@ -0,0 +1,178 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import express from "express";
import request from "supertest";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import { healthRoutes } from "../routes/health.js";
import {
WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY,
WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY,
WORKSPACE_HANDOFF_KEY_ENV_KEY,
WORKSPACE_READINESS_TOKEN_ENV_KEY,
WORKSPACE_READINESS_TOKEN_HEADER,
WORKSPACE_READINESS_USER_EMAIL_HEADER,
WORKSPACE_READINESS_USER_ID_HEADER,
} from "../auth/workspace-login-handoff.js";
const tempDirs: string[] = [];
const envKeys = [
"PAPERCLIP_CONFIG",
WORKSPACE_HANDOFF_KEY_ENV_KEY,
WORKSPACE_READINESS_TOKEN_ENV_KEY,
WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY,
WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY,
] as const;
const previousEnv = new Map<string, string | undefined>();
function setEnv(values: Record<string, string>) {
for (const key of envKeys) {
if (!previousEnv.has(key)) previousEnv.set(key, process.env[key]);
}
for (const [key, value] of Object.entries(values)) process.env[key] = value;
}
function createSeededWorkspace() {
const dir = mkdtempSync(path.join(os.tmpdir(), "paperclip-health-workspace-"));
tempDirs.push(dir);
writeFileSync(path.join(dir, "config.json"), "{}\n", "utf8");
writeFileSync(
path.join(dir, "seed-manifest.json"),
JSON.stringify({ version: 2, state: "verified", phase: "complete", seedMode: "minimal" }),
"utf8",
);
return path.join(dir, "config.json");
}
/**
* Health's own queries (`instance_user_roles` / `invites` counts) and the
* readiness probe's queries both come through `select`; a count projection
* answers 1 and a row projection answers one row.
*/
function stubDb() {
return {
execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
select: vi.fn((projection?: Record<string, unknown>) => {
const rows = projection && "count" in projection ? [{ count: 1 }] : [{ companyId: "company-1" }];
const chain: Record<string, unknown> = {};
for (const method of ["from", "where", "innerJoin", "limit", "orderBy"]) {
chain[method] = vi.fn(() => chain);
}
chain.then = (resolve: (value: unknown) => unknown) => Promise.resolve(rows).then(resolve);
return chain;
}),
} as unknown as Db;
}
function createApp(actorType: "board" | "none") {
const app = express();
app.use((req, _res, next) => {
(req as express.Request & { actor: { type: string } }).actor = { type: actorType };
next();
});
app.use(
"/api/health",
healthRoutes(stubDb(), {
deploymentMode: "authenticated",
deploymentExposure: "private",
authReady: true,
companyDeletionEnabled: true,
}),
);
return app;
}
afterEach(() => {
for (const [key, value] of previousEnv) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
previousEnv.clear();
for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
vi.restoreAllMocks();
});
describe("GET /api/health workspace readiness", () => {
it("exposes readiness to a board actor of a managed workspace", async () => {
setEnv({
PAPERCLIP_CONFIG: createSeededWorkspace(),
[WORKSPACE_HANDOFF_KEY_ENV_KEY]: "handoff-key",
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: "ews-1",
[WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY]: "company-1",
});
const response = await request(createApp("board")).get("/api/health").expect(200);
expect(response.body.workspace).toMatchObject({
state: "ready",
databaseReady: true,
cloneDataReady: true,
authHandoffReady: true,
seedState: "verified",
executionWorkspaceId: "ews-1",
companyId: "company-1",
});
});
it("keeps public health redacted for an anonymous caller", async () => {
setEnv({
PAPERCLIP_CONFIG: createSeededWorkspace(),
[WORKSPACE_HANDOFF_KEY_ENV_KEY]: "handoff-key",
[WORKSPACE_READINESS_TOKEN_ENV_KEY]: "probe-token",
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: "ews-1",
[WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY]: "company-1",
});
const response = await request(createApp("none")).get("/api/health").expect(200);
expect(response.body.status).toBe("ok");
expect(response.body.workspace).toBeUndefined();
expect(response.body.serverInfo).toBeUndefined();
});
it("accepts the derived probe token on an otherwise redacted response", async () => {
setEnv({
PAPERCLIP_CONFIG: createSeededWorkspace(),
[WORKSPACE_HANDOFF_KEY_ENV_KEY]: "handoff-key",
[WORKSPACE_READINESS_TOKEN_ENV_KEY]: "probe-token",
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: "ews-1",
[WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY]: "company-1",
});
const response = await request(createApp("none"))
.get("/api/health")
.set(WORKSPACE_READINESS_TOKEN_HEADER, "probe-token")
.set(WORKSPACE_READINESS_USER_ID_HEADER, "user-1")
.set(WORKSPACE_READINESS_USER_EMAIL_HEADER, "operator@example.com")
.expect(200);
expect(response.body.workspace).toMatchObject({
state: "ready",
instanceId: expect.any(String),
authHandoffUserId: "user-1",
});
// Still redacted apart from readiness: the token buys one contract, not full detail.
expect(response.body.serverInfo).toBeUndefined();
});
it("rejects a wrong probe token without leaking readiness", async () => {
setEnv({
PAPERCLIP_CONFIG: createSeededWorkspace(),
[WORKSPACE_HANDOFF_KEY_ENV_KEY]: "handoff-key",
[WORKSPACE_READINESS_TOKEN_ENV_KEY]: "probe-token",
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: "ews-1",
[WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY]: "company-1",
});
const response = await request(createApp("none"))
.get("/api/health")
.set(WORKSPACE_READINESS_TOKEN_HEADER, "probe-token-but-wrong")
.expect(200);
expect(response.body.workspace).toBeUndefined();
});
it("omits readiness entirely on an instance that is not a cloned workspace", async () => {
const dir = mkdtempSync(path.join(os.tmpdir(), "paperclip-health-primary-"));
tempDirs.push(dir);
writeFileSync(path.join(dir, "config.json"), "{}\n", "utf8");
setEnv({ PAPERCLIP_CONFIG: path.join(dir, "config.json") });
delete process.env[WORKSPACE_HANDOFF_KEY_ENV_KEY];
delete process.env[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY];
const response = await request(createApp("board")).get("/api/health").expect(200);
expect(response.body.workspace).toBeUndefined();
});
});
@@ -72,6 +72,7 @@ import { instanceSettingsService } from "../services/instance-settings.ts";
import { issueService } from "../services/issues.ts";
import { runningProcesses } from "../adapters/index.ts";
import { DEFAULT_LIVENESS_REESCALATION_COOLDOWN_MS } from "../services/recovery/service.ts";
import { buildIssueBlockersResolvedWakeStateKey } from "../services/issue-dependency-wakeups.ts";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
@@ -484,7 +485,12 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
.then((rows) => rows[0] ?? null);
expect(wake?.reason).toBe("issue_blockers_resolved");
expect(wake?.idempotencyKey).toBe(`issue_blockers_resolved:${blockedIssueId}:${blockerIssueId}`);
expect(wake?.idempotencyKey).toBe(
buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: blockedIssueId,
blockerIssueIds: [blockerIssueId],
}),
);
expect(["queued", "claimed", "completed"]).toContain(wake?.status);
const events = await db
@@ -522,7 +528,12 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
.then((rows) => rows[0] ?? null);
expect(wake?.reason).toBe("issue_blockers_resolved");
expect(wake?.idempotencyKey).toBe(`issue_blockers_resolved:${blockedIssueId}:${blockerIssueId}`);
expect(wake?.idempotencyKey).toBe(
buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: blockedIssueId,
blockerIssueIds: [blockerIssueId],
}),
);
expect(["queued", "claimed", "completed"]).toContain(wake?.status);
const events = await db
@@ -564,14 +575,23 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
.then((rows) => rows[0] ?? null);
expect(wake).toMatchObject({
reason: "issue_blockers_resolved",
idempotencyKey: `issue_blockers_resolved:${blockedIssueId}:${blockerIssueId}`,
idempotencyKey: buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: blockedIssueId,
blockerIssueIds: [blockerIssueId],
}),
});
});
it("retries a resolved dependency wake when the prior wake was skipped as stale", async () => {
const { companyId, agentId, blockedIssueId, blockerIssueId } =
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
const idempotencyKey = `issue_blockers_resolved:${blockedIssueId}:${blockerIssueId}`;
// The route-time wake writes the level-triggered state key. A skip records a
// `skipped` row with that key. `skipped` is not an in-flight status, so the
// backstop must still re-emit for the same ready state.
const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: blockedIssueId,
blockerIssueIds: [blockerIssueId],
});
await db.insert(agentWakeupRequests).values({
companyId,
agentId,
@@ -653,7 +673,10 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
.then((rows) => rows[0] ?? null);
expect(wake).toMatchObject({
reason: "issue_blockers_resolved",
idempotencyKey: `issue_blockers_resolved:${blockedIssueId}:${blockerIssueId}`,
idempotencyKey: buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: blockedIssueId,
blockerIssueIds: [blockerIssueId],
}),
});
});
@@ -716,6 +739,79 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
);
});
it("heals a multi-blocker dependent when only a completed wake for an earlier blocker exists", async () => {
await enableAutoRecovery();
const { companyId, agentId, blockedIssueId, blockerIssueId } =
await seedResolvedDependencyBackstopFixture({ workspaceState: "none" });
const secondBlockerIssueId = randomUUID();
await db.insert(issues).values({
id: secondBlockerIssueId,
companyId,
title: "Earlier completed blocker",
status: "done",
priority: "medium",
issueNumber: 3,
identifier: "R-MULTI-3",
});
await db.insert(issueRelations).values({
companyId,
issueId: secondBlockerIssueId,
relatedIssueId: blockedIssueId,
type: "blocks",
});
// An earlier partial resolution left a `completed` per-edge wake. The bug was
// that this stale wake suppressed the wake for the current ready state. The
// level-triggered dedup keys on the full blocker set, so this completed wake
// no longer strands the dependent.
await db.insert(agentWakeupRequests).values({
companyId,
agentId,
source: "automation",
triggerDetail: "system",
reason: "issue_blockers_resolved",
payload: {
issueId: blockedIssueId,
resolvedBlockerIssueId: secondBlockerIssueId,
},
status: "completed",
finishedAt: new Date(),
idempotencyKey: `issue_blockers_resolved:${blockedIssueId}:${secondBlockerIssueId}`,
});
const readiness = await issueService(db).getDependencyReadiness(blockedIssueId);
expect(readiness.isDependencyReady).toBe(true);
const result = await heartbeatService(db).reconcileIssueGraphLiveness();
expect(result.dependencyWakesHealed).toBe(1);
expect(result.dependencyWakeIssueIds).toEqual([blockedIssueId]);
expect(result.dependencyWakeExistingSkipped).toBe(0);
const stateKey = buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: blockedIssueId,
blockerIssueIds: readiness.blockerIssueIds,
});
const healedWake = await db
.select({ status: agentWakeupRequests.status, idempotencyKey: agentWakeupRequests.idempotencyKey })
.from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.idempotencyKey, stateKey)))
.then((rows) => rows[0] ?? null);
expect(healedWake).not.toBeNull();
expect(["queued", "claimed", "completed"]).toContain(healedWake?.status);
// A second reconciliation pass finds the state-key wake and stays bounded:
// it heals nothing more and never enqueues a second wake for the same state.
const secondPass = await heartbeatService(db).reconcileIssueGraphLiveness();
expect(secondPass.dependencyWakesHealed).toBe(0);
const stateKeyWakes = await db
.select({ id: agentWakeupRequests.id })
.from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.idempotencyKey, stateKey)));
expect(stateKeyWakes).toHaveLength(1);
});
it("counts null dependency wake returns as deferred instead of enqueue failures", async () => {
await enableAutoRecovery();
const { companyId, agentId } =
@@ -1019,17 +1019,18 @@ describe("buildReferencedProjectRunObservability", () => {
failures: [
{ projectId: "project-b", reason: "authorization" },
{ projectId: "project-c", reason: "resolution" },
{ projectId: "project-d", reason: "staging" },
{ projectId: "project-d", reason: "staging", error: "extract failed: boom" },
],
});
// Requested is the synced count plus every dropped project, so the counts reconcile.
expect(observability.referenced_projects_requested).toBe(4);
expect(observability.referenced_projects_synced).toBe(1);
// A staging failure carries its error message; a failure without one omits the field.
expect(observability.referenced_project_failures).toEqual([
{ project_id: "project-b", reason: "authorization" },
{ project_id: "project-c", reason: "resolution" },
{ project_id: "project-d", reason: "staging" },
{ project_id: "project-d", reason: "staging", error: "extract failed: boom" },
]);
});
@@ -3,7 +3,7 @@ import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mockWakeup = vi.hoisted(() => vi.fn(async () => undefined));
const mockFindExistingIssueBlockersResolvedWake = vi.hoisted(() => vi.fn(async () => null));
const mockFindExistingIssueBlockersResolvedWakeForReadyState = vi.hoisted(() => vi.fn(async () => null));
const mockIssueService = vi.hoisted(() => ({
getAncestors: vi.fn(),
getById: vi.fn(),
@@ -98,7 +98,8 @@ vi.mock("../services/issue-dependency-wakeups.js", async () => {
);
return {
...actual,
findExistingIssueBlockersResolvedWake: mockFindExistingIssueBlockersResolvedWake,
findExistingIssueBlockersResolvedWakeForReadyState:
mockFindExistingIssueBlockersResolvedWakeForReadyState,
};
});
@@ -145,7 +146,7 @@ describe("issue dependency wakeups in issue routes", () => {
vi.doUnmock("../routes/authz.js");
vi.doUnmock("../middleware/index.js");
vi.clearAllMocks();
mockFindExistingIssueBlockersResolvedWake.mockResolvedValue(null);
mockFindExistingIssueBlockersResolvedWakeForReadyState.mockResolvedValue(null);
mockIssueService.getAncestors.mockResolvedValue([]);
mockIssueService.getByIdForUpdate.mockImplementation(async () => mockIssueService.getById());
mockIssueService.getComment.mockResolvedValue(null);
@@ -0,0 +1,155 @@
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
resetRuntimeServicesForTests,
startRuntimeServicesForWorkspaceControl,
} from "../services/workspace-runtime.js";
import {
doesLocalServiceCommandLineMatch,
listLocalServiceRegistryRecords,
readLocalServicePortOwner,
resolveLocalServiceLogPath,
terminateLocalService,
} from "../services/local-service-supervisor.js";
describe("local service supervision", () => {
afterEach(async () => {
await resetRuntimeServicesForTests({ terminateProcesses: true });
});
it("keeps request-logging runtime stdio usable after the supervisor side closes", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-service-stdio-"));
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-service-home-"));
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
const previousInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = `service-stdio-${randomUUID()}`;
let registryRecord: Awaited<ReturnType<typeof listLocalServiceRegistryRecords>>[number] | null = null;
try {
const [service] = await startRuntimeServicesForWorkspaceControl({
actor: { id: null, name: "Board", companyId: randomUUID() },
issue: null,
workspace: {
baseCwd: workspaceRoot,
source: "agent_home",
projectId: null,
workspaceId: null,
repoUrl: null,
repoRef: null,
strategy: "project_primary",
cwd: workspaceRoot,
branchName: null,
worktreePath: null,
warnings: [],
created: false,
},
config: {
workspaceRuntime: {
services: [{
name: "web",
command: "node -e \"const http=require('node:http'); process.on('SIGTERM',()=>{}); http.createServer((req,res)=>{ process.stdout.write('request '+req.url+'\\\\n',(error)=>{ if (!error) res.end('ok'); }); }).listen(Number(process.env.PORT), '127.0.0.1')\"",
port: { type: "auto" },
readiness: {
type: "http",
urlTemplate: "http://127.0.0.1:{{port}}",
timeoutSec: 10,
intervalMs: 100,
},
lifecycle: "shared",
reuseScope: "agent",
stopPolicy: { type: "manual" },
}],
},
},
adapterEnv: {},
});
expect(service?.url).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
expect(service?.port).toBeTypeOf("number");
registryRecord = (await listLocalServiceRegistryRecords({ profileKind: "workspace-runtime" }))
.find((record) => record.runtimeServiceId === service!.id) ?? null;
expect(registryRecord).not.toBeNull();
await resetRuntimeServicesForTests({ simulateSupervisorExit: true });
await expect(fetch(`${service!.url}/after-restart`)).resolves.toMatchObject({ ok: true });
const log = await fs.readFile(resolveLocalServiceLogPath(registryRecord!.serviceKey), "utf8");
expect(log).toContain("request /after-restart");
await terminateLocalService(registryRecord!);
expect(await readLocalServicePortOwner(service!.port!)).toBeNull();
await expect(fetch(service!.url!)).rejects.toThrow();
registryRecord = null;
} finally {
if (registryRecord) await terminateLocalService(registryRecord).catch(() => undefined);
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
if (previousInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
else process.env.PAPERCLIP_INSTANCE_ID = previousInstanceId;
await fs.rm(paperclipHome, { recursive: true, force: true });
await fs.rm(workspaceRoot, { recursive: true, force: true });
}
}, 15_000);
it("does not wait for an unrelated process that takes over the service port", async () => {
if (process.platform === "win32") return;
const unrelatedListener = net.createServer();
await new Promise<void>((resolve) => unrelatedListener.listen(0, "127.0.0.1", resolve));
const address = unrelatedListener.address();
const port = typeof address === "object" && address ? address.port : null;
if (!port) throw new Error("Failed to allocate unrelated listener port");
const child = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], {
detached: true,
stdio: "ignore",
});
child.unref();
try {
await terminateLocalService({
pid: child.pid!,
processGroupId: child.pid!,
port,
});
expect(await readLocalServicePortOwner(port)).toBe(process.pid);
} finally {
try {
process.kill(-child.pid!, "SIGKILL");
} catch {
// The target process group was already terminated.
}
await new Promise<void>((resolve, reject) => {
unrelatedListener.close((error) => error ? reject(error) : resolve());
});
}
});
it("recognizes a pnpm command after the launcher becomes pnpm.cjs", () => {
expect(doesLocalServiceCommandLineMatch({
commandLine: "/usr/bin/node /opt/pnpm/pnpm.cjs dev -- --bind custom --bind-host 127.0.0.1",
recordedCommand: "pnpm dev -- --bind custom --bind-host 127.0.0.1",
serviceName: "paperclip-dev",
})).toBe(true);
});
it("does not accept a different command merely because it uses node", () => {
expect(doesLocalServiceCommandLineMatch({
commandLine: "/usr/bin/node /workspace/server/dist/index.js",
recordedCommand: "pnpm dev -- --bind custom --bind-host 127.0.0.1",
serviceName: "paperclip-dev",
})).toBe(false);
});
it("does not collapse different script paths to the same basename", () => {
expect(doesLocalServiceCommandLineMatch({
commandLine: "/usr/bin/node /other-workspace/server.js",
recordedCommand: "node ./server.js",
serviceName: "web",
})).toBe(false);
});
});
@@ -311,4 +311,21 @@ describe("clampProviderSpanAttributes", () => {
// A non-finite number yields no attribute; `exec.command` is not allowed.
});
});
it("keeps the transfer direction for each closed-set value", () => {
expect(clampProviderSpanAttributes({ [A.transferDirection]: "inbound" })).toEqual({
[A.transferDirection]: "inbound",
});
expect(clampProviderSpanAttributes({ [A.transferDirection]: "outbound" })).toEqual({
[A.transferDirection]: "outbound",
});
});
it("drops a transfer direction outside the closed set or of the wrong type", () => {
// A free-form string, an empty string, and a non-string all yield no
// attribute, so the direction stays bounded and low-cardinality.
expect(clampProviderSpanAttributes({ [A.transferDirection]: "sideways" })).toEqual({});
expect(clampProviderSpanAttributes({ [A.transferDirection]: "" })).toEqual({});
expect(clampProviderSpanAttributes({ [A.transferDirection]: 1 })).toEqual({});
});
});
@@ -0,0 +1,505 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it, vi } from "vitest";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js";
const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures");
const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join(
FIXTURES_DIR,
"plugin-worker-duplex-channel.cjs",
);
const TEST_MANIFEST: PaperclipPluginManifestV1 = {
id: "test.plugin",
apiVersion: 1,
version: "1.0.0",
displayName: "Test plugin",
description: "Test plugin",
author: "Paperclip",
categories: ["automation"],
capabilities: [],
entrypoints: { worker: "dist/worker.js" },
};
function makeDuplexHandle(extra?: Record<string, unknown>) {
return createPluginWorkerHandle("test.plugin", {
entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT,
manifest: TEST_MANIFEST,
config: {},
instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" },
apiVersion: 1,
hostHandlers: {},
...extra,
});
}
// The test directive rides in `providerLeaseId`, an opaque field the manager
// forwards to the worker unchanged. The duplex channel is generic, so the
// command is a plain fixed string with no allowlist.
function duplexOpenInput(directive: unknown) {
return {
driverKey: "daytona",
companyId: "company-1",
environmentId: "env-1",
providerLeaseId: JSON.stringify(directive),
command: "bridge-callback",
};
}
describe("plugin worker manager duplex channel route", () => {
it("delivers data only for the exact bound worker session id and drops a mismatch", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
workerSessionId: "ws-A",
data: [
{ chunk: "good-1" },
{ chunk: "forged", sid: "ws-EVIL" },
{ chunk: "good-2" },
],
exitCode: 0,
}),
);
const chunks: string[] = [];
session.onData((chunk) => chunks.push(chunk));
await expect(session.wait()).resolves.toEqual({ exitCode: 0 });
// The forged notification carries a wrong worker session id, so the host
// drops it. Only the two bound chunks reach the listener, in order.
expect(chunks).toEqual(["good-1", "good-2"]);
await session.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
it("routes input to the worker and back to the listener", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({ workerSessionId: "ws-A", echoInput: true }),
);
const chunks: string[] = [];
session.onData((chunk) => chunks.push(chunk));
session.write("callback-payload");
// The worker echoes the input as one data notification for the bound
// session, so the listener receives it.
await vi.waitFor(() => expect(chunks).toContain("echo:callback-payload"));
await session.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
it("buffers early data in order until a listener attaches and drains it in order", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
data: [{ chunk: "one" }, { chunk: "two" }, { chunk: "three" }],
}),
);
// Wait so the three data notifications arrive and buffer before a listener
// attaches. The drain then delivers them in order.
await new Promise((resolve) => setTimeout(resolve, 60));
const chunks: string[] = [];
session.onData((chunk) => chunks.push(chunk));
await vi.waitFor(() => expect(chunks.length).toBe(3));
expect(chunks).toEqual(["one", "two", "three"]);
await session.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
it("isolates a throwing listener during live delivery so later chunks still route", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
workerSessionId: "ws-A",
data: [{ chunk: "ok-1" }, { chunk: "boom" }, { chunk: "ok-2" }],
exitCode: 0,
}),
);
const chunks: string[] = [];
// The listener throws on one chunk. The manager catches the throw, so it
// does not escape the worker stdout notification handler. The later chunk
// still routes and the route still settles.
session.onData((chunk) => {
chunks.push(chunk);
if (chunk === "boom") throw new Error("listener failure");
});
await expect(session.wait()).resolves.toEqual({ exitCode: 0 });
expect(chunks).toEqual(["ok-1", "boom", "ok-2"]);
await session.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
it("isolates a throwing listener during the buffered replay so every buffered chunk routes", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
data: [{ chunk: "one" }, { chunk: "boom" }, { chunk: "three" }],
}),
);
// Wait so the three data notifications arrive and buffer before a listener
// attaches. The drain then delivers them in order.
await new Promise((resolve) => setTimeout(resolve, 60));
const chunks: string[] = [];
// The listener throws on one buffered chunk. The manager catches the throw
// inside the drain, so it does not escape `onData` and every buffered chunk
// still routes.
expect(() =>
session.onData((chunk) => {
chunks.push(chunk);
if (chunk === "boom") throw new Error("listener failure");
}),
).not.toThrow();
expect(chunks).toEqual(["one", "boom", "three"]);
await session.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
it("binds the worker session id one time and ignores a duplicate open reply", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
mode: "duplicate-open-reply",
workerSessionId: "ws-A",
data: [{ chunk: "hello" }],
exitCode: 0,
}),
);
const chunks: string[] = [];
session.onData((chunk) => chunks.push(chunk));
// The duplicate open reply never rebinds or reopens the route, so the
// session runs normally on the one bind.
await expect(session.wait()).resolves.toEqual({ exitCode: 0 });
expect(chunks).toEqual(["hello"]);
await session.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
it("terminalizes and fails closed on a malformed open reply, then admits a later open", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
await expect(
handle.openDuplexChannel(duplexOpenInput({ mode: "malformed-open" })),
).rejects.toThrow("DUPLEX_CHANNEL_OPEN_FAILED");
// The terminalize closed the route by the host route id and the worker
// acknowledged the close, so a later open is admitted.
const session = await handle.openDuplexChannel(duplexOpenInput({ mode: "normal" }));
expect(session).toBeDefined();
await session.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
it("permits one active duplex channel per worker", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const first = await handle.openDuplexChannel(duplexOpenInput({ mode: "normal" }));
// A second open while the first route is not closed rejects with one fixed
// non-secret error before it reaches the worker.
await expect(
handle.openDuplexChannel(duplexOpenInput({ mode: "normal" })),
).rejects.toThrow("DUPLEX_CHANNEL_ROUTE_BUSY");
await first.close();
// After the first route closes and the worker acknowledges the close, a new
// open is admitted.
const second = await handle.openDuplexChannel(duplexOpenInput({ mode: "normal" }));
await second.close();
} finally {
await handle.stop().catch(() => undefined);
}
});
// -------------------------------------------------------------------------
// The five explicit bounds. Each bound ends the route when it is exceeded.
// -------------------------------------------------------------------------
it("ends the route when the pre-bind buffered bytes pass the bound", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxPreBindBufferedChars: 10 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
data: [
{ chunk: "aaaaa" }, // total 5 → buffered
{ chunk: "bbbbb" }, // total 10 → buffered
{ chunk: "ccccc" }, // total 15 > 10 → end route
],
}),
);
// No listener attaches, so the data buffers. The cumulative bytes pass the
// bound and the route ends. The login wait resolves with a null exit code.
await expect(session.wait()).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the route when the pre-bind buffered frame count passes the bound", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxPreBindBufferedFrames: 2 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
data: [{ chunk: "a" }, { chunk: "b" }, { chunk: "c" }],
}),
);
// No listener attaches, so the data buffers. The third frame passes the
// frame-count bound and the route ends.
await expect(session.wait()).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the route when the pending request count passes the bound", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxPendingRequests: 2 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({ mode: "no-write-reply", workerSessionId: "ws-A" }),
);
const waitResult = session.wait();
// The worker never replies to a write, so each write stays pending. The
// third write passes the pending-request bound and the route ends.
session.write("one");
session.write("two");
session.write("three");
await expect(waitResult).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the route when one host-to-worker write passes the size bound", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxWriteChars: 8 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({ workerSessionId: "ws-A" }),
);
const waitResult = session.wait();
// One write is larger than the size bound, so the host rejects it and ends
// the route before it reaches the worker.
session.write("this-write-is-too-large");
await expect(waitResult).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the route when the protocol error count passes the bound", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxProtocolErrors: 2 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
workerSessionId: "ws-A",
data: [
{ chunk: "e1", sid: "ws-EVIL" },
{ chunk: "e2", sid: "ws-EVIL" },
{ chunk: "e3", sid: "ws-EVIL" },
],
}),
);
// Each mismatched-session data frame is a protocol error. The third frame
// passes the error bound and the route ends.
await expect(session.wait()).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the route when the total data bytes pass the cap for a bound listener", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxTotalDataBytes: 10 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
workerSessionId: "ws-A",
data: [
{ chunk: "aaaaa" }, // total 5 → deliver
{ chunk: "bbbbb" }, // total 10 → deliver
{ chunk: "ccccc" }, // total 15 > 10 → end route
],
}),
);
const chunks: string[] = [];
session.onData((chunk) => chunks.push(chunk));
// A listener is bound, so the host forwards each chunk until the cumulative
// bytes pass the cap. The third chunk passes the cap, so the host drops it
// and ends the route. The listener never receives data past the cap.
await expect(session.wait()).resolves.toEqual({ exitCode: null });
expect(chunks).toEqual(["aaaaa", "bbbbb"]);
} finally {
await handle.stop().catch(() => undefined);
}
});
it("counts inbound bytes, not characters, against the total cap", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxTotalDataBytes: 4 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
workerSessionId: "ws-A",
// "€" is one character but three bytes in UTF-8. The first chunk is 3
// bytes (≤ 4), so the host delivers it. The second chunk brings the
// total to 6 bytes (> 4), so the host ends the route. A character count
// would admit both chunks (2 ≤ 4), so one delivered chunk proves the
// host counts bytes.
data: [{ chunk: "€" }, { chunk: "€" }],
}),
);
const chunks: string[] = [];
session.onData((chunk) => chunks.push(chunk));
await expect(session.wait()).resolves.toEqual({ exitCode: null });
expect(chunks).toEqual(["€"]);
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the active route when the lifetime timer expires", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxDurationMs: 100 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({ mode: "normal", workerSessionId: "ws-A" }),
);
const waitResult = session.wait();
// The route sends no exit. The lifetime timer expires, so the host ends the
// route and resolves the wait with the fixed null exit code.
await expect(waitResult).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the route at once when one inbound chunk passes the per-chunk limit before a listener binds", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxChunkChars: 4 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
workerSessionId: "ws-A",
data: [{ chunk: "this-one-chunk-is-too-large" }],
}),
);
// No listener attaches. One inbound chunk is larger than the per-chunk
// limit, so the host ends the route at once. The default protocol-error
// budget is far above one, so a single chunk that ends the route proves the
// host does not treat it as a protocol error.
await expect(session.wait()).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("ends the route at once when one inbound chunk passes the per-chunk limit after a listener binds", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { maxChunkChars: 4 },
});
try {
await handle.start();
const session = await handle.openDuplexChannel(
duplexOpenInput({
workerSessionId: "ws-A",
data: [{ chunk: "this-one-chunk-is-too-large" }],
}),
);
const chunks: string[] = [];
session.onData((chunk) => chunks.push(chunk));
// A listener is bound. One inbound chunk is larger than the per-chunk
// limit, so the host ends the route at once and never forwards the chunk.
await expect(session.wait()).resolves.toEqual({ exitCode: null });
expect(chunks).toEqual([]);
} finally {
await handle.stop().catch(() => undefined);
}
});
// -------------------------------------------------------------------------
// Authoritative closure and worker retirement.
// -------------------------------------------------------------------------
it("closes the route with a fixed exit when the worker exits", async () => {
const handle = makeDuplexHandle();
try {
await handle.start();
const session = await handle.openDuplexChannel(duplexOpenInput({ mode: "normal" }));
const waitResult = session.wait();
await handle.stop();
// A worker exit closes the one route and resolves the wait with the fixed
// non-secret exit.
await expect(waitResult).resolves.toEqual({ exitCode: null });
} finally {
await handle.stop().catch(() => undefined);
}
});
it("retires the worker on an unconfirmed close acknowledgement", async () => {
const handle = makeDuplexHandle({
duplexChannelLimits: { closeTimeoutMs: 200 },
});
try {
await handle.start();
const exited = new Promise<void>((resolve) => {
handle.on("exit", () => resolve());
});
const session = await handle.openDuplexChannel(
duplexOpenInput({ mode: "normal", closeMode: "bad-ack" }),
);
await session.close();
// The close acknowledgement carried a mismatched host route id, so the host
// fails closed and retires the worker before any reuse.
await exited;
await expect(
handle.openDuplexChannel(duplexOpenInput({ mode: "normal" })),
).rejects.toThrow();
} finally {
await handle.stop().catch(() => undefined);
}
});
});
@@ -239,6 +239,136 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => {
.then((rows) => rows[0]!);
}
it("clears transient routine run failures when execution issues resume", async () => {
const { companyId, issueSvc, routine, svc } = await seedFixture();
const runId = randomUUID();
const executionIssue = await issueSvc.create(companyId, {
projectId: routine.projectId,
title: routine.title,
description: routine.description,
status: "blocked",
priority: routine.priority,
assigneeAgentId: routine.assigneeAgentId,
originKind: "routine_execution",
originId: routine.id,
originRunId: runId,
});
await db.insert(routineRuns).values({
id: runId,
companyId,
routineId: routine.id,
source: "manual",
status: "issue_created",
triggeredAt: new Date("2026-07-16T12:00:00.000Z"),
linkedIssueId: executionIssue.id,
});
await svc.syncRunStatusForIssue(executionIssue.id);
const [failedRun] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId));
expect(failedRun).toMatchObject({
status: "failed",
failureReason: "Execution issue moved to blocked",
triggerPayload: {
transientFailure: {
code: "execution_issue_status",
status: "blocked",
},
},
});
await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, executionIssue.id));
await svc.syncRunStatusForIssue(executionIssue.id);
const [run] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId));
expect(run).toMatchObject({
status: "issue_created",
failureReason: null,
completedAt: null,
triggerPayload: {
transientFailure: {
code: "execution_issue_status",
status: "blocked",
clearedAt: expect.any(String),
},
},
});
const clearedAt = (run?.triggerPayload as { transientFailure?: { clearedAt?: string } } | null)
?.transientFailure?.clearedAt;
expect(clearedAt).toEqual(expect.any(String));
await db.update(issues).set({ status: "done" }).where(eq(issues.id, executionIssue.id));
await svc.syncRunStatusForIssue(executionIssue.id);
const [completedRun] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId));
expect(completedRun).toMatchObject({
status: "completed",
failureReason: null,
triggerPayload: {
transientFailure: {
code: "execution_issue_status",
status: "blocked",
clearedAt,
},
},
});
expect(completedRun?.completedAt).toBeInstanceOf(Date);
});
it("moves transient routine run failures into completion context", async () => {
const { companyId, issueSvc, routine, svc } = await seedFixture();
const runId = randomUUID();
const executionIssue = await issueSvc.create(companyId, {
projectId: routine.projectId,
title: routine.title,
description: routine.description,
status: "blocked",
priority: routine.priority,
assigneeAgentId: routine.assigneeAgentId,
originKind: "routine_execution",
originId: routine.id,
originRunId: runId,
});
await db.insert(routineRuns).values({
id: runId,
companyId,
routineId: routine.id,
source: "manual",
status: "issue_created",
triggeredAt: new Date("2026-07-16T12:00:00.000Z"),
linkedIssueId: executionIssue.id,
triggerPayload: { input: "preserved" },
});
await svc.syncRunStatusForIssue(executionIssue.id);
const [failedRun] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId));
expect(failedRun).toMatchObject({
status: "failed",
failureReason: "Execution issue moved to blocked",
});
await db.update(issues).set({ status: "done" }).where(eq(issues.id, executionIssue.id));
await svc.syncRunStatusForIssue(executionIssue.id);
const [run] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId));
expect(run).toMatchObject({
status: "completed",
failureReason: null,
triggerPayload: {
input: "preserved",
transientFailure: {
code: "execution_issue_status",
status: "blocked",
reason: "Execution issue moved to blocked",
},
},
});
expect(run?.completedAt).toBeInstanceOf(Date);
expect(run?.triggerPayload).toMatchObject({
transientFailure: { clearedAt: expect.any(String) },
});
});
it("filters listed routines by project", async () => {
const { companyId, agentId, projectId, routine, svc } = await seedFixture();
const otherProjectId = randomUUID();
@@ -290,6 +290,82 @@ describe("sandbox capability contract normalizer", () => {
expect(effective.incrementalSessionOutput).toBe(false);
});
it("test_concurrent_sync_operations_is_opt_in_and_needs_both_sync_verbs", () => {
// Parallel bidirectional file sync is opt-in and direction-neutral. It needs
// both sync verbs, so a provider that verifies only one direction cannot get
// the capability. An absent declaration denies it even with both verbs.
const undeclared = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["environmentSyncIn", "environmentSyncOut"],
declared: null,
});
expect(undeclared.concurrentSyncOperations).toBe(false);
// A positive declaration with both verified verbs resolves true.
const bothVerbs = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["environmentSyncIn", "environmentSyncOut"],
declared: { concurrentSyncOperations: true },
});
expect(bothVerbs.concurrentSyncOperations).toBe(true);
// Only the inbound verb: the outbound prerequisite is missing, so it resolves
// false.
const inOnly = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["environmentSyncIn"],
declared: { concurrentSyncOperations: true },
});
expect(inOnly.concurrentSyncOperations).toBe(false);
// Only the outbound verb: the inbound prerequisite is missing, so it resolves
// false.
const outOnly = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["environmentSyncOut"],
declared: { concurrentSyncOperations: true },
});
expect(outOnly.concurrentSyncOperations).toBe(false);
});
it("test_duplex_command_stream_absent_declaration_resolves_false", () => {
// The duplex channel is opt-in and fail-closed. An absent declaration denies
// the capability even when the worker verifies the duplex open verb. This
// matches the incremental-session-output pattern: an opt-in behavioral
// guarantee needs a positive declaration, not just a verified verb.
const undeclared = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["duplexChannelOpen"],
declared: null,
});
expect(undeclared.duplexCommandStream).toBe(false);
});
it("test_duplex_command_stream_needs_verified_worker_method", () => {
// A declaration never grants the capability without the verified duplex open
// verb. A provider that declares the capability but whose worker does not
// report the duplex open method resolves false.
const declaredButUnverified = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["environmentExecute"],
declared: { duplexCommandStream: true },
});
expect(declaredButUnverified.duplexCommandStream).toBe(false);
});
it("test_duplex_command_stream_declared_and_verified_resolves_true_but_narrowing_removes_it", () => {
// A provider that declares the capability and whose worker verifies the
// duplex open verb gets the capability.
const granted = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["duplexChannelOpen"],
declared: { duplexCommandStream: true },
});
expect(granted.duplexCommandStream).toBe(true);
// Per-target narrowing still removes a verified and declared capability, so a
// lease that cannot use the duplex channel keeps the file bridge.
const narrowed = resolveEffectiveSandboxCapabilities({
verifiedMethods: ["duplexChannelOpen"],
declared: { duplexCommandStream: true },
narrowing: { duplexCommandStream: false },
});
expect(narrowed.duplexCommandStream).toBe(false);
});
it("test_unknown_or_unavailable_verification_resolves_false", () => {
const declaredAll = {
reusableLeases: true,
@@ -298,6 +374,7 @@ describe("sandbox capability contract normalizer", () => {
persistentProcessSessions: true,
independentControlCommands: true,
incrementalSessionOutput: true,
concurrentSyncOperations: true,
};
for (const verifiedMethods of [null, undefined, [] as string[]]) {
@@ -0,0 +1,78 @@
import { describe, expect, it } from "vitest";
import type { IssueWorkProduct } from "@paperclipai/shared";
import { reconcileRuntimeServiceWorkProducts } from "../services/work-products.js";
function workProduct(overrides: Partial<IssueWorkProduct> = {}): IssueWorkProduct {
return {
id: "wp-1",
companyId: "company-1",
projectId: "project-1",
issueId: "issue-1",
executionWorkspaceId: "ews-1",
runtimeServiceId: "runtime-1",
type: "runtime_service",
provider: "paperclip",
externalId: null,
title: "Workspace preview",
url: "https://workspace.example.ts.net:42013/",
status: "open",
reviewState: "none",
isPrimary: true,
healthStatus: "healthy",
summary: null,
metadata: null,
sourceTrust: null,
createdByRunId: null,
createdAt: new Date("2026-08-19T00:00:00.000Z"),
updatedAt: new Date("2026-08-19T00:00:00.000Z"),
...overrides,
};
}
describe("reconcileRuntimeServiceWorkProducts", () => {
it("adopts the live runtime URL after a port change", () => {
const [reconciled] = reconcileRuntimeServiceWorkProducts(
[workProduct()],
[{ id: "runtime-1", url: "https://workspace.example.ts.net:42055/", status: "running", healthStatus: "healthy" }],
);
expect(reconciled!.url).toBe("https://workspace.example.ts.net:42055/");
expect(reconciled!.healthStatus).toBe("healthy");
});
it("marks a stopped or unhealthy runtime unhealthy instead of advertising it", () => {
const [stopped] = reconcileRuntimeServiceWorkProducts(
[workProduct()],
[{ id: "runtime-1", url: "https://workspace.example.ts.net:42013/", status: "stopped", healthStatus: "healthy" }],
);
expect(stopped!.healthStatus).toBe("unhealthy");
const [unhealthy] = reconcileRuntimeServiceWorkProducts(
[workProduct()],
[{ id: "runtime-1", url: "https://workspace.example.ts.net:42013/", status: "running", healthStatus: "unhealthy" }],
);
expect(unhealthy!.healthStatus).toBe("unhealthy");
});
it("keeps the recorded URL when the runtime row is gone, but stops calling it healthy", () => {
const [reconciled] = reconcileRuntimeServiceWorkProducts([workProduct()], []);
expect(reconciled!.url).toBe("https://workspace.example.ts.net:42013/");
expect(reconciled!.healthStatus).toBe("unhealthy");
});
it("leaves non-runtime work products and unlinked rows untouched", () => {
const pullRequest = workProduct({ id: "wp-pr", type: "pull_request", runtimeServiceId: null });
const unlinked = workProduct({ id: "wp-unlinked", runtimeServiceId: null });
const reconciled = reconcileRuntimeServiceWorkProducts([pullRequest, unlinked], []);
expect(reconciled[0]).toBe(pullRequest);
expect(reconciled[1]).toBe(unlinked);
});
it("returns the same objects when nothing drifted", () => {
const product = workProduct();
const reconciled = reconcileRuntimeServiceWorkProducts(
[product],
[{ id: "runtime-1", url: product.url, status: "running", healthStatus: "healthy" }],
);
expect(reconciled[0]).toBe(product);
});
});
@@ -114,7 +114,7 @@ describe("worktree instance cleanup", () => {
await expect(fs.stat(instanceRoot)).rejects.toMatchObject({ code: "ENOENT" });
});
it("falls back to deterministic instance ownership when persisted root metadata is absent", async () => {
it("preserves a collision-resistant active instance when persisted ownership is absent", async () => {
const worktreesDir = await makeTempRoot("paperclip-managed-worktrees-");
const workspacePath = await makeTempRoot("paperclip-cleanup-workspace-");
const instanceId = deriveWorktreeInstanceId(workspacePath);
@@ -133,8 +133,9 @@ describe("worktree instance cleanup", () => {
worktreesDir,
});
expect(result).toMatchObject({ status: "removed", instanceRoot });
await expect(fs.stat(instanceRoot)).rejects.toMatchObject({ code: "ENOENT" });
expect(result).toMatchObject({ status: "refused", instanceRoot });
expect((result as { warning: string }).warning).toContain("no persisted instance root");
await expect(fs.readFile(path.join(instanceRoot, "marker"), "utf8")).resolves.toBe("remove me");
});
it("refuses and logs an instance pointer outside the managed worktree root", async () => {
@@ -0,0 +1,251 @@
/**
* HTTP-level coverage for the Better Auth exchange endpoint (PAP-17572).
*
* The security matrix itself lives in `workspace-login-handoff.test.ts`, which
* exercises the pure verifier and the dependency-injected exchange. This suite
* proves the wiring a browser actually meets: the plugin is registered on the
* Better Auth mount, a valid ticket answers with a redirect and a session cookie,
* a rejected ticket redirects to the labeled credential fallback without one, and
* the exchange never trusts a forwarded host header.
*/
import express from "express";
import request from "supertest";
import { betterAuth } from "better-auth";
import { memoryAdapter } from "better-auth/adapters/memory";
import { toNodeHandler } from "better-auth/node";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import {
deriveWorkspaceHandoffKey,
issueWorkspaceHandoffTicket,
WORKSPACE_HANDOFF_TICKET_QUERY_PARAM,
} from "../auth/workspace-login-handoff.js";
import { workspaceLoginHandoffPlugin } from "../auth/workspace-login-handoff-plugin.js";
const ROOT_SECRET = "endpoint-test-root-secret";
const INSTANCE_ID = "pap-17572-endpoint-1122334455";
const EXECUTION_WORKSPACE_ID = "ews-endpoint-1";
const COMPANY_ID = "company-endpoint-1";
const ORIGIN = "http://workspace.localhost:42013";
const USER_ID = "cloned-user-1";
const USER_EMAIL = "operator@example.com";
const KEY = deriveWorkspaceHandoffKey({
rootSecret: ROOT_SECRET,
instanceId: INSTANCE_ID,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
});
type MemoryStore = Record<string, Record<string, unknown>[]>;
/**
* Membership lookup goes through the app `db`, not the Better Auth adapter, and
* asks for existence — so the stub answers with rows, not a count.
*/
function stubDb(activeMembershipCount: number) {
const rows = Array.from({ length: Math.max(0, activeMembershipCount) }, (_, index) => ({
id: `membership-${index}`,
}));
return {
select: vi.fn(() => {
const chain: Record<string, unknown> = {};
for (const method of ["from", "where", "innerJoin", "limit", "orderBy"]) {
chain[method] = vi.fn(() => chain);
}
chain.then = (resolve: (value: unknown) => unknown) => Promise.resolve(rows).then(resolve);
return chain;
}),
} as unknown as Db;
}
function createApp(input: {
activeMembershipCount?: number;
expectedOrigin?: string | null;
expectedInstanceId?: string | null;
expectedExecutionWorkspaceId?: string | null;
expectedCompanyId?: string | null;
key?: string | null;
}) {
const store: MemoryStore = { user: [], session: [], account: [], verification: [] };
store.user!.push({
id: USER_ID,
email: USER_EMAIL,
name: "Operator",
emailVerified: true,
createdAt: new Date(),
updatedAt: new Date(),
});
const auth = betterAuth({
secret: "better-auth-secret-for-endpoint-tests",
baseURL: ORIGIN,
trustedOrigins: [ORIGIN],
database: memoryAdapter(store),
emailAndPassword: { enabled: true },
advanced: { useSecureCookies: false },
plugins: [
workspaceLoginHandoffPlugin({
db: stubDb(input.activeMembershipCount ?? 1),
resolveExpectedIdentity: () => ({
key: input.key === undefined ? KEY : input.key,
instanceId: input.expectedInstanceId === undefined ? INSTANCE_ID : input.expectedInstanceId,
executionWorkspaceId:
input.expectedExecutionWorkspaceId === undefined
? EXECUTION_WORKSPACE_ID
: input.expectedExecutionWorkspaceId,
companyId: input.expectedCompanyId === undefined ? COMPANY_ID : input.expectedCompanyId,
origin: input.expectedOrigin === undefined ? ORIGIN : input.expectedOrigin,
}),
}),
],
});
const app = express();
app.all("/api/auth/*splat", (req, res, next) => {
void Promise.resolve(toNodeHandler(auth)(req, res)).catch(next);
});
return { app, store };
}
function mintTicket(overrides: Partial<Parameters<typeof issueWorkspaceHandoffTicket>[0]> = {}) {
return issueWorkspaceHandoffTicket({
key: KEY,
userId: USER_ID,
email: USER_EMAIL,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
instanceId: INSTANCE_ID,
origin: ORIGIN,
issuerInstanceId: "primary",
...overrides,
}).ticket;
}
function exchange(app: express.Express, ticket: string | null) {
const path = `/api/auth/workspace-handoff/exchange${
ticket === null ? "" : `?${WORKSPACE_HANDOFF_TICKET_QUERY_PARAM}=${encodeURIComponent(ticket)}`
}`;
return request(app).get(path);
}
function sessionCookies(response: request.Response): string[] {
const raw = response.headers["set-cookie"];
const cookies = Array.isArray(raw) ? raw : raw ? [raw] : [];
return cookies.filter((cookie) => cookie.includes("session_token"));
}
beforeEach(() => {
vi.spyOn(console, "warn").mockImplementation(() => undefined);
});
afterEach(() => {
vi.restoreAllMocks();
});
describe("GET /api/auth/workspace-handoff/exchange", () => {
it("redirects to the requested board path and sets exactly one session cookie", async () => {
const { app, store } = createApp({});
const response = await exchange(app, mintTicket({ next: "/PAP/issues/PAP-17572" }));
expect(response.status).toBe(302);
expect(response.headers.location).toBe(`${ORIGIN}/PAP/issues/PAP-17572`);
// The ticket must not survive into the landing URL, the cache, or the referrer.
expect(response.headers.location).not.toContain(WORKSPACE_HANDOFF_TICKET_QUERY_PARAM);
expect(response.headers["cache-control"]).toContain("no-store");
expect(response.headers["referrer-policy"]).toBe("no-referrer");
expect(sessionCookies(response)).toHaveLength(1);
expect(store.session).toHaveLength(1);
expect(store.session![0]!.userId).toBe(USER_ID);
});
it("does not create a second session for a replayed ticket", async () => {
const { app, store } = createApp({});
const ticket = mintTicket();
expect((await exchange(app, ticket)).status).toBe(302);
const replay = await exchange(app, ticket);
expect(replay.status).toBe(302);
expect(replay.headers.location).toContain("workspaceHandoffError=replayed");
expect(sessionCookies(replay)).toHaveLength(0);
expect(store.session).toHaveLength(1);
});
it("sends a rejected ticket to the labeled credential fallback with no session", async () => {
const { app, store } = createApp({});
const expired = mintTicket({ now: new Date(Date.now() - 10 * 60_000), ttlSeconds: 30 });
const response = await exchange(app, expired);
expect(response.status).toBe(302);
const location = new URL(response.headers.location as string);
expect(location.pathname).toBe("/auth");
expect(location.searchParams.get("workspaceHandoffError")).toBe("expired");
expect(sessionCookies(response)).toHaveLength(0);
expect(store.session).toHaveLength(0);
});
it("rejects a ticket for a cloned user with no active membership", async () => {
const { app, store } = createApp({ activeMembershipCount: 0 });
const response = await exchange(app, mintTicket());
expect(response.headers.location).toContain("workspaceHandoffError=missing_membership");
expect(store.session).toHaveLength(0);
});
it("rejects a ticket naming a user that did not survive the clone", async () => {
const { app, store } = createApp({});
const response = await exchange(app, mintTicket({ userId: "user-that-was-not-cloned" }));
expect(response.headers.location).toContain("workspaceHandoffError=unknown_user");
expect(store.session).toHaveLength(0);
});
it("ignores forwarded host and proto headers when checking the audience", async () => {
const { app, store } = createApp({});
// A ticket minted for the attacker's host must fail even when the request
// arrives claiming to be that host.
const spoofedTicket = mintTicket({ origin: "https://attacker.example.com" });
const response = await exchange(app, spoofedTicket)
.set("host", "attacker.example.com")
.set("x-forwarded-host", "attacker.example.com")
.set("x-forwarded-proto", "https")
.set("x-forwarded-for", "203.0.113.7");
expect(response.headers.location).toContain("workspaceHandoffError=origin_mismatch");
expect(sessionCookies(response)).toHaveLength(0);
expect(store.session).toHaveLength(0);
});
it("rejects a ticket bound to another workspace, instance, or company on this host", async () => {
const { app } = createApp({});
const otherWorkspace = await exchange(app, mintTicket({ executionWorkspaceId: "ews-sibling" }));
expect(otherWorkspace.headers.location).toContain("workspaceHandoffError=workspace_mismatch");
const otherInstance = await exchange(app, mintTicket({ instanceId: "another-instance" }));
expect(otherInstance.headers.location).toContain("workspaceHandoffError=instance_mismatch");
const otherCompany = await exchange(app, mintTicket({ companyId: "company-sibling" }));
expect(otherCompany.headers.location).toContain("workspaceHandoffError=company_mismatch");
});
it("rejects a missing or garbage ticket", async () => {
const { app } = createApp({});
expect((await exchange(app, null)).headers.location).toContain("workspaceHandoffError=malformed");
expect((await exchange(app, "wh1.aaaa.bbbb")).headers.location).toContain(
"workspaceHandoffError=bad_signature",
);
});
it("answers 503 rather than a redirect when the guest has no verification key", async () => {
const { app } = createApp({ key: null });
const response = await exchange(app, mintTicket());
expect(response.status).toBe(503);
expect(sessionCookies(response)).toHaveLength(0);
});
it("fails closed when the guest cannot resolve its own origin", async () => {
const { app, store } = createApp({ expectedOrigin: null });
const response = await exchange(app, mintTicket());
expect(response.status).toBe(302);
expect(response.headers.location).toContain("workspaceHandoffError=origin_mismatch");
expect(store.session).toHaveLength(0);
});
});
@@ -0,0 +1,302 @@
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import {
issueWorkspaceLoginHandoff,
resolveWorkspaceHandoffBoardIdentity,
workspaceLoginHandoffFailureStatus,
} from "../services/workspace-login-handoff-issuer.js";
import { verifyWorkspaceHandoffTicket } from "../auth/workspace-login-handoff.js";
import { resolveManagedWorkspaceIdentity } from "../services/managed-workspace-identity.js";
import { WORKSPACE_HANDOFF_TICKET_QUERY_PARAM } from "../auth/workspace-login-handoff.js";
const INSTANCE_ID = "pap-17572-live-workspace-aa11bb22cc33";
const EXECUTION_WORKSPACE_ID = "ews-live-1";
const COMPANY_ID = "company-1";
const RUNTIME_URL = "https://workspace.example.ts.net:42013/";
const tempDirs: string[] = [];
let previousSecret: string | undefined;
function createWorkspaceCwd(instanceId: string | null) {
const cwd = mkdtempSync(path.join(os.tmpdir(), "paperclip-handoff-issuer-"));
tempDirs.push(cwd);
mkdirSync(path.join(cwd, ".paperclip"), { recursive: true });
if (instanceId) {
writeFileSync(
path.join(cwd, ".paperclip", ".env"),
`PAPERCLIP_HOME=/srv/home\nPAPERCLIP_INSTANCE_ID=${instanceId}\n`,
"utf8",
);
}
return cwd;
}
/**
* Minimal drizzle-shaped stub. `select({ id, url, updatedAt })` is the live
* runtime lookup; anything else is the instance-admin lookup.
*/
function stubDb(input: {
runtimeRows?: Array<{ id: string; url: string | null; updatedAt: Date }>;
adminRows?: Array<{ id: string; email: string | null; createdAt: Date }>;
}) {
return {
select: vi.fn((projection: Record<string, unknown>) => {
const rows = "url" in projection ? input.runtimeRows ?? [] : input.adminRows ?? [];
const chain: Record<string, unknown> = {};
for (const method of ["from", "where", "innerJoin", "orderBy", "limit"]) {
chain[method] = vi.fn(() => chain);
}
chain.then = (resolve: (value: unknown) => unknown) => Promise.resolve(rows).then(resolve);
return chain;
}),
} as unknown as Db;
}
function readyProbe() {
return vi.fn(async () => ({
ok: true as const,
readiness: {
state: "ready" as const,
databaseReady: true,
cloneDataReady: true,
authHandoffReady: true,
authHandoffUserId: "user-1",
seedState: "verified" as const,
seedPhase: "complete",
seedMode: "minimal" as const,
instanceId: INSTANCE_ID,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
failurePhase: null,
},
}));
}
beforeEach(() => {
previousSecret = process.env.PAPERCLIP_WORKSPACE_HANDOFF_SECRET;
process.env.PAPERCLIP_WORKSPACE_HANDOFF_SECRET = "issuer-test-root-secret";
});
afterEach(() => {
if (previousSecret === undefined) delete process.env.PAPERCLIP_WORKSPACE_HANDOFF_SECRET;
else process.env.PAPERCLIP_WORKSPACE_HANDOFF_SECRET = previousSecret;
for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
vi.restoreAllMocks();
});
describe("issueWorkspaceLoginHandoff", () => {
it("mints a ticket the target workspace accepts, bound to the live runtime origin", async () => {
const cwd = createWorkspaceCwd(INSTANCE_ID);
const probe = readyProbe();
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ runtimeRows: [{ id: "runtime-1", url: RUNTIME_URL, updatedAt: new Date() }] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd },
actor: { userId: "user-1", userEmail: "operator@example.com", source: "session" },
next: "/PAP/issues/PAP-17572",
probe,
});
expect(result.ok).toBe(true);
if (!result.ok) return;
const url = new URL(result.issuance.url);
expect(url.origin).toBe("https://workspace.example.ts.net:42013");
expect(url.pathname).toBe("/api/auth/workspace-handoff/exchange");
// The guest independently derives the same key, so the round trip proves the
// control plane and the workspace agree without sharing a stored token.
const identity = resolveManagedWorkspaceIdentity({
workspaceCwd: cwd,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
});
const verification = verifyWorkspaceHandoffTicket({
ticket: url.searchParams.get(WORKSPACE_HANDOFF_TICKET_QUERY_PARAM),
key: identity!.handoffKey,
expected: {
instanceId: INSTANCE_ID,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
origin: "https://workspace.example.ts.net:42013",
},
});
expect(verification.ok).toBe(true);
expect(verification.ok && verification.payload).toMatchObject({
sub: "user-1",
email: "operator@example.com",
next: "/PAP/issues/PAP-17572",
});
// Readiness is probed against the origin the user will be sent to, not a
// separately configured address.
expect(probe).toHaveBeenCalledTimes(1);
expect(probe.mock.calls[0]![0]!.healthUrl).toBe("https://workspace.example.ts.net:42013/api/health");
expect(probe.mock.calls[0]![0]!.handoffSubject).toEqual({
userId: "user-1",
email: "operator@example.com",
});
});
it("refuses to mint a ticket for a workspace that is not ready", async () => {
const cwd = createWorkspaceCwd(INSTANCE_ID);
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ runtimeRows: [{ id: "runtime-1", url: RUNTIME_URL, updatedAt: new Date() }] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd },
actor: { userId: "user-1", userEmail: "operator@example.com", source: "session" },
probe: vi.fn(async () => ({
ok: false as const,
reason: "not_ready" as const,
readiness: null,
detail: "clone_data_missing",
})),
});
expect(result).toMatchObject({
ok: false,
failure: { reason: "workspace_not_ready", detail: "clone_data_missing" },
});
});
it("refuses when no healthy runtime row publishes a URL", async () => {
const cwd = createWorkspaceCwd(INSTANCE_ID);
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ runtimeRows: [] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd },
actor: { userId: "user-1", userEmail: "operator@example.com", source: "session" },
probe: readyProbe(),
});
expect(result).toMatchObject({ ok: false, failure: { reason: "runtime_not_running" } });
});
it("refuses a runtime row whose URL is not an absolute http(s) origin", async () => {
const cwd = createWorkspaceCwd(INSTANCE_ID);
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ runtimeRows: [{ id: "runtime-1", url: "not-a-url", updatedAt: new Date() }] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd },
actor: { userId: "user-1", userEmail: "operator@example.com", source: "session" },
probe: readyProbe(),
});
expect(result).toMatchObject({ ok: false, failure: { reason: "runtime_url_unusable" } });
});
it("fails closed when the worktree instance identity cannot be resolved", async () => {
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ runtimeRows: [{ id: "runtime-1", url: RUNTIME_URL, updatedAt: new Date() }] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd: null },
actor: { userId: "user-1", userEmail: "operator@example.com", source: "session" },
probe: readyProbe(),
});
expect(result).toMatchObject({ ok: false, failure: { reason: "handoff_not_configured" } });
});
it("falls back to credentials when no signing secret is configured at all", async () => {
delete process.env.PAPERCLIP_WORKSPACE_HANDOFF_SECRET;
const previousAuthSecret = process.env.BETTER_AUTH_SECRET;
const previousJwtSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET;
delete process.env.BETTER_AUTH_SECRET;
delete process.env.PAPERCLIP_AGENT_JWT_SECRET;
try {
const cwd = createWorkspaceCwd(INSTANCE_ID);
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ runtimeRows: [{ id: "runtime-1", url: RUNTIME_URL, updatedAt: new Date() }] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd },
actor: { userId: "user-1", userEmail: "operator@example.com", source: "session" },
probe: readyProbe(),
});
expect(result).toMatchObject({ ok: false, failure: { reason: "handoff_not_configured" } });
} finally {
if (previousAuthSecret !== undefined) process.env.BETTER_AUTH_SECRET = previousAuthSecret;
if (previousJwtSecret !== undefined) process.env.PAPERCLIP_AGENT_JWT_SECRET = previousJwtSecret;
}
});
it("refuses an actor with no resolvable board identity", async () => {
const cwd = createWorkspaceCwd(INSTANCE_ID);
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ adminRows: [] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd },
actor: { userId: null, userEmail: null, source: "session" },
probe: readyProbe(),
});
expect(result).toMatchObject({ ok: false, failure: { reason: "no_board_identity" } });
});
it("normalizes a landing target that tries to leave the workspace origin", async () => {
const cwd = createWorkspaceCwd(INSTANCE_ID);
const result = await issueWorkspaceLoginHandoff({
db: stubDb({ runtimeRows: [{ id: "runtime-1", url: RUNTIME_URL, updatedAt: new Date() }] }),
companyId: COMPANY_ID,
executionWorkspace: { id: EXECUTION_WORKSPACE_ID, cwd },
actor: { userId: "user-1", userEmail: "operator@example.com", source: "session" },
next: "https://attacker.example.com/steal",
probe: readyProbe(),
});
expect(result.ok).toBe(true);
if (!result.ok) return;
const identity = resolveManagedWorkspaceIdentity({
workspaceCwd: cwd,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
});
const verification = verifyWorkspaceHandoffTicket({
ticket: new URL(result.issuance.url).searchParams.get(WORKSPACE_HANDOFF_TICKET_QUERY_PARAM),
key: identity!.handoffKey,
expected: {
instanceId: INSTANCE_ID,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
origin: "https://workspace.example.ts.net:42013",
},
});
expect(verification.ok && verification.payload.next).toBe("/");
});
});
describe("resolveWorkspaceHandoffBoardIdentity", () => {
it("uses the caller's own session identity", async () => {
await expect(
resolveWorkspaceHandoffBoardIdentity(stubDb({}), {
userId: "user-9",
userEmail: "nine@example.com",
source: "session",
}),
).resolves.toEqual({ userId: "user-9", email: "nine@example.com" });
});
it("stands in the instance admin for the implicit local-trusted actor", async () => {
await expect(
resolveWorkspaceHandoffBoardIdentity(
stubDb({ adminRows: [{ id: "admin-1", email: "admin@example.com", createdAt: new Date(0) }] }),
{ userId: "local-board", userEmail: null, source: "local_implicit" },
),
).resolves.toEqual({ userId: "admin-1", email: "admin@example.com" });
});
it("returns null when a local-trusted instance has no instance admin", async () => {
await expect(
resolveWorkspaceHandoffBoardIdentity(stubDb({ adminRows: [] }), {
userId: "local-board",
userEmail: null,
source: "local_implicit",
}),
).resolves.toBeNull();
});
});
describe("workspaceLoginHandoffFailureStatus", () => {
it("separates authorization, unsupported, and conflict outcomes", () => {
expect(workspaceLoginHandoffFailureStatus({ reason: "no_board_identity" })).toBe(403);
expect(workspaceLoginHandoffFailureStatus({ reason: "handoff_not_configured" })).toBe(501);
expect(workspaceLoginHandoffFailureStatus({ reason: "runtime_not_running" })).toBe(409);
expect(
workspaceLoginHandoffFailureStatus({ reason: "workspace_not_ready", detail: null, readiness: null }),
).toBe(409);
});
});
@@ -0,0 +1,397 @@
import { describe, expect, it, vi } from "vitest";
import {
buildWorkspaceHandoffExchangeUrl,
deriveWorkspaceHandoffKey,
deriveWorkspaceReadinessToken,
issueWorkspaceHandoffTicket,
normalizeWorkspaceHandoffOrigin,
redactWorkspaceHandoffTicket,
resolveWorkspaceHandoffRootSecret,
sanitizeWorkspaceHandoffRedirectPath,
verifyWorkspaceHandoffTicket,
WORKSPACE_HANDOFF_TICKET_VERSION,
} from "../auth/workspace-login-handoff.js";
import {
exchangeWorkspaceHandoffTicket,
isWorkspaceHandoffRepairReason,
workspaceHandoffFailureStatus,
type WorkspaceHandoffExchangeDeps,
} from "../auth/workspace-login-handoff-exchange.js";
const ROOT_SECRET = "root-secret-for-tests";
const INSTANCE_ID = "pap-17572-workspace-abc123def456";
const EXECUTION_WORKSPACE_ID = "ews-1111-2222";
const COMPANY_ID = "company-1111";
const ORIGIN = "https://workspace.example.ts.net:42013";
const USER_ID = "user-1";
const USER_EMAIL = "Operator@Example.com";
const KEY = deriveWorkspaceHandoffKey({
rootSecret: ROOT_SECRET,
instanceId: INSTANCE_ID,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
});
const EXPECTED = {
instanceId: INSTANCE_ID,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
origin: ORIGIN,
};
function mintTicket(overrides: Partial<Parameters<typeof issueWorkspaceHandoffTicket>[0]> = {}) {
return issueWorkspaceHandoffTicket({
key: KEY,
userId: USER_ID,
email: USER_EMAIL,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
instanceId: INSTANCE_ID,
origin: ORIGIN,
issuerInstanceId: "primary",
...overrides,
});
}
function exchangeDeps(overrides: Partial<WorkspaceHandoffExchangeDeps> = {}) {
const reserved = new Set<string>();
return {
findClonedIdentity: vi.fn(async ({ userId }: { userId: string; companyId: string }) =>
userId === USER_ID
? { userId, email: USER_EMAIL, hasActiveMembership: true }
: null,
),
reserveNonce: vi.fn(async ({ nonce }: { nonce: string }) => {
if (reserved.has(nonce)) return false;
reserved.add(nonce);
return true;
}),
createSession: vi.fn(async () => undefined),
...overrides,
} satisfies WorkspaceHandoffExchangeDeps;
}
describe("workspace handoff key derivation", () => {
it("derives a distinct key per instance and per workspace", () => {
const otherWorkspace = deriveWorkspaceHandoffKey({
rootSecret: ROOT_SECRET,
instanceId: INSTANCE_ID,
executionWorkspaceId: "ews-other",
});
const otherInstance = deriveWorkspaceHandoffKey({
rootSecret: ROOT_SECRET,
instanceId: "other-instance",
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
});
expect(KEY).not.toBe(otherWorkspace);
expect(KEY).not.toBe(otherInstance);
expect(KEY).not.toContain(ROOT_SECRET);
});
it("keeps the readiness probe token separate from the signing key", () => {
const token = deriveWorkspaceReadinessToken({
rootSecret: ROOT_SECRET,
instanceId: INSTANCE_ID,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
});
expect(token).not.toBe(KEY);
});
it("prefers a dedicated secret and otherwise derives one that is not the auth secret", () => {
expect(
resolveWorkspaceHandoffRootSecret({ PAPERCLIP_WORKSPACE_HANDOFF_SECRET: "dedicated" }),
).toEqual({ secret: "dedicated", source: "dedicated" });
const derived = resolveWorkspaceHandoffRootSecret({ BETTER_AUTH_SECRET: "auth-secret" });
expect(derived?.source).toBe("derived");
expect(derived?.secret).not.toBe("auth-secret");
expect(resolveWorkspaceHandoffRootSecret({})).toBeNull();
});
});
describe("verifyWorkspaceHandoffTicket", () => {
it("accepts a ticket bound to this workspace", () => {
const { ticket, payload } = mintTicket({ next: "/PAP/issues/PAP-1" });
const result = verifyWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED });
expect(result).toEqual({ ok: true, payload });
});
it("rejects a ticket signed with another workspace's key", () => {
const foreignKey = deriveWorkspaceHandoffKey({
rootSecret: ROOT_SECRET,
instanceId: INSTANCE_ID,
executionWorkspaceId: "ews-other",
});
const { ticket } = mintTicket({ key: foreignKey });
expect(verifyWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED })).toEqual({
ok: false,
reason: "bad_signature",
});
});
it("rejects a ticket minted for another origin, workspace, or instance", () => {
const wrongOrigin = mintTicket({ origin: "https://other.example.ts.net:42099" });
expect(verifyWorkspaceHandoffTicket({ ticket: wrongOrigin.ticket, key: KEY, expected: EXPECTED }).ok).toBe(false);
expect(
verifyWorkspaceHandoffTicket({ ticket: wrongOrigin.ticket, key: KEY, expected: EXPECTED }),
).toEqual({ ok: false, reason: "origin_mismatch" });
// A ticket for a sibling workspace on the same host cannot be re-signed with
// this workspace's key, so bind-mismatch is proven with a matching key.
const siblingWorkspaceKey = KEY;
const wrongWorkspace = issueWorkspaceHandoffTicket({
key: siblingWorkspaceKey,
userId: USER_ID,
email: USER_EMAIL,
executionWorkspaceId: "ews-sibling",
companyId: COMPANY_ID,
instanceId: INSTANCE_ID,
origin: ORIGIN,
issuerInstanceId: "primary",
});
expect(verifyWorkspaceHandoffTicket({ ticket: wrongWorkspace.ticket, key: KEY, expected: EXPECTED })).toEqual({
ok: false,
reason: "workspace_mismatch",
});
const wrongInstance = issueWorkspaceHandoffTicket({
key: KEY,
userId: USER_ID,
email: USER_EMAIL,
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
companyId: COMPANY_ID,
instanceId: "some-other-instance",
origin: ORIGIN,
issuerInstanceId: "primary",
});
expect(verifyWorkspaceHandoffTicket({ ticket: wrongInstance.ticket, key: KEY, expected: EXPECTED })).toEqual({
ok: false,
reason: "instance_mismatch",
});
});
it("rejects a ticket minted for another company on the same workspace", () => {
// The signing key is per workspace, so a same-host sibling company shares it —
// which is exactly why the company has to be bound and compared.
const { ticket } = mintTicket({ companyId: "company-other" });
expect(verifyWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED })).toEqual({
ok: false,
reason: "company_mismatch",
});
});
it("rejects an expired ticket and one issued in the future", () => {
const now = new Date("2026-08-19T00:00:00.000Z");
const { ticket } = mintTicket({ now, ttlSeconds: 30 });
expect(
verifyWorkspaceHandoffTicket({
ticket,
key: KEY,
expected: EXPECTED,
now: new Date(now.getTime() + 121_000),
}),
).toEqual({ ok: false, reason: "expired" });
expect(
verifyWorkspaceHandoffTicket({
ticket,
key: KEY,
expected: EXPECTED,
now: new Date(now.getTime() - 120_000),
}),
).toEqual({ ok: false, reason: "not_yet_valid" });
});
it("fails closed when local identity cannot be resolved", () => {
const { ticket } = mintTicket();
expect(
verifyWorkspaceHandoffTicket({ ticket, key: KEY, expected: { ...EXPECTED, origin: null } }),
).toEqual({ ok: false, reason: "origin_mismatch" });
expect(
verifyWorkspaceHandoffTicket({ ticket, key: KEY, expected: { ...EXPECTED, instanceId: null } }),
).toEqual({ ok: false, reason: "instance_mismatch" });
expect(
verifyWorkspaceHandoffTicket({ ticket, key: KEY, expected: { ...EXPECTED, executionWorkspaceId: null } }),
).toEqual({ ok: false, reason: "workspace_mismatch" });
expect(
verifyWorkspaceHandoffTicket({ ticket, key: KEY, expected: { ...EXPECTED, companyId: null } }),
).toEqual({ ok: false, reason: "company_mismatch" });
expect(verifyWorkspaceHandoffTicket({ ticket, key: "", expected: EXPECTED })).toEqual({
ok: false,
reason: "not_configured",
});
});
it("rejects malformed, truncated, and re-versioned tickets", () => {
const { ticket } = mintTicket();
const [version, payload, signature] = ticket.split(".");
expect(verifyWorkspaceHandoffTicket({ ticket: "", key: KEY, expected: EXPECTED }).ok).toBe(false);
expect(verifyWorkspaceHandoffTicket({ ticket: `${version}.${payload}`, key: KEY, expected: EXPECTED })).toEqual({
ok: false,
reason: "malformed",
});
expect(
verifyWorkspaceHandoffTicket({ ticket: `wh0.${payload}.${signature}`, key: KEY, expected: EXPECTED }),
).toEqual({ ok: false, reason: "unsupported_version" });
expect(
verifyWorkspaceHandoffTicket({ ticket: `${version}.${payload}.not+base64url`, key: KEY, expected: EXPECTED }),
).toEqual({ ok: false, reason: "malformed" });
// Payload tampering has to fail on the signature, never on JSON parsing.
const tampered = Buffer.from(
JSON.stringify({ ...JSON.parse(Buffer.from(payload!, "base64url").toString()), sub: "attacker" }),
).toString("base64url");
expect(
verifyWorkspaceHandoffTicket({ ticket: `${version}.${tampered}.${signature}`, key: KEY, expected: EXPECTED }),
).toEqual({ ok: false, reason: "bad_signature" });
});
it("uses the declared envelope version", () => {
expect(mintTicket().ticket.startsWith(`${WORKSPACE_HANDOFF_TICKET_VERSION}.`)).toBe(true);
});
});
describe("exchangeWorkspaceHandoffTicket", () => {
it("creates exactly one session for a valid ticket", async () => {
const { ticket } = mintTicket({ next: "/PAP/issues/PAP-2" });
const deps = exchangeDeps();
const result = await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps });
expect(result.ok).toBe(true);
expect(result.ok && result.redirectTo).toBe("/PAP/issues/PAP-2");
expect(deps.createSession).toHaveBeenCalledTimes(1);
expect(deps.createSession).toHaveBeenCalledWith(USER_ID);
});
it("rejects a replayed ticket without creating a second session", async () => {
const { ticket } = mintTicket();
const deps = exchangeDeps();
expect((await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps })).ok).toBe(true);
const replay = await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps });
expect(replay).toEqual({ ok: false, reason: "replayed", payload: expect.objectContaining({ sub: USER_ID }) });
expect(deps.createSession).toHaveBeenCalledTimes(1);
});
it("asks for membership in the ticket's company, not any membership", async () => {
const { ticket } = mintTicket();
const deps = exchangeDeps();
expect((await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps })).ok).toBe(true);
expect(deps.findClonedIdentity).toHaveBeenCalledWith({ userId: USER_ID, companyId: COMPANY_ID });
});
it("reports a missing cloned user and a missing membership distinctly", async () => {
const { ticket } = mintTicket();
const missingUser = exchangeDeps({ findClonedIdentity: vi.fn(async () => null) });
expect(
(await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps: missingUser })),
).toMatchObject({ ok: false, reason: "unknown_user" });
expect(missingUser.createSession).not.toHaveBeenCalled();
const noMembership = exchangeDeps({
findClonedIdentity: vi.fn(async () => ({ userId: USER_ID, email: USER_EMAIL, hasActiveMembership: false })),
});
expect(
(await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps: noMembership })),
).toMatchObject({ ok: false, reason: "missing_membership" });
expect(noMembership.reserveNonce).not.toHaveBeenCalled();
expect(noMembership.createSession).not.toHaveBeenCalled();
});
it("rejects a clone whose user id was reused for a different email", async () => {
const { ticket } = mintTicket();
const deps = exchangeDeps({
findClonedIdentity: vi.fn(async () => ({
userId: USER_ID,
email: "someone-else@example.com",
hasActiveMembership: true,
})),
});
expect(
await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps }),
).toMatchObject({ ok: false, reason: "user_mismatch" });
expect(deps.createSession).not.toHaveBeenCalled();
});
it("matches the cloned email case-insensitively", async () => {
const { ticket } = mintTicket();
const deps = exchangeDeps({
findClonedIdentity: vi.fn(async () => ({
userId: USER_ID,
email: USER_EMAIL.toUpperCase(),
hasActiveMembership: true,
})),
});
expect((await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps })).ok).toBe(true);
});
it("burns the nonce when session creation fails so a captured ticket cannot be retried", async () => {
const { ticket } = mintTicket();
const deps = exchangeDeps({
createSession: vi.fn(async () => {
throw new Error("session store unavailable");
}),
});
expect(
await exchangeWorkspaceHandoffTicket({ ticket, key: KEY, expected: EXPECTED, deps }),
).toMatchObject({ ok: false, reason: "session_failed" });
expect(deps.reserveNonce).toHaveBeenCalledTimes(1);
});
it("never consults request headers: identity comes only from `expected`", async () => {
// A ticket minted for the spoofed host fails against the configured origin,
// which is the whole forwarded-header defense.
const spoofed = mintTicket({ origin: "https://attacker.example.com" });
expect(
await exchangeWorkspaceHandoffTicket({
ticket: spoofed.ticket,
key: KEY,
expected: EXPECTED,
deps: exchangeDeps(),
}),
).toEqual({ ok: false, reason: "origin_mismatch", payload: null });
});
it("maps failures onto stable statuses that do not leak which check failed", () => {
expect(workspaceHandoffFailureStatus("not_configured")).toBe(503);
expect(workspaceHandoffFailureStatus("session_failed")).toBe(500);
for (const reason of ["expired", "replayed", "origin_mismatch", "workspace_mismatch", "company_mismatch", "unknown_user"] as const) {
expect(workspaceHandoffFailureStatus(reason)).toBe(401);
}
expect(isWorkspaceHandoffRepairReason("missing_membership")).toBe(true);
expect(isWorkspaceHandoffRepairReason("expired")).toBe(false);
});
});
describe("handoff URL handling", () => {
it("reduces a landing target to a same-origin path", () => {
expect(sanitizeWorkspaceHandoffRedirectPath("/PAP/issues/PAP-1")).toBe("/PAP/issues/PAP-1");
expect(sanitizeWorkspaceHandoffRedirectPath("https://attacker.example.com/")).toBe("/");
expect(sanitizeWorkspaceHandoffRedirectPath("//attacker.example.com")).toBe("/");
expect(sanitizeWorkspaceHandoffRedirectPath("/\\attacker.example.com")).toBe("/");
expect(sanitizeWorkspaceHandoffRedirectPath("/ok\nSet-Cookie: x=1")).toBe("/");
expect(sanitizeWorkspaceHandoffRedirectPath(undefined)).toBe("/");
});
it("normalizes origins and refuses non-http schemes", () => {
expect(normalizeWorkspaceHandoffOrigin("https://host:443/path")).toBe("https://host");
expect(normalizeWorkspaceHandoffOrigin("http://host:42013/x")).toBe("http://host:42013");
expect(normalizeWorkspaceHandoffOrigin("file:///etc/passwd")).toBeNull();
expect(normalizeWorkspaceHandoffOrigin("not a url")).toBeNull();
});
it("builds an exchange URL on the bound origin", () => {
const { ticket } = mintTicket();
const url = new URL(buildWorkspaceHandoffExchangeUrl({ origin: ORIGIN, ticket }));
expect(url.origin).toBe(ORIGIN);
expect(url.pathname).toBe("/api/auth/workspace-handoff/exchange");
expect(url.searchParams.get("ticket")).toBe(ticket);
});
it("redacts the ticket from anything log-shaped", () => {
const { ticket } = mintTicket();
const line = `GET /api/auth/workspace-handoff/exchange?ticket=${ticket}&next=%2F 302`;
const redacted = redactWorkspaceHandoffTicket(line);
expect(redacted).not.toContain(ticket);
expect(redacted).toContain("ticket=[redacted]");
expect(redacted).toContain("next=%2F");
});
});
@@ -44,14 +44,14 @@ describe("workspace runtime control serialization", () => {
await expect(runExclusiveWorkspaceRuntimeControl({
executionWorkspaceId: "workspace-1",
action: "restart",
run: async () => "restarted",
action: "repair",
run: async () => "repaired",
})).rejects.toMatchObject({
status: 409,
details: {
code: "workspace_runtime_control_in_progress",
activeAction: "start",
requestedAction: "restart",
requestedAction: "repair",
},
});
@@ -204,4 +204,71 @@ describeEmbeddedPostgres("workspace operation reconciliation", () => {
.then((rows) => rows[0]);
expect(unrelated?.status).toBe("running");
});
it("persists bounded repair phase progress on the operation row and log", async () => {
const companyId = randomUUID();
const projectId = randomUUID();
const executionWorkspaceId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Workspace repair diagnostics",
issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(projects).values({
id: projectId,
companyId,
name: "Repair diagnostics",
status: "in_progress",
});
await db.insert(executionWorkspaces).values({
id: executionWorkspaceId,
companyId,
projectId,
mode: "isolated_workspace",
strategyType: "git_worktree",
name: "Repair diagnostics workspace",
status: "active",
cwd: "/tmp/repair-diagnostics-workspace",
});
const operation = await workspaceOperationService(db)
.createRecorder({ companyId, executionWorkspaceId })
.recordOperation({
phase: "workspace_repair",
command: "workspace command repair",
cwd: "/tmp/repair-diagnostics-workspace",
metadata: { action: "repair" },
run: async (reportProgress) => {
await reportProgress({
metadata: {
repairPhase: "target_backup",
repairDiagnostics: [{
phase: "target_backup",
status: "succeeded",
at: "2026-08-18T00:00:00.000Z",
}],
databaseOnly: true,
worktreePreserved: true,
},
system: "Workspace repair target_backup: succeeded.\n",
});
return { status: "succeeded", metadata: { backupRetained: true } };
},
});
expect(operation).toMatchObject({
status: "succeeded",
metadata: {
action: "repair",
repairPhase: "target_backup",
databaseOnly: true,
worktreePreserved: true,
backupRetained: true,
},
});
expect((await workspaceOperationService(db).readLog(operation.id)).content).toContain(
"Workspace repair target_backup: succeeded.",
);
});
});
@@ -0,0 +1,687 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import type { WorkspaceReadinessProbeResult } from "@paperclipai/shared";
import {
isManagedWorkspaceInstance,
resetManagedWorkspaceInstanceCacheForTests,
resolveWorkspaceReadiness,
resolveWorkspaceReadinessState,
resolveWorkspaceSeedMarkerDir,
} from "../services/workspace-readiness.js";
import {
buildManagedWorkspaceGuestEnv,
listManagedWorkspaceHandoffSubjects,
probeManagedWorkspaceHandoffSubjects,
probeManagedWorkspaceReadiness,
resolveManagedWorkspaceIdentity,
resolveWorkspaceReadinessGateMode,
shouldBlockPublicationOnReadiness,
waitForManagedWorkspaceReadiness,
type ManagedWorkspaceIdentity,
} from "../services/managed-workspace-identity.js";
import {
WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY,
WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY,
WORKSPACE_HANDOFF_KEY_ENV_KEY,
WORKSPACE_READINESS_TOKEN_ENV_KEY,
WORKSPACE_READINESS_TOKEN_HEADER,
WORKSPACE_READINESS_USER_EMAIL_HEADER,
WORKSPACE_READINESS_USER_ID_HEADER,
} from "../auth/workspace-login-handoff.js";
const tempDirs: string[] = [];
function createMarkerDir(files: Record<string, string> = {}) {
const dir = mkdtempSync(path.join(os.tmpdir(), "paperclip-workspace-readiness-"));
tempDirs.push(dir);
const configPath = path.join(dir, "config.json");
writeFileSync(configPath, "{}\n", "utf8");
for (const [name, contents] of Object.entries(files)) {
writeFileSync(path.join(dir, name), contents, "utf8");
}
return { dir, configPath };
}
function verifiedManifest(overrides: Record<string, unknown> = {}) {
return JSON.stringify({
version: 2,
state: "verified",
phase: "complete",
seedMode: "minimal",
...overrides,
});
}
/** A `db` stand-in whose clone/identity probes both find one row. */
function readyDb() {
return {
execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
// Both readiness probes are `limit(1)` existence queries, so one row is enough
// to answer either of them.
select: vi.fn(() => {
const result = [{ companyId: "company-1", userId: "user-1" }];
const chain = {
from: vi.fn(() => chain),
innerJoin: vi.fn(() => chain),
where: vi.fn(() => chain),
limit: vi.fn(() => Promise.resolve(result)),
then: (resolve: (rows: unknown) => unknown) => Promise.resolve(result).then(resolve),
};
return chain;
}),
} as unknown as Db;
}
function handoffSubjectDb(rows: Array<{ userId: string; email: string | null }>) {
return {
select: vi.fn(() => {
const chain = {
from: vi.fn(() => chain),
innerJoin: vi.fn(() => chain),
then: (resolve: (value: unknown) => unknown) => Promise.resolve(rows).then(resolve),
};
return chain;
}),
} as unknown as Db;
}
function readyGuestEnv(configPath: string, overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
return {
PAPERCLIP_CONFIG: configPath,
[WORKSPACE_HANDOFF_KEY_ENV_KEY]: "key",
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: "ews-1",
[WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY]: "company-1",
...overrides,
};
}
afterEach(() => {
for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
resetManagedWorkspaceInstanceCacheForTests();
vi.restoreAllMocks();
});
describe("resolveWorkspaceReadinessState", () => {
const ready = {
databaseReady: true,
cloneDataReady: true,
authHandoffReady: true,
seedState: "verified",
} as const;
it("is ready only when every signal agrees and the seed is verified", () => {
expect(resolveWorkspaceReadinessState(ready)).toBe("ready");
expect(resolveWorkspaceReadinessState({ ...ready, seedState: "unknown" })).toBe("validating");
expect(resolveWorkspaceReadinessState({ ...ready, seedState: "absent" })).toBe("validating");
});
it("reports an unfinished clone as provisioning, not degraded", () => {
expect(resolveWorkspaceReadinessState({ ...ready, seedState: "pending", databaseReady: false })).toBe("provisioning");
expect(resolveWorkspaceReadinessState({ ...ready, seedState: "running" })).toBe("provisioning");
});
it("reports a regressed verified clone as degraded", () => {
expect(resolveWorkspaceReadinessState({ ...ready, databaseReady: false })).toBe("degraded");
expect(resolveWorkspaceReadinessState({ ...ready, cloneDataReady: false })).toBe("degraded");
expect(resolveWorkspaceReadinessState({ ...ready, authHandoffReady: false })).toBe("degraded");
});
it("reports a recorded seed failure as failed", () => {
expect(resolveWorkspaceReadinessState({ ...ready, seedState: "failed" })).toBe("failed");
});
});
describe("resolveWorkspaceReadiness", () => {
it("reports a verified, fully readable clone as ready", async () => {
const { configPath } = createMarkerDir({ "seed-manifest.json": verifiedManifest() });
const readiness = await resolveWorkspaceReadiness({
db: readyDb(),
env: readyGuestEnv(configPath),
});
expect(readiness).toMatchObject({
state: "ready",
databaseReady: true,
cloneDataReady: true,
authHandoffReady: true,
authHandoffUserId: null,
seedState: "verified",
seedMode: "minimal",
executionWorkspaceId: "ews-1",
companyId: "company-1",
failurePhase: null,
});
});
it("does not claim the handoff is ready without a signing key", async () => {
const { configPath } = createMarkerDir({ "seed-manifest.json": verifiedManifest() });
const readiness = await resolveWorkspaceReadiness({
db: readyDb(),
env: readyGuestEnv(configPath, { [WORKSPACE_HANDOFF_KEY_ENV_KEY]: undefined }),
});
expect(readiness.authHandoffReady).toBe(false);
expect(readiness.state).toBe("degraded");
expect(readiness.failurePhase).toBe("auth_handoff_not_configured");
});
it("fails closed when the guest has no company binding", async () => {
const { configPath } = createMarkerDir({ "seed-manifest.json": verifiedManifest() });
const db = readyDb();
const readiness = await resolveWorkspaceReadiness({
db,
env: readyGuestEnv(configPath, {
[WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY]: undefined,
}),
});
expect(readiness).toMatchObject({
state: "degraded",
cloneDataReady: false,
authHandoffReady: false,
failurePhase: "workspace_company_not_configured",
});
expect(db.select).not.toHaveBeenCalled();
});
it("does not claim handoff readiness without the execution-workspace identity", async () => {
const { configPath } = createMarkerDir({ "seed-manifest.json": verifiedManifest() });
const readiness = await resolveWorkspaceReadiness({
db: readyDb(),
env: readyGuestEnv(configPath, {
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: undefined,
}),
});
expect(readiness).toMatchObject({
state: "degraded",
cloneDataReady: true,
authHandoffReady: false,
failurePhase: "workspace_identity_not_configured",
});
});
it("surfaces the failing seed phase instead of an inferred one", async () => {
const { configPath } = createMarkerDir({
"seed-manifest.json": verifiedManifest({ state: "failed", phase: "restore" }),
});
const readiness = await resolveWorkspaceReadiness({
db: readyDb(),
env: readyGuestEnv(configPath),
});
expect(readiness).toMatchObject({ state: "failed", seedState: "failed", failurePhase: "restore" });
});
it("treats an unreadable manifest as a failure rather than assuming it seeded", async () => {
const { configPath } = createMarkerDir({ "seed-manifest.json": "{ truncated" });
const readiness = await resolveWorkspaceReadiness({
db: readyDb(),
env: readyGuestEnv(configPath),
});
expect(readiness).toMatchObject({ state: "failed", failurePhase: "seed_manifest_unreadable" });
});
it("reports an interrupted seed as provisioning", async () => {
const { configPath } = createMarkerDir({ "seed-pending": "{}" });
const readiness = await resolveWorkspaceReadiness({
db: readyDb(),
env: readyGuestEnv(configPath),
});
expect(readiness).toMatchObject({ state: "provisioning", seedState: "pending" });
});
it("reports a legacy marker clone as validating, never verified", async () => {
const { configPath } = createMarkerDir({ "seed-complete": "" });
const readiness = await resolveWorkspaceReadiness({
db: readyDb(),
env: readyGuestEnv(configPath),
});
expect(readiness).toMatchObject({ state: "validating", seedState: "unknown" });
});
it("reports an unreachable database without throwing", async () => {
const { configPath } = createMarkerDir({ "seed-manifest.json": verifiedManifest() });
const db = {
execute: vi.fn().mockRejectedValue(new Error("ECONNREFUSED")),
select: vi.fn(),
} as unknown as Db;
const readiness = await resolveWorkspaceReadiness({
db,
env: readyGuestEnv(configPath),
});
expect(readiness).toMatchObject({
state: "degraded",
databaseReady: false,
failurePhase: "database_unreachable",
});
});
it("resolves the marker directory from the configured config path", () => {
const { dir, configPath } = createMarkerDir();
expect(resolveWorkspaceSeedMarkerDir({ PAPERCLIP_CONFIG: configPath })).toBe(dir);
});
it("does not re-stat marker files on every health request", () => {
const { configPath } = createMarkerDir({ "seed-manifest.json": verifiedManifest() });
const env = { PAPERCLIP_CONFIG: configPath };
let clock = 0;
expect(isManagedWorkspaceInstance(env, () => clock)).toBe(true);
// Removing the marker inside the TTL keeps the cached answer; past it, the
// filesystem is consulted again.
rmSync(path.join(path.dirname(configPath), "seed-manifest.json"));
clock = 1_000;
expect(isManagedWorkspaceInstance(env, () => clock)).toBe(true);
clock = 10_000;
expect(isManagedWorkspaceInstance(env, () => clock)).toBe(false);
});
it("only treats a process with clone evidence as a managed workspace", () => {
const { configPath } = createMarkerDir();
expect(isManagedWorkspaceInstance({ PAPERCLIP_CONFIG: configPath })).toBe(false);
expect(
isManagedWorkspaceInstance({ PAPERCLIP_CONFIG: configPath, [WORKSPACE_HANDOFF_KEY_ENV_KEY]: "key" }),
).toBe(true);
const seeded = createMarkerDir({ "seed-manifest.json": verifiedManifest() });
expect(isManagedWorkspaceInstance({ PAPERCLIP_CONFIG: seeded.configPath })).toBe(true);
});
});
describe("probeManagedWorkspaceReadiness", () => {
const identity: ManagedWorkspaceIdentity = {
instanceId: "instance-a",
executionWorkspaceId: "ews-1",
companyId: "company-1",
handoffKey: "handoff-key",
readinessToken: "probe-token",
secretSource: "derived",
};
function respond(body: unknown, init: { status?: number } = {}) {
return vi.fn(async () =>
new Response(JSON.stringify(body), {
status: init.status ?? 200,
headers: { "content-type": "application/json" },
}),
) as unknown as typeof fetch;
}
const readyPayload = {
status: "ok",
workspace: {
state: "ready",
databaseReady: true,
cloneDataReady: true,
authHandoffReady: true,
authHandoffUserId: null,
seedState: "verified",
seedPhase: "complete",
seedMode: "minimal",
instanceId: "instance-a",
executionWorkspaceId: "ews-1",
companyId: "company-1",
failurePhase: null,
},
};
it("accepts a ready workspace and sends the derived probe token", async () => {
const fetchImpl = respond(readyPayload);
const result = await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl,
});
expect(result.ok).toBe(true);
const [, init] = (fetchImpl as unknown as ReturnType<typeof vi.fn>).mock.calls[0]!;
expect((init as RequestInit & { headers: Record<string, string> }).headers[WORKSPACE_READINESS_TOKEN_HEADER])
.toBe("probe-token");
});
it("binds a caller-scoped readiness probe to the exact handoff user", async () => {
const fetchImpl = respond({
...readyPayload,
workspace: { ...readyPayload.workspace, authHandoffUserId: "user-1" },
});
const result = await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
handoffSubject: { userId: "user-1", email: "operator@example.com" },
fetchImpl,
});
expect(result.ok).toBe(true);
const [, init] = (fetchImpl as unknown as ReturnType<typeof vi.fn>).mock.calls[0]!;
expect((init as RequestInit & { headers: Record<string, string> }).headers).toMatchObject({
[WORKSPACE_READINESS_USER_ID_HEADER]: "user-1",
[WORKSPACE_READINESS_USER_EMAIL_HEADER]: "operator@example.com",
});
});
it("rejects a readiness response scoped to another handoff user", async () => {
expect(
await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
handoffSubject: { userId: "user-1", email: "operator@example.com" },
fetchImpl: respond({
...readyPayload,
workspace: { ...readyPayload.workspace, authHandoffUserId: "user-other" },
}),
}),
).toMatchObject({ ok: false, reason: "identity_mismatch" });
});
it("proves every active control-plane board identity before publication", async () => {
const db = handoffSubjectDb([
{ userId: "user-1", email: "one@example.com" },
{ userId: "user-2", email: "two@example.com" },
]);
await expect(listManagedWorkspaceHandoffSubjects(db, "company-1")).resolves.toEqual([
{ userId: "user-1", email: "one@example.com" },
{ userId: "user-2", email: "two@example.com" },
]);
const fetchImpl = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
const headers = init?.headers as Record<string, string>;
return new Response(JSON.stringify({
...readyPayload,
workspace: {
...readyPayload.workspace,
authHandoffUserId: headers[WORKSPACE_READINESS_USER_ID_HEADER],
},
}), { status: 200, headers: { "content-type": "application/json" } });
}) as unknown as typeof fetch;
await expect(probeManagedWorkspaceHandoffSubjects({
db,
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl,
})).resolves.toMatchObject({ ok: true });
expect(fetchImpl).toHaveBeenCalledTimes(2);
});
it("blocks publication when no current board identity can use the handoff", async () => {
const fetchImpl = vi.fn() as unknown as typeof fetch;
await expect(probeManagedWorkspaceHandoffSubjects({
db: handoffSubjectDb([]),
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl,
})).resolves.toMatchObject({ ok: false, reason: "not_ready" });
expect(fetchImpl).not.toHaveBeenCalled();
});
it("rejects a workspace serving another instance, workspace, or company", async () => {
const wrongInstance = await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: respond({
...readyPayload,
workspace: { ...readyPayload.workspace, instanceId: "instance-b" },
}),
});
expect(wrongInstance).toMatchObject({ ok: false, reason: "identity_mismatch" });
const wrongWorkspace = await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: respond({
...readyPayload,
workspace: { ...readyPayload.workspace, executionWorkspaceId: "ews-other" },
}),
});
expect(wrongWorkspace).toMatchObject({ ok: false, reason: "identity_mismatch" });
const wrongCompany = await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: respond({
...readyPayload,
workspace: { ...readyPayload.workspace, companyId: "company-other" },
}),
});
expect(wrongCompany).toMatchObject({ ok: false, reason: "identity_mismatch" });
});
it("rejects a partial readiness contract with no company identity", async () => {
const { companyId: _companyId, ...partialReadiness } = readyPayload.workspace;
expect(
await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: respond({ ...readyPayload, workspace: partialReadiness }),
}),
).toMatchObject({ ok: false, reason: "identity_mismatch", readiness: null });
});
it("rejects a 200 response whose payload is not semantically healthy", async () => {
expect(
await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: respond({ status: "unhealthy" }),
}),
).toMatchObject({ ok: false, reason: "unhealthy_payload" });
});
it("rejects a response with no readiness block, so a legacy guest cannot publish", async () => {
expect(
await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: respond({ status: "ok" }),
}),
).toMatchObject({ ok: false, reason: "readiness_missing" });
});
it("rejects a workspace whose clone or handoff is not ready and keeps the failure phase", async () => {
expect(
await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: respond({
...readyPayload,
workspace: {
...readyPayload.workspace,
state: "degraded",
cloneDataReady: false,
failurePhase: "clone_data_missing",
},
}),
}),
).toMatchObject({ ok: false, reason: "not_ready", detail: "clone_data_missing" });
});
it("reports an unreachable guest", async () => {
const result = await probeManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:42013/api/health",
identity,
fetchImpl: (async () => {
throw new Error("connect ECONNREFUSED");
}) as unknown as typeof fetch,
});
expect(result).toMatchObject({ ok: false, reason: "unreachable" });
});
it("fails closed when any part of the workspace identity is unresolved", () => {
// Every readiness and handoff check keys off this. Returning a partial identity
// would silently downgrade the whole gate to the legacy transport check, so the
// three inputs are all required.
const complete = {
workspaceCwd: "/srv/worktree",
executionWorkspaceId: "ews-1",
companyId: "company-1",
env: { PAPERCLIP_WORKSPACE_HANDOFF_SECRET: "root" },
};
expect(resolveManagedWorkspaceIdentity(complete)).not.toBeNull();
expect(resolveManagedWorkspaceIdentity({ ...complete, companyId: null })).toBeNull();
expect(resolveManagedWorkspaceIdentity({ ...complete, executionWorkspaceId: null })).toBeNull();
expect(resolveManagedWorkspaceIdentity({ ...complete, workspaceCwd: null })).toBeNull();
expect(resolveManagedWorkspaceIdentity({ ...complete, env: {} })).toBeNull();
});
it("derives distinct key material per company on the same workspace", () => {
const base = {
workspaceCwd: "/srv/worktree",
executionWorkspaceId: "ews-1",
env: { PAPERCLIP_WORKSPACE_HANDOFF_SECRET: "root" },
};
const first = resolveManagedWorkspaceIdentity({ ...base, companyId: "company-1" });
const second = resolveManagedWorkspaceIdentity({ ...base, companyId: "company-2" });
// The signing key is deliberately per instance+workspace, not per company —
// the company is enforced by the signed `cid` claim instead — so the keys match
// while the recorded company differs.
expect(first?.handoffKey).toBe(second?.handoffKey);
expect(first?.companyId).toBe("company-1");
expect(second?.companyId).toBe("company-2");
});
it("hands the guest only derived per-workspace material", () => {
expect(buildManagedWorkspaceGuestEnv(identity)).toEqual({
[WORKSPACE_HANDOFF_KEY_ENV_KEY]: "handoff-key",
[WORKSPACE_READINESS_TOKEN_ENV_KEY]: "probe-token",
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: "ews-1",
PAPERCLIP_EXECUTION_WORKSPACE_COMPANY_ID: "company-1",
});
});
});
describe("shouldBlockPublicationOnReadiness", () => {
function rejection(reason: Extract<WorkspaceReadinessProbeResult, { ok: false }>["reason"]) {
return { ok: false as const, reason, readiness: null, detail: null };
}
it("blocks on any real disagreement about the clone", () => {
for (const reason of ["unreachable", "http_error", "unhealthy_payload", "not_ready", "identity_mismatch"] as const) {
expect(shouldBlockPublicationOnReadiness(rejection(reason), "auto")).toBe(true);
expect(shouldBlockPublicationOnReadiness(rejection(reason), "strict")).toBe(true);
}
});
it("does not turn an upgrade lag into an outage by default", () => {
expect(shouldBlockPublicationOnReadiness(rejection("readiness_missing"), "auto")).toBe(false);
expect(shouldBlockPublicationOnReadiness(rejection("readiness_missing"), "strict")).toBe(true);
});
it("reads the deployment-level mode from the environment", () => {
expect(resolveWorkspaceReadinessGateMode({})).toBe("auto");
expect(resolveWorkspaceReadinessGateMode({ PAPERCLIP_WORKSPACE_READINESS_GATE: "STRICT" })).toBe("strict");
expect(resolveWorkspaceReadinessGateMode({ PAPERCLIP_WORKSPACE_READINESS_GATE: "anything-else" })).toBe("auto");
});
});
describe("waitForManagedWorkspaceReadiness", () => {
const identity: ManagedWorkspaceIdentity = {
instanceId: "instance-a",
executionWorkspaceId: "ews-1",
companyId: "company-1",
handoffKey: "k",
readinessToken: "t",
secretSource: "derived",
};
const ready = {
status: "ok",
workspace: {
state: "ready",
databaseReady: true,
cloneDataReady: true,
authHandoffReady: true,
authHandoffUserId: null,
seedState: "verified",
seedPhase: "complete",
seedMode: "minimal",
instanceId: "instance-a",
executionWorkspaceId: "ews-1",
companyId: "company-1",
failurePhase: null,
},
};
it("absorbs a guest that becomes ready a beat after its listener", async () => {
let attempt = 0;
const fetchImpl = (async () => {
attempt += 1;
const body = attempt < 3
? { ...ready, workspace: { ...ready.workspace, databaseReady: false, state: "validating" } }
: ready;
return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" } });
}) as unknown as typeof fetch;
let clock = 0;
const result = await waitForManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:1/api/health",
identity,
fetchImpl,
now: () => clock,
sleep: async (ms) => {
clock += ms;
},
});
expect(result.ok).toBe(true);
expect(attempt).toBe(3);
});
it("does not poll a guest that has no readiness contract to satisfy", async () => {
let attempt = 0;
const fetchImpl = (async () => {
attempt += 1;
return new Response(JSON.stringify({ status: "ok" }), {
status: 200,
headers: { "content-type": "application/json" },
});
}) as unknown as typeof fetch;
const result = await waitForManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:1/api/health",
identity,
fetchImpl,
now: () => 0,
sleep: async () => undefined,
});
expect(result).toMatchObject({ ok: false, reason: "readiness_missing" });
expect(attempt).toBe(1);
});
it("gives up immediately when another instance owns the port", async () => {
let attempt = 0;
const fetchImpl = (async () => {
attempt += 1;
return new Response(
JSON.stringify({ ...ready, workspace: { ...ready.workspace, instanceId: "someone-else" } }),
{ status: 200, headers: { "content-type": "application/json" } },
);
}) as unknown as typeof fetch;
const result = await waitForManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:1/api/health",
identity,
fetchImpl,
now: () => 0,
sleep: async () => undefined,
});
expect(result).toMatchObject({ ok: false, reason: "identity_mismatch" });
expect(attempt).toBe(1);
});
it("stops at the gate budget instead of holding a start open forever", async () => {
let attempt = 0;
const fetchImpl = (async () => {
attempt += 1;
throw new Error("connect ECONNREFUSED");
}) as unknown as typeof fetch;
let clock = 0;
const result = await waitForManagedWorkspaceReadiness({
healthUrl: "http://127.0.0.1:1/api/health",
identity,
fetchImpl,
timeoutMs: 1_000,
now: () => clock,
sleep: async (ms) => {
clock += ms;
},
});
expect(result).toMatchObject({ ok: false, reason: "unreachable" });
// 1s budget at the 250ms retry interval: four probes, then the budget is spent.
expect(attempt).toBe(5);
});
});
@@ -0,0 +1,54 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { loadConfig } from "../config.ts";
// The terminal-workspace reaper reads PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS.
// These tests lock the parser contract: the default is 7 days, an explicit 0
// means immediate reaping, and empty, whitespace-only, negative, or non-numeric
// values fall back to the default. An empty or whitespace-only value must not
// become 0, because that would delete terminal workspaces immediately.
describe("workspace reaper cooldown config parsing", () => {
afterEach(() => {
vi.unstubAllEnvs();
});
it("uses the 7 day default when the variable is not set", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", undefined);
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
});
it("uses the 7 day default for an empty value", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "");
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
});
it("uses the 7 day default for a whitespace-only value", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", " ");
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
});
it("keeps an explicit 0 as immediate reaping", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "0");
expect(loadConfig().workspaceReaperCooldownDays).toBe(0);
});
it("reads a positive whole number", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "14");
expect(loadConfig().workspaceReaperCooldownDays).toBe(14);
});
it("trims surrounding whitespace before it reads the number", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", " 3 ");
expect(loadConfig().workspaceReaperCooldownDays).toBe(3);
});
it("uses the 7 day default for a negative value", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "-1");
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
});
it("uses the 7 day default for a non-numeric value", () => {
vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "soon");
expect(loadConfig().workspaceReaperCooldownDays).toBe(7);
});
});
@@ -54,7 +54,7 @@ vi.mock("../services/index.js", () => ({
secretService: () => mockSecretService,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/workspace-runtime.js", () => ({
@@ -84,7 +84,7 @@ function registerWorkspaceRouteMocks() {
secretService: () => mockSecretService,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.doMock("../services/workspace-runtime.js", () => ({
+546 -18
View File
@@ -134,6 +134,17 @@ async function runPnpm(cwd: string, args: string[]) {
await execFileAsync("pnpm", args, { cwd });
}
async function writeRegisteredSourceConfig(baseCwd: string, instanceId = "source-instance") {
const configDir = path.join(baseCwd, ".paperclip");
await fs.mkdir(configDir, { recursive: true });
await fs.writeFile(path.join(configDir, "config.json"), "{}\n", "utf8");
await fs.writeFile(
path.join(configDir, ".env"),
`PAPERCLIP_INSTANCE_ID=${instanceId}\n`,
"utf8",
);
}
async function createTempRepo(defaultBranch = "main") {
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-repo-"));
await runGit(repoRoot, ["init"]);
@@ -437,8 +448,6 @@ describe("resolveRuntimeProvisionCommand", () => {
path.join(baseCwd, "scripts", "provision-worktree-runtime.sh"),
"#!/usr/bin/env bash\n",
);
await fs.mkdir(path.join(cwd, ".paperclip"), { recursive: true });
await fs.writeFile(path.join(cwd, ".paperclip", "seed-pending"), "{}\n");
const workspace = {
...buildWorkspace(cwd),
baseCwd,
@@ -446,6 +455,17 @@ describe("resolveRuntimeProvisionCommand", () => {
worktreePath: cwd,
};
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.mkdir(path.join(cwd, ".paperclip"), { recursive: true });
await fs.writeFile(path.join(cwd, ".paperclip", "config.json"), "{}\n");
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(path.join(cwd, ".paperclip", "seed-pending"), "{}\n");
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
@@ -455,6 +475,41 @@ describe("resolveRuntimeProvisionCommand", () => {
})).toBe("./custom-provision.sh");
await fs.writeFile(path.join(cwd, ".paperclip", "seed-complete"), "{}\n");
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({ version: 2, state: "failed" }),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({ version: 2, state: "verified" }),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: "/source/config.json" },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "full",
migrationRevision: "0001",
targetInstanceId: "target",
phase: "complete",
state: "verified",
attemptId: "attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe("");
} finally {
await fs.rm(baseCwd, { recursive: true, force: true });
@@ -728,6 +783,55 @@ describe("realizeExecutionWorkspace", () => {
expect(second.branchName).toBe(first.branchName);
});
it("defaults the repo-provided worktree provisioner for git worktree strategies", async () => {
const repoRoot = await createTempRepo();
await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true });
await fs.writeFile(
path.join(repoRoot, "scripts", "provision-worktree.sh"),
[
"#!/usr/bin/env bash",
"set -euo pipefail",
"mkdir -p .paperclip",
"printf 'provisioned\\n' > .paperclip/default-provision-ran",
"",
].join("\n"),
"utf8",
);
await runGit(repoRoot, ["add", "scripts/provision-worktree.sh"]);
await runGit(repoRoot, ["commit", "-m", "Add repository worktree provisioner"]);
const workspace = await realizeExecutionWorkspace({
base: {
baseCwd: repoRoot,
source: "project_primary",
projectId: "project-1",
workspaceId: "workspace-1",
repoUrl: null,
repoRef: "HEAD",
},
config: {
workspaceStrategy: {
type: "git_worktree",
branchTemplate: "{{issue.identifier}}-{{slug}}",
},
},
issue: {
id: "issue-default-provision",
identifier: "PAP-17684",
title: "Default worktree provisioning",
},
agent: {
id: "agent-1",
name: "Codex Coder",
companyId: "company-1",
},
});
await expect(
fs.readFile(path.join(workspace.cwd, ".paperclip", "default-provision-ran"), "utf8"),
).resolves.toBe("provisioned\n");
});
it("warns when reusing a git worktree whose base ref has advanced", async () => {
const repoRoot = await createTempRepo();
@@ -1373,8 +1477,13 @@ describe("realizeExecutionWorkspace", () => {
it("writes an isolated repo-local Paperclip config and worktree branding when provisioning", async () => {
const repoRoot = await createTempRepo();
await writeRegisteredSourceConfig(repoRoot, "worktree-base-source");
const previousCwd = process.cwd();
const previousPath = process.env.PATH;
const previousConfig = process.env.PAPERCLIP_CONFIG;
const previousHome = process.env.PAPERCLIP_HOME;
const previousInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
const previousWorktreesDir = process.env.PAPERCLIP_WORKTREES_DIR;
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-home-"));
const isolatedWorktreeHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktrees-"));
const isolatedBin = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-bin-"));
@@ -1386,6 +1495,7 @@ describe("realizeExecutionWorkspace", () => {
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = instanceId;
process.env.PAPERCLIP_WORKTREES_DIR = isolatedWorktreeHome;
delete process.env.PAPERCLIP_CONFIG;
// Keep this server-side fixture on provision-worktree.sh's config writer path;
// CLI/database seeding is covered by the CLI worktree tests.
await fs.symlink(process.execPath, path.join(isolatedBin, "node"));
@@ -1557,6 +1667,18 @@ describe("realizeExecutionWorkspace", () => {
} else {
process.env.PATH = previousPath;
}
for (const [key, value] of [
["PAPERCLIP_CONFIG", previousConfig],
["PAPERCLIP_HOME", previousHome],
["PAPERCLIP_INSTANCE_ID", previousInstanceId],
["PAPERCLIP_WORKTREES_DIR", previousWorktreesDir],
] as const) {
if (value === undefined) {
delete process.env[key];
} else {
process.env[key] = value;
}
}
}
}, 15_000);
@@ -1564,6 +1686,7 @@ describe("realizeExecutionWorkspace", () => {
"provisions worktree-local pnpm node_modules instead of reusing base-repo links",
async () => {
const repoRoot = await createTempRepo();
await writeRegisteredSourceConfig(repoRoot);
await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true });
await fs.mkdir(path.join(repoRoot, "packages", "shared"), { recursive: true });
await fs.mkdir(path.join(repoRoot, "server"), { recursive: true });
@@ -1666,6 +1789,7 @@ describe("realizeExecutionWorkspace", () => {
it("provisions successfully when install is needed but there are no symlinked node_modules to move", async () => {
const repoRoot = await createTempRepo();
await writeRegisteredSourceConfig(repoRoot);
await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true });
await fs.writeFile(
path.join(repoRoot, "package.json"),
@@ -1748,6 +1872,7 @@ describe("realizeExecutionWorkspace", () => {
try {
await fs.mkdir(path.join(baseRoot, "node_modules"), { recursive: true });
await writeRegisteredSourceConfig(baseRoot);
await fs.mkdir(path.join(worktreeRoot, "node_modules"), { recursive: true });
await fs.mkdir(path.join(worktreeRoot, "ui"), { recursive: true });
await fs.mkdir(fakeBin, { recursive: true });
@@ -1840,6 +1965,7 @@ describe("realizeExecutionWorkspace", () => {
try {
await fs.mkdir(baseRoot, { recursive: true });
await writeRegisteredSourceConfig(baseRoot);
await fs.mkdir(worktreeRoot, { recursive: true });
await fs.mkdir(fakeBin, { recursive: true });
await fs.copyFile(provisionWorktreeScriptPath, scriptPath);
@@ -1897,6 +2023,7 @@ describe("realizeExecutionWorkspace", () => {
try {
await fs.mkdir(baseRoot, { recursive: true });
await writeRegisteredSourceConfig(baseRoot);
await fs.mkdir(paperclipDir, { recursive: true });
await fs.mkdir(fakeBin, { recursive: true });
await fs.copyFile(provisionWorktreeScriptPath, scriptPath);
@@ -1987,6 +2114,7 @@ describe("realizeExecutionWorkspace", () => {
try {
await fs.mkdir(path.join(baseRoot, "node_modules"), { recursive: true });
await writeRegisteredSourceConfig(baseRoot);
await fs.mkdir(worktreeRoot, { recursive: true });
await fs.mkdir(fakeBin, { recursive: true });
await fs.copyFile(provisionWorktreeScriptPath, scriptPath);
@@ -2056,6 +2184,7 @@ describe("realizeExecutionWorkspace", () => {
"provisions worktree-local pnpm node_modules instead of reusing base-repo links",
async () => {
const repoRoot = await createTempRepo();
await writeRegisteredSourceConfig(repoRoot);
await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true });
await fs.mkdir(path.join(repoRoot, "packages", "shared"), { recursive: true });
await fs.mkdir(path.join(repoRoot, "server"), { recursive: true });
@@ -3813,6 +3942,107 @@ describe("ensureRuntimeServicesForRun", () => {
}
});
it("records the built-in deferred seed as failed when its manifest is not verified", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-seed-operation-"));
const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "workspace-seed-operation");
const scriptsDir = path.join(workspaceRoot, "scripts");
const markerDir = path.join(workspaceRoot, ".paperclip");
await fs.mkdir(scriptsDir, { recursive: true });
await fs.mkdir(markerDir, { recursive: true });
await fs.writeFile(path.join(markerDir, "seed-pending"), "{}\n", "utf8");
await fs.writeFile(
path.join(scriptsDir, "provision-worktree-runtime.sh"),
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`printf '%s\\n' '${JSON.stringify({ version: 2, state: "failed", phase: "source_validation" })}' > .paperclip/seed-manifest.json`,
].join("\n"),
"utf8",
);
const config = runtimeProvisionTestConfig({});
const workspace = {
...buildWorkspace(workspaceRoot),
source: "task_session" as const,
strategy: "git_worktree" as const,
worktreePath: workspaceRoot,
};
const { recorder, operations } = createWorkspaceOperationRecorderDouble();
try {
await expect(
startRuntimeServicesForWorkspaceControl(
runtimeProvisionStartInput({ workspace, config, recorder }),
),
).rejects.toThrow(/without a verified manifest.*source_validation/);
expect(operations).toEqual([
expect.objectContaining({
phase: "workspace_seed",
result: expect.objectContaining({
status: "failed",
exitCode: 1,
metadata: expect.objectContaining({
provisionKind: "workspace_seed",
seedState: "failed",
seedPhase: "source_validation",
seedFailurePhase: "source_validation",
}),
}),
}),
]);
} finally {
await stopRuntimeServicesForExecutionWorkspace({
executionWorkspaceId: "execution-workspace-1",
workspaceCwd: workspaceRoot,
});
await fs.rm(workspaceRoot, { recursive: true, force: true });
restorePaperclipEnv();
}
});
it("keeps an explicit command matching the built-in seed command as runtime provisioning", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-explicit-runtime-provision-"));
const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "explicit-runtime-provision");
const scriptsDir = path.join(workspaceRoot, "scripts");
await fs.mkdir(scriptsDir, { recursive: true });
await fs.writeFile(
path.join(scriptsDir, "provision-worktree-runtime.sh"),
"#!/usr/bin/env bash\nset -euo pipefail\n",
"utf8",
);
const config = runtimeProvisionTestConfig({
provisionCommand: "bash ./scripts/provision-worktree-runtime.sh",
});
const workspace = buildWorkspace(workspaceRoot);
const { recorder, operations } = createWorkspaceOperationRecorderDouble();
try {
const services = await startRuntimeServicesForWorkspaceControl(
runtimeProvisionStartInput({ workspace, config, recorder }),
);
expect(services).toHaveLength(1);
expect(operations).toEqual([
expect.objectContaining({
phase: "workspace_runtime_provision",
metadata: expect.objectContaining({
provisionKind: "runtime_dependencies",
}),
result: expect.objectContaining({
status: "succeeded",
}),
}),
]);
} finally {
await stopRuntimeServicesForExecutionWorkspace({
executionWorkspaceId: "execution-workspace-1",
workspaceCwd: workspaceRoot,
});
await fs.rm(workspaceRoot, { recursive: true, force: true });
restorePaperclipEnv();
}
});
it("does not create a runtime provision operation when the command is absent", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-provision-noop-"));
const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "runtime-provision-noop");
@@ -3919,6 +4149,86 @@ describe("ensureRuntimeServicesForRun", () => {
}
});
it("replaces a reused Paperclip dev runtime whose 2xx health payload is unhealthy", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-misreported-health-"));
const workspace = buildWorkspace(workspaceRoot);
const serviceCommand =
"node -e \"let healthy=true;const http=require('node:http');http.createServer((req,res)=>{if(req.url==='/misreport'){healthy=false;res.end('failed');return;}if(req.url==='/api/health'){res.setHeader('content-type','application/json');res.end(JSON.stringify(healthy?{status:'ok'}:{status:'unhealthy',error:'database_unreachable'}));return;}res.end('ok')}).listen(Number(process.env.PORT),'127.0.0.1')\"";
const input = {
actor: { id: "agent-1", name: "Codex Coder", companyId: "company-1" },
issue: null,
workspace,
executionWorkspaceId: "execution-workspace-health",
config: { workspaceRuntime: { services: [{
name: "paperclip-dev",
command: serviceCommand,
cwd: ".",
port: { type: "auto" as const },
readiness: { type: "http" as const, urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 3, intervalMs: 100 },
expose: { type: "url" as const, urlTemplate: "http://127.0.0.1:{{port}}" },
lifecycle: "shared" as const,
stopPolicy: { type: "manual" as const },
}] } },
adapterEnv: {},
};
try {
const [first] = await startRuntimeServicesForWorkspaceControl(input);
await expect(fetch(`${first!.url}/misreport`)).resolves.toMatchObject({ ok: true });
await expect(fetch(`${first!.url}/api/health`)).resolves.toMatchObject({ ok: true });
const [[replacement], [concurrentReuse]] = await Promise.all([
startRuntimeServicesForWorkspaceControl(input),
startRuntimeServicesForWorkspaceControl(input),
]);
expect(replacement?.id).not.toBe(first?.id);
expect(replacement?.reused).toBe(false);
expect(concurrentReuse?.id).toBe(replacement?.id);
expect(concurrentReuse?.reused).toBe(true);
} finally {
await stopRuntimeServicesForExecutionWorkspace({
executionWorkspaceId: "execution-workspace-health",
workspaceCwd: workspaceRoot,
});
await fs.rm(workspaceRoot, { recursive: true, force: true });
}
});
it("reuses a shared Paperclip dev runtime after one transient unhealthy response", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-transient-health-"));
const workspace = buildWorkspace(workspaceRoot);
const serviceCommand =
"node -e \"let failNext=false;const http=require('node:http');http.createServer((req,res)=>{if(req.url==='/fail-next'){failNext=true;res.end('armed');return;}if(req.url==='/api/health'){res.setHeader('content-type','application/json');const healthy=!failNext;failNext=false;res.end(JSON.stringify({status:healthy?'ok':'unhealthy'}));return;}res.end('ok')}).listen(Number(process.env.PORT),'127.0.0.1')\"";
const input = {
actor: { id: "agent-1", name: "Codex Coder", companyId: "company-1" },
issue: null,
workspace,
executionWorkspaceId: "execution-workspace-transient-health",
config: { workspaceRuntime: { services: [{
name: "paperclip-dev",
command: serviceCommand,
cwd: ".",
port: { type: "auto" as const },
readiness: { type: "http" as const, urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 3, intervalMs: 100 },
expose: { type: "url" as const, urlTemplate: "http://127.0.0.1:{{port}}" },
lifecycle: "shared" as const,
stopPolicy: { type: "manual" as const },
}] } },
adapterEnv: {},
};
try {
const [first] = await startRuntimeServicesForWorkspaceControl(input);
await expect(fetch(`${first!.url}/fail-next`)).resolves.toMatchObject({ ok: true });
const [reused] = await startRuntimeServicesForWorkspaceControl(input);
expect(reused?.id).toBe(first?.id);
expect(reused?.reused).toBe(true);
} finally {
await stopRuntimeServicesForExecutionWorkspace({
executionWorkspaceId: "execution-workspace-transient-health",
workspaceCwd: workspaceRoot,
});
await fs.rm(workspaceRoot, { recursive: true, force: true });
}
});
it("uses explicit readiness URL when exposed URL is not the local probe address", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-explicit-readiness-"));
const workspace = buildWorkspace(workspaceRoot);
@@ -4085,8 +4395,10 @@ describe("ensureRuntimeServicesForRun", () => {
branchName: "PAP-874-chat-speed-issues",
worktreePath: worktreeWorkspaceRoot,
};
// A Paperclip dev runtime must answer `/api/health` semantically before it may
// be published, so the fake serves the same shape a real one does.
const serviceCommand =
"node -e \"require('node:http').createServer((req,res)=>res.end(process.env.PAPERCLIP_HOME)).listen(Number(process.env.PORT), '127.0.0.1')\"";
"node -e \"require('node:http').createServer((req,res)=>{if(req.url==='/api/health'){res.setHeader('content-type','application/json');res.end(JSON.stringify({status:'ok'}));return;}res.end(process.env.PAPERCLIP_HOME)}).listen(Number(process.env.PORT), '127.0.0.1')\"";
const config = {
workspaceRuntime: {
services: [
@@ -5971,7 +6283,7 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () =>
const delayedHmrScript = [
"const http=require('node:http');",
"const port=Number(process.env.PORT);",
"http.createServer((_req,res)=>res.end('ok')).listen(port,'127.0.0.1');",
"http.createServer((req,res)=>{if(req.url==='/api/health'){res.setHeader('content-type','application/json');res.end(JSON.stringify({status:'ok'}));return;}res.end('ok')}).listen(port,'127.0.0.1');",
"setTimeout(()=>http.createServer((_req,res)=>res.end('hmr')).listen(port+10000,'127.0.0.1'),750);",
"setInterval(()=>{},1000);",
].join("");
@@ -6803,7 +7115,7 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
const projectWorkspaceId = randomUUID();
// Binds the app port and its HMR companion, both loopback-only.
const command =
"node -e \"const http=require('node:http');const p=Number(process.env.PORT);for(const q of [p,p+10000])http.createServer((req,res)=>res.end('ok')).listen(q,'127.0.0.1');setInterval(()=>{},1000)\"";
"node -e \"const http=require('node:http');const p=Number(process.env.PORT);for(const q of [p,p+10000])http.createServer((req,res)=>{if(req.url==='/api/health'){res.setHeader('content-type','application/json');res.end(JSON.stringify({status:'ok'}));return;}res.end('ok')}).listen(q,'127.0.0.1');setInterval(()=>{},1000)\"";
const workspaceRuntime = {
services: [
{
@@ -6978,7 +7290,7 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
}
}, 40_000);
it("adopts a live auto-port shared service after runtime state is reset", async () => {
it("re-adopts a request-logging service on the same auto port after supervisor stdio closes", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-reconcile-"));
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-"));
process.env.PAPERCLIP_HOME = paperclipHome;
@@ -6987,7 +7299,6 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
const companyId = randomUUID();
const agentId = randomUUID();
const runId = randomUUID();
const executionWorkspaceId = randomUUID();
await db.insert(companies).values({
id: companyId,
@@ -7021,6 +7332,16 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
projectId: null,
workspaceId: null,
};
await fs.writeFile(
path.join(workspaceRoot, "request-logger.cjs"),
[
"const http = require('node:http');",
"http.createServer((req, res) => {",
" process.stdout.write(`request ${req.url}\\n`, (error) => { if (!error) res.end('ok'); });",
"}).listen(Number(process.env.PORT), '127.0.0.1');",
].join("\n"),
"utf8",
);
leasedRunIds.add(runId);
const services = await ensureRuntimeServicesForRun({
@@ -7038,8 +7359,7 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
services: [
{
name: "web",
command:
"node -e \"require('node:http').createServer((req,res)=>res.end('ok')).listen(Number(process.env.PORT), '127.0.0.1')\"",
command: "node request-logger.cjs",
port: { type: "auto" },
readiness: {
type: "http",
@@ -7063,9 +7383,13 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
const service = services[0];
expect(service?.url).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
await expect(fetch(service!.url!)).resolves.toMatchObject({ ok: true });
const originalPort = service!.port;
const originalProviderRef = service!.providerRef;
await fs.rm(paperclipHome, { recursive: true, force: true });
await resetRuntimeServicesForTests();
// Closing the parent's pipe endpoints reproduces a real control-plane exit.
// A service whose per-request logger still targets those pipes wedges (or
// exits) on the reconciliation health probe instead of answering it.
await resetRuntimeServicesForTests({ simulateSupervisorExit: true });
const result = await reconcilePersistedRuntimeServicesOnStartup(db);
expect(result).toMatchObject({ reconciled: 1, adopted: 1, stopped: 0 });
@@ -7076,17 +7400,221 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
.where(eq(workspaceRuntimeServices.id, service!.id))
.then((rows) => rows[0] ?? null);
expect(persisted?.status).toBe("running");
expect(persisted?.providerRef).toMatch(/^\d+$/);
expect(persisted?.port).toBe(originalPort);
expect(persisted?.providerRef).toBe(originalProviderRef);
await expect(fetch(service!.url!)).resolves.toMatchObject({ ok: true });
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId,
workspaceCwd: workspace.cwd,
});
await resetRuntimeServicesForTests({ terminateProcesses: true });
leasedRunIds.delete(runId);
await fs.rm(paperclipHome, { recursive: true, force: true });
await fs.rm(workspaceRoot, { recursive: true, force: true });
await expect(fetch(service!.url!)).rejects.toThrow();
});
it("re-adopts a desired service when pnpm is represented as the pnpm.cjs launcher", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-pnpm-reconcile-"));
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-"));
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
const previousInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = `runtime-pnpm-reconcile-${randomUUID()}`;
const portProbe = net.createServer();
await new Promise<void>((resolve) => portProbe.listen(0, "127.0.0.1", resolve));
const address = portProbe.address();
const port = typeof address === "object" && address ? address.port : null;
await new Promise<void>((resolve, reject) => {
portProbe.close((error) => error ? reject(error) : resolve());
});
if (!port) throw new Error("Failed to reserve pnpm reconciliation test port");
const companyId = randomUUID();
const projectId = randomUUID();
const executionWorkspaceId = randomUUID();
const runtimeServiceId = randomUUID();
const command = "pnpm dev";
const service = {
name: "web",
command,
port,
readiness: {
type: "http",
urlTemplate: "http://127.0.0.1:{{port}}",
timeoutSec: 10,
intervalMs: 50,
},
expose: null,
lifecycle: "shared",
reuseScope: "execution_workspace",
stopPolicy: { type: "manual" },
};
const reuseKey = createHash("sha256")
.update(stableStringifyForTest({
scopeType: "execution_workspace",
scopeId: executionWorkspaceId,
serviceName: service.name,
command,
cwd: workspaceRoot,
port,
env: {},
expose: null,
}))
.digest("hex");
const wrapperPath = path.join(workspaceRoot, "pnpm.cjs");
await fs.writeFile(
wrapperPath,
[
"const http = require('node:http');",
"const port = Number(process.argv[3]);",
"http.createServer((_req, res) => res.end('ok')).listen(port, '127.0.0.1');",
].join("\n"),
"utf8",
);
const child = spawn(process.execPath, [wrapperPath, "dev", String(port)], {
cwd: workspaceRoot,
detached: process.platform !== "win32",
stdio: "ignore",
});
child.unref();
try {
for (let attempt = 0; attempt < 50; attempt += 1) {
try {
if ((await fetch(`http://127.0.0.1:${port}`)).ok) break;
} catch {
// Wait for the simulated package-manager launcher to bind.
}
await new Promise((resolve) => setTimeout(resolve, 50));
}
await expect(fetch(`http://127.0.0.1:${port}`)).resolves.toMatchObject({ ok: true });
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(projects).values({
id: projectId,
companyId,
name: "Runtime pnpm reconciliation",
status: "in_progress",
});
await db.insert(executionWorkspaces).values({
id: executionWorkspaceId,
companyId,
projectId,
projectWorkspaceId: null,
mode: "isolated_workspace",
strategyType: "git_worktree",
name: "Runtime pnpm reconciliation workspace",
status: "active",
cwd: workspaceRoot,
providerType: "local_fs",
providerRef: workspaceRoot,
metadata: {
config: {
workspaceRuntime: { services: [service] },
desiredState: "running",
serviceStates: { "0": "running" },
},
},
});
await db.insert(workspaceRuntimeServices).values({
id: runtimeServiceId,
companyId,
projectId,
projectWorkspaceId: null,
executionWorkspaceId,
issueId: null,
scopeType: "execution_workspace",
scopeId: executionWorkspaceId,
serviceName: service.name,
status: "running",
lifecycle: "shared",
reuseKey,
command,
cwd: workspaceRoot,
port,
url: `http://127.0.0.1:${port}`,
provider: "local_process",
providerRef: String(child.pid),
ownerAgentId: null,
startedByRunId: null,
lastUsedAt: new Date(),
startedAt: new Date(),
stoppedAt: null,
stopPolicy: { type: "manual" },
healthStatus: "healthy",
});
await writeLocalServiceRegistryRecord({
version: 1,
serviceKey: `workspace-runtime-web-${randomUUID()}`,
profileKind: "workspace-runtime",
serviceName: service.name,
command,
cwd: workspaceRoot,
envFingerprint: reuseKey,
port,
url: `http://127.0.0.1:${port}`,
pid: child.pid!,
processGroupId: child.pid ?? null,
provider: "local_process",
runtimeServiceId,
reuseKey,
startedAt: new Date().toISOString(),
lastSeenAt: new Date().toISOString(),
metadata: { executionWorkspaceId },
});
const result = await reconcilePersistedRuntimeServicesOnStartup(db);
expect(result).toMatchObject({
reconciled: 1,
adopted: 1,
stopped: 0,
restarted: 0,
restartFailed: 0,
});
const rows = await db
.select()
.from(workspaceRuntimeServices)
.where(eq(workspaceRuntimeServices.executionWorkspaceId, executionWorkspaceId));
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({
id: runtimeServiceId,
status: "running",
healthStatus: "healthy",
port,
providerRef: String(child.pid),
});
expect(await readLocalServicePortOwner(port)).toBe(child.pid);
await expect(fetch(`http://127.0.0.1:${port}`)).resolves.toMatchObject({ ok: true });
} finally {
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId,
workspaceCwd: workspaceRoot,
}).catch(() => undefined);
if (child.pid) {
try {
if (process.platform === "win32") process.kill(child.pid, "SIGKILL");
else process.kill(-child.pid, "SIGKILL");
} catch {
// The adopted runtime was already stopped through the managed path.
}
}
await resetRuntimeServicesForTests();
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
if (previousInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
else process.env.PAPERCLIP_INSTANCE_ID = previousInstanceId;
await fs.rm(paperclipHome, { recursive: true, force: true });
await fs.rm(workspaceRoot, { recursive: true, force: true });
}
}, 20_000);
it("does not reuse a stopped auto-port service port while another process owns it", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-unhealthy-adopt-"));
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-"));
@@ -7112,7 +7640,7 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
const executionWorkspaceId = randomUUID();
const stoppedServiceId = randomUUID();
const serviceCommand =
"node -e \"const http=require('node:http'); const stale=process.env.STALE_HEALTH==='1'; http.createServer((req,res)=>{ if (req.url==='/api/health' && stale) { res.statusCode=503; res.end('database_unreachable'); return; } res.end('ok'); }).listen(Number(process.env.PORT), '127.0.0.1')\"";
"node -e \"const http=require('node:http'); const stale=process.env.STALE_HEALTH==='1'; http.createServer((req,res)=>{ if (req.url==='/api/health') { if (stale) { res.statusCode=503; res.end('database_unreachable'); return; } res.setHeader('content-type','application/json'); res.end(JSON.stringify({status:'ok'})); return; } res.end('ok'); }).listen(Number(process.env.PORT), '127.0.0.1')\"";
const scopeType = "agent";
const scopeId = agentId;
const reuseKey = createHash("sha256")
+17 -3
View File
@@ -444,13 +444,27 @@ describe("worktree config repair", () => {
const firstConfigPath = path.join(firstWorktreeRoot, ".paperclip", "config.json");
const secondConfigPath = path.join(secondWorktreeRoot, ".paperclip", "config.json");
const writeWorktree = async (worktreeRoot: string, name: string) => {
const writeWorktree = async (
worktreeRoot: string,
name: string,
databaseMode: "embedded-postgres" | "postgres" = "embedded-postgres",
) => {
const paperclipDir = path.join(worktreeRoot, ".paperclip");
const instanceRoot = path.join(isolatedHome, "instances", name.toLowerCase());
const config = buildIsolatedConfig(instanceRoot, 45439, 55439);
await fs.mkdir(paperclipDir, { recursive: true });
await fs.writeFile(
path.join(paperclipDir, "config.json"),
`${JSON.stringify(buildIsolatedConfig(instanceRoot, 45439, 55439), null, 2)}\n`,
`${JSON.stringify({
...config,
database: {
...config.database,
mode: databaseMode,
...(databaseMode === "postgres"
? { connectionString: "postgres://paperclip:paperclip@127.0.0.1:55439/paperclip" }
: {}),
},
}, null, 2)}\n`,
"utf8",
);
await fs.writeFile(
@@ -480,7 +494,7 @@ describe("worktree config repair", () => {
delete process.env.DATABASE_URL;
};
await writeWorktree(firstWorktreeRoot, "PAP-14013-import-bulk-skills");
await writeWorktree(firstWorktreeRoot, "PAP-14013-import-bulk-skills", "postgres");
await writeWorktree(secondWorktreeRoot, "PAP-14069-port-conflicts");
const staleLockPath = path.join(isolatedHome, ".worktree-port-reservations.lock");
await fs.mkdir(staleLockPath, { recursive: true });
@@ -0,0 +1,229 @@
import { execFile } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { afterEach, describe, expect, it } from "vitest";
import {
ensureWorktreeSeeded,
readWorktreeSeedManifest,
} from "../../../cli/src/commands/worktree.ts";
import { realizeExecutionWorkspace } from "../services/workspace-runtime.ts";
const execFileAsync = promisify(execFile);
const cleanup: string[] = [];
const originalConfig = process.env.PAPERCLIP_CONFIG;
const originalWorktreesDir = process.env.PAPERCLIP_WORKTREES_DIR;
async function runGit(cwd: string, args: string[]) {
await execFileAsync("git", args, { cwd });
}
async function writeConfig(configPath: string, instanceId: string) {
const instanceRoot = path.dirname(configPath);
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.writeFile(configPath, `${JSON.stringify({
$meta: { version: 1, updatedAt: "2026-08-19T00:00:00.000Z", source: "configure" },
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(instanceRoot, "db"),
embeddedPostgresPort: 54329,
backup: { enabled: false, intervalMinutes: 60, retentionDays: 30, dir: path.join(instanceRoot, "backups") },
},
logging: { mode: "file", logDir: path.join(instanceRoot, "logs") },
server: {
deploymentMode: "local_trusted",
exposure: "private",
host: "127.0.0.1",
port: 3100,
allowedHostnames: [],
serveUi: true,
},
}, null, 2)}\n`, "utf8");
await fs.writeFile(
path.join(path.dirname(configPath), ".env"),
`PAPERCLIP_INSTANCE_ID=${instanceId}\n`,
"utf8",
);
}
function verifiedSeedResult() {
return {
backupSummary: "snapshot.sql",
snapshotAt: "2026-08-19T00:00:00.000Z",
migrationRevision: "0223_test.sql",
pausedScheduledRoutines: 0,
executionQuarantine: {
disabledTimerHeartbeats: 0,
resetRunningAgents: 0,
quarantinedInProgressIssues: 0,
unassignedTodoIssues: 0,
unassignedReviewIssues: 0,
stoppedProjectWorkspaceRuntimes: 0,
stoppedExecutionWorkspaceRuntimes: 0,
stoppedRuntimeServices: 0,
},
reboundWorkspaces: [],
validation: {
authUserCount: 1,
credentialAccountCount: 1,
instanceAdminCount: 1,
activeMembershipCount: 1,
companyCount: 1,
issueCount: 1,
representativeCompanyId: "00000000-0000-4000-8000-000000000001",
representativeIssueId: "00000000-0000-4000-8000-000000000002",
migrationRevision: "0223_test.sql",
},
};
}
afterEach(async () => {
if (originalConfig === undefined) delete process.env.PAPERCLIP_CONFIG;
else process.env.PAPERCLIP_CONFIG = originalConfig;
if (originalWorktreesDir === undefined) delete process.env.PAPERCLIP_WORKTREES_DIR;
else process.env.PAPERCLIP_WORKTREES_DIR = originalWorktreesDir;
for (const dir of cleanup.splice(0)) {
await fs.rm(dir, { recursive: true, force: true });
}
});
describe("managed worktree seed source through the server spawn path", () => {
it("re-derives an ambient-instance manifest written before provisioning", async () => {
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-server-seed-source-"));
cleanup.push(tempRoot);
const repoRoot = path.join(tempRoot, "repo");
const hooksDir = path.join(tempRoot, "hooks");
const ambientConfigPath = path.join(tempRoot, "ambient", "config.json");
const registeredConfigPath = path.join(repoRoot, ".paperclip", "config.json");
const worktreeHome = path.join(tempRoot, "worktree-home");
await fs.mkdir(repoRoot, { recursive: true });
await runGit(repoRoot, ["init", "-q"]);
await runGit(repoRoot, ["config", "user.email", "paperclip@example.com"]);
await runGit(repoRoot, ["config", "user.name", "Paperclip Test"]);
await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true });
await fs.writeFile(path.join(repoRoot, "README.md"), "server spawn regression\n", "utf8");
await fs.copyFile(
fileURLToPath(new URL("../../../scripts/provision-worktree.sh", import.meta.url)),
path.join(repoRoot, "scripts", "provision-worktree.sh"),
);
await fs.chmod(path.join(repoRoot, "scripts", "provision-worktree.sh"), 0o755);
await runGit(repoRoot, ["add", "README.md", "scripts/provision-worktree.sh"]);
await runGit(repoRoot, ["commit", "-qm", "Add managed provision script"]);
await writeConfig(registeredConfigPath, "registered-source");
await writeConfig(ambientConfigPath, "ambient-instance");
await fs.mkdir(worktreeHome, { recursive: true });
// Reproduce the integrated lane's ordering with a checkout hook: an earlier
// worktree-init-style writer inherits the server environment and leaves a valid
// target config whose manifest diagnostic points at the ambient instance.
await fs.mkdir(hooksDir, { recursive: true });
const hookPath = path.join(hooksDir, "post-checkout");
await fs.writeFile(
hookPath,
`#!/usr/bin/env node
const crypto = require("node:crypto");
const fs = require("node:fs");
const path = require("node:path");
const cwd = process.cwd();
if (cwd === ${JSON.stringify(repoRoot)}) process.exit(0);
const stateDir = path.join(cwd, ".paperclip");
const normalized = path.basename(cwd).trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/-+/g, "-").replace(/^[-_]+|[-_]+$/g, "");
const instanceId = \`${"${(normalized || \"worktree\").slice(0, 48)}"}-\${crypto.createHash("sha256").update(path.resolve(cwd)).digest("hex").slice(0, 12)}\`;
const targetConfigPath = path.join(stateDir, "config.json");
const instanceRoot = path.join(process.env.PAPERCLIP_WORKTREES_DIR, "instances", instanceId);
fs.mkdirSync(stateDir, { recursive: true });
fs.mkdirSync(instanceRoot, { recursive: true });
fs.writeFileSync(targetConfigPath, JSON.stringify({
$meta: { version: 1, updatedAt: "2026-08-19T00:00:00.000Z", source: "configure" },
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(instanceRoot, "db"),
embeddedPostgresPort: 54330,
backup: { enabled: false, intervalMinutes: 60, retentionDays: 30, dir: path.join(instanceRoot, "backups") },
},
logging: { mode: "file", logDir: path.join(instanceRoot, "logs") },
server: {
deploymentMode: "local_trusted",
exposure: "private",
host: "127.0.0.1",
port: 3101,
allowedHostnames: [],
serveUi: true,
},
}, null, 2) + "\\n");
fs.writeFileSync(path.join(stateDir, ".env"), [
"PAPERCLIP_HOME=" + JSON.stringify(process.env.PAPERCLIP_WORKTREES_DIR),
"PAPERCLIP_INSTANCE_ID=" + JSON.stringify(instanceId),
"PAPERCLIP_CONFIG=" + JSON.stringify(targetConfigPath),
"",
].join("\\n"));
fs.writeFileSync(path.join(stateDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "ambient-instance", configPath: process.env.PAPERCLIP_CONFIG },
snapshotAt: null,
seedMode: "minimal",
migrationRevision: null,
targetInstanceId: instanceId,
phase: "pending",
state: "pending",
attemptId: "ambient-writer",
startedAt: null,
finishedAt: null,
diagnostics: [{ phase: "pending", status: "succeeded", at: new Date().toISOString() }],
}, null, 2) + "\\n");
`,
"utf8",
);
await fs.chmod(hookPath, 0o755);
await runGit(repoRoot, ["config", "core.hooksPath", hooksDir]);
process.env.PAPERCLIP_CONFIG = ambientConfigPath;
process.env.PAPERCLIP_WORKTREES_DIR = worktreeHome;
const workspace = await realizeExecutionWorkspace({
base: {
baseCwd: repoRoot,
source: "project_primary",
projectId: "project-1",
workspaceId: "project-workspace-1",
repoUrl: null,
repoRef: "HEAD",
},
config: {
workspaceStrategy: {
type: "git_worktree",
branchTemplate: "{{issue.identifier}}-{{slug}}",
provisionCommand: "bash ./scripts/provision-worktree.sh",
},
},
issue: {
id: "issue-1",
identifier: "PAP-17681",
title: "Recover ambient seed source",
},
agent: { id: "agent-1", name: "Coder", companyId: "company-1" },
});
expect(readWorktreeSeedManifest(path.join(workspace.cwd, ".paperclip", "config.json"))).toMatchObject({
source: { instanceId: "ambient-instance", configPath: ambientConfigPath },
state: "pending",
});
await expect(ensureWorktreeSeeded({
config: path.join(workspace.cwd, ".paperclip", "config.json"),
registeredBaseWorkspaceCwd: repoRoot,
registeredProjectWorkspaceId: "project-workspace-1",
expectedCompanyId: "company-1",
}, {
seedDatabase: async () => verifiedSeedResult(),
})).resolves.toMatchObject({ seeded: true, reason: "seeded" });
expect(readWorktreeSeedManifest(path.join(workspace.cwd, ".paperclip", "config.json"))).toMatchObject({
source: { instanceId: "registered-source", configPath: registeredConfigPath },
state: "verified",
phase: "complete",
});
}, 20_000);
});
+48
View File
@@ -0,0 +1,48 @@
import { describe, expect, it } from "vitest";
import type { AdapterLoginCapability } from "@paperclipai/adapter-utils";
import { validateAdapterModule } from "./plugin-loader.js";
// A minimal external adapter module. The loader calls `createServerAdapter()`
// and then validates the returned module. Each test controls the returned
// `loginCapability` to check the fail-closed rule at load time.
function makeModule(loginCapability?: unknown) {
return {
createServerAdapter: () => ({
type: "vendor_local",
execute: async () => ({}),
testEnvironment: async () => ({}),
...(loginCapability === undefined ? {} : { loginCapability }),
}),
};
}
const validLoginCapability: AdapterLoginCapability = {
panelMode: "displayed_code",
sandboxTransport: "streamed_exec",
timeoutPolicy: "caller_bounded",
getCommand: () => "vendor login",
parsePrompt: () => null,
};
describe("validateAdapterModule login capability", () => {
it("loads an adapter with no login capability", () => {
expect(() => validateAdapterModule(makeModule(), "vendor-pkg")).not.toThrow();
});
it("loads an adapter with a well-formed login capability", () => {
expect(() => validateAdapterModule(makeModule(validLoginCapability), "vendor-pkg")).not.toThrow();
});
it("rejects an adapter with a malformed login capability", () => {
const bad = { ...validLoginCapability, panelMode: "hidden_code" };
expect(() => validateAdapterModule(makeModule(bad), "vendor-pkg")).toThrow(
/invalid login capability/,
);
});
it("rejects an adapter with a non-object login capability", () => {
expect(() => validateAdapterModule(makeModule("displayed_code"), "vendor-pkg")).toThrow(
/invalid login capability/,
);
});
});
+15 -1
View File
@@ -13,6 +13,7 @@ import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import type { ServerAdapterModule } from "./types.js";
import { validateAdapterLoginCapability } from "@paperclipai/adapter-utils";
import { logger } from "../middleware/logger.js";
import {
@@ -145,7 +146,7 @@ function extractUiParserSource(
// Load / reload
// ---------------------------------------------------------------------------
function validateAdapterModule(mod: unknown, packageName: string): ServerAdapterModule {
export function validateAdapterModule(mod: unknown, packageName: string): ServerAdapterModule {
const m = mod as Record<string, unknown>;
const createServerAdapter = m.createServerAdapter;
if (typeof createServerAdapter !== "function") {
@@ -161,6 +162,19 @@ function validateAdapterModule(mod: unknown, packageName: string): ServerAdapter
`createServerAdapter() from "${packageName}" returned an invalid module (missing "type").`,
);
}
// Fail closed on a malformed login capability. The validator throws a clear
// error, so the loader rejects the adapter instead of loading it with a
// partial capability.
try {
validateAdapterLoginCapability(adapterModule);
} catch (err) {
throw new Error(
`createServerAdapter() from "${packageName}" returned an invalid login capability: ` +
`${err instanceof Error ? err.message : String(err)}`,
);
}
return adapterModule;
}
+37
View File
@@ -0,0 +1,37 @@
import { describe, expect, it } from "vitest";
import { assertValidAdapterLoginCapability } from "@paperclipai/adapter-utils";
import { requireServerAdapter } from "./registry.js";
// The registry registers a login capability for the two built-in interactive
// adapters. The test checks the scalar values and the presence of the required
// callbacks. It also runs the shared validator, so the built-in capabilities
// obey the same fail-closed contract as an external adapter.
describe("built-in adapter login capabilities", () => {
it("registers the Codex device-login capability", () => {
const capability = requireServerAdapter("codex_local").loginCapability;
expect(capability).toBeDefined();
if (!capability) return;
expect(capability.panelMode).toBe("displayed_code");
expect(capability.sandboxTransport).toBe("streamed_exec");
expect(capability.timeoutPolicy).toBe("caller_bounded");
expect(capability.completionClaim).toBeUndefined();
expect(typeof capability.getCommand).toBe("function");
expect(typeof capability.parsePrompt).toBe("function");
expect(() => assertValidAdapterLoginCapability(capability, "codex_local")).not.toThrow();
});
it("registers the Claude setup-token capability", () => {
const capability = requireServerAdapter("claude_local").loginCapability;
expect(capability).toBeDefined();
if (!capability) return;
expect(capability.panelMode).toBe("submitted_browser_code");
expect(capability.sandboxTransport).toBe("pseudo_terminal");
expect(capability.timeoutPolicy).toBe("fixed");
expect(capability.completionClaim).toBe("storedSessionId");
expect(typeof capability.getCommand).toBe("function");
expect(typeof capability.parsePrompt).toBe("function");
expect(typeof capability.captureCredential).toBe("function");
expect(() => assertValidAdapterLoginCapability(capability, "claude_local")).not.toThrow();
});
});
+45
View File
@@ -9,6 +9,7 @@ import {
buildSandboxNpmInstallCommand,
getAdapterSessionManagement,
} from "@paperclipai/adapter-utils";
import type { AdapterLoginCapability } from "@paperclipai/adapter-utils";
import {
execute as claudeExecute,
listClaudeSkills,
@@ -19,6 +20,9 @@ import {
sessionCodec as claudeSessionCodec,
getQuotaWindows as claudeGetQuotaWindows,
getConfigSchema as getClaudeConfigSchema,
CLAUDE_SETUP_TOKEN_COMMAND,
parseSetupTokenPrompt,
parseSetupTokenCredential,
} from "@paperclipai/adapter-claude-local/server";
import {
agentConfigurationDoc as claudeAgentConfigurationDoc,
@@ -33,6 +37,8 @@ import {
sessionCodec as codexSessionCodec,
getQuotaWindows as codexGetQuotaWindows,
getConfigSchema as getCodexConfigSchema,
CODEX_DEVICE_LOGIN_COMMAND,
parseDeviceLoginPrompt,
} from "@paperclipai/adapter-codex-local/server";
import {
agentConfigurationDoc as codexAgentConfigurationDoc,
@@ -181,6 +187,43 @@ The standalone ACPX adapter has been retired. Use:
Paperclip keeps this tombstone registered so stale acpx_local rows fail clearly instead of falling back to the process adapter.
`;
// The Claude interactive login capability. Claude runs `claude setup-token` on a
// real pseudo-terminal. The user pastes a browser code back into the flow. The
// flow uses a fixed host-side timeout and records a stored session identifier on
// success. The capability data holds no secret; the callbacks return runtime
// values only.
const claudeLoginCapability: AdapterLoginCapability = {
panelMode: "submitted_browser_code",
sandboxTransport: "pseudo_terminal",
timeoutPolicy: "fixed",
getCommand: () => CLAUDE_SETUP_TOKEN_COMMAND,
parsePrompt: (output) => {
const prompt = parseSetupTokenPrompt(output);
return prompt ? { url: prompt.url } : null;
},
captureCredential: (output) => {
const token = parseSetupTokenCredential(output);
return token === null ? null : Buffer.from(token, "utf8");
},
completionClaim: "storedSessionId",
};
// The Codex interactive login capability. Codex runs `codex login --device-auth`
// over the streamed exec channel. The flow shows a one-time code that the user
// enters in the browser. The caller sets the host-side timeout. The device-login
// flow writes its credential inside the sandbox, so the capability declares no
// terminal credential capture and no completion claim.
const codexLoginCapability: AdapterLoginCapability = {
panelMode: "displayed_code",
sandboxTransport: "streamed_exec",
timeoutPolicy: "caller_bounded",
getCommand: () => CODEX_DEVICE_LOGIN_COMMAND,
parsePrompt: (output) => {
const prompt = parseDeviceLoginPrompt(output);
return prompt ? { url: prompt.url, code: prompt.code } : null;
},
};
const claudeLocalAdapter: ServerAdapterModule = {
type: "claude_local",
execute: stampClaudeAgentIdHeader(claudeExecute),
@@ -210,6 +253,7 @@ const claudeLocalAdapter: ServerAdapterModule = {
agentConfigurationDoc: claudeAgentConfigurationDoc,
getConfigSchema: getClaudeConfigSchema,
getQuotaWindows: claudeGetQuotaWindows,
loginCapability: claudeLoginCapability,
};
const acpxLocalAdapter: ServerAdapterModule = {
@@ -282,6 +326,7 @@ const codexLocalAdapter: ServerAdapterModule = {
agentConfigurationDoc: codexAgentConfigurationDoc,
getConfigSchema: getCodexConfigSchema,
getQuotaWindows: codexGetQuotaWindows,
loginCapability: codexLoginCapability,
};
const cursorLocalAdapter: ServerAdapterModule = {
+3 -5
View File
@@ -493,12 +493,10 @@ export async function createApp(
confidentialEdgeTlsTerminated: setupTokenLoginEdgeTlsTerminated,
setupTokenLogin: setupTokenLoginTransport,
onSetupTokenLoginService: (service) => {
// Capture the service, so the graceful-shutdown hook cancels every live
// session and releases each lease. The standalone scheduled reaper owns
// the startup and interval lease cleanup now (SR-4).
setupTokenLoginService = service;
// Startup reaper (SR-4): release any lease whose login session is
// terminal or past its deadline after a restart. The DB is ready here.
void service.reap().catch((err) => {
logger.error({ err }, "Setup-token login startup reaper failed");
});
},
}),
);
+60
View File
@@ -12,6 +12,16 @@ import {
} from "@paperclipai/db";
import type { Config } from "../config.js";
import { resolvePaperclipInstanceId } from "../home-paths.js";
import {
workspaceLoginHandoffPlugin,
type WorkspaceHandoffExpectedIdentity,
} from "./workspace-login-handoff-plugin.js";
import {
normalizeWorkspaceHandoffOrigin,
resolveWorkspaceHandoffLocalCompanyId,
resolveWorkspaceHandoffLocalKey,
resolveWorkspaceHandoffLocalWorkspaceId,
} from "./workspace-login-handoff.js";
export type BetterAuthSessionUser = {
id: string;
@@ -144,6 +154,34 @@ export function deriveAuthTrustedOrigins(config: Config, opts?: { listenPort?: n
return Array.from(trustedOrigins);
}
/**
* Identity a managed workspace instance compares an inbound handoff ticket
* against. Every field comes from persisted configuration or injected runtime
* identity — never from request headers — so a spoofed `X-Forwarded-Host` or
* Tailscale identity header cannot retarget a ticket. Returns null when this
* process was not started as a managed workspace, which leaves the exchange
* endpoint unregistered.
*/
export function resolveWorkspaceHandoffIdentity(
config: Config,
env: NodeJS.ProcessEnv = process.env,
): WorkspaceHandoffExpectedIdentity | null {
const key = resolveWorkspaceHandoffLocalKey(env);
if (!key) return null;
const configuredOrigin =
normalizeWorkspaceHandoffOrigin(env.PAPERCLIP_PUBLIC_URL)
?? (config.authBaseUrlMode === "explicit"
? normalizeWorkspaceHandoffOrigin(config.authPublicBaseUrl)
: null);
return {
key,
instanceId: resolvePaperclipInstanceId(),
executionWorkspaceId: resolveWorkspaceHandoffLocalWorkspaceId(env),
companyId: resolveWorkspaceHandoffLocalCompanyId(env),
origin: configuredOrigin,
};
}
export function createBetterAuthInstance(db: Db, config: Config, trustedOrigins: string[]): BetterAuthInstance {
const baseUrl = config.authBaseUrlMode === "explicit" ? config.authPublicBaseUrl : undefined;
const publicUrl = process.env.PAPERCLIP_PUBLIC_URL?.trim() || baseUrl;
@@ -186,6 +224,28 @@ export function createBetterAuthInstance(db: Db, config: Config, trustedOrigins:
override: process.env.PAPERCLIP_AUTH_RATE_LIMIT_ENABLED,
}),
advanced: buildBetterAuthAdvancedOptions({ disableSecureCookies }),
// Registered only for a managed workspace instance: the plugin is what makes
// `Open workspace` password-independent, and a control-plane instance that
// was never handed a workspace key must not expose the exchange at all.
...(resolveWorkspaceHandoffIdentity(config)
? {
plugins: [
workspaceLoginHandoffPlugin({
db,
// Re-resolved per exchange so a hot restart cannot keep validating
// against an origin the control plane has since republished.
resolveExpectedIdentity: () =>
resolveWorkspaceHandoffIdentity(config) ?? {
key: null,
instanceId: null,
executionWorkspaceId: null,
companyId: null,
origin: null,
},
}),
],
}
: {}),
};
if (!baseUrl) {
@@ -0,0 +1,127 @@
/**
* Ticket exchange, isolated from Better Auth and Express so the whole
* security matrix (replay, expiry, wrong user/origin/workspace/instance,
* missing membership, spoofed forwarded headers) is unit-testable.
*
* Every dependency is injected. The caller supplies expected identity from
* *configuration*; this module never reads request headers, which is what makes
* forwarded-header spoofing structurally impossible rather than merely checked.
*/
import {
sanitizeWorkspaceHandoffRedirectPath,
verifyWorkspaceHandoffTicket,
type WorkspaceHandoffTicketPayload,
type WorkspaceHandoffVerificationFailure,
} from "./workspace-login-handoff.js";
export type WorkspaceHandoffClonedIdentity = {
userId: string;
email: string | null;
name?: string | null;
/**
* Whether the cloned user has an active membership **in the ticket's company**.
* An unscoped "has some membership" answer would let the exchange hand out a
* session with no access to the board the caller was opening.
*/
hasActiveMembership: boolean;
};
export type WorkspaceHandoffExchangeDeps = {
/** Resolve the cloned user by the id the ticket names, scoped to its company. */
findClonedIdentity: (
input: { userId: string; companyId: string },
) => Promise<WorkspaceHandoffClonedIdentity | null>;
/**
* Record the nonce, returning false when it was already recorded.
* Must be atomic: this is the only defense against a same-window replay.
*/
reserveNonce: (input: { nonce: string; expiresAt: Date }) => Promise<boolean>;
/** Create an instance-scoped session for the verified cloned user. */
createSession: (userId: string) => Promise<void>;
};
export type WorkspaceHandoffExchangeFailureReason =
| WorkspaceHandoffVerificationFailure
| "replayed"
| "unknown_user"
| "user_mismatch"
| "missing_membership"
| "session_failed";
export type WorkspaceHandoffExchangeResult =
| { ok: true; redirectTo: string; payload: WorkspaceHandoffTicketPayload }
| { ok: false; reason: WorkspaceHandoffExchangeFailureReason; payload: WorkspaceHandoffTicketPayload | null };
/**
* HTTP status for a failed exchange.
*
* `not_configured` is the only server-fault case (the guest was started without
* a key); everything else is a rejected credential and answers 401 so an
* attacker cannot distinguish "wrong workspace" from "expired" by status alone.
*/
export function workspaceHandoffFailureStatus(reason: WorkspaceHandoffExchangeFailureReason): number {
if (reason === "not_configured") return 503;
if (reason === "session_failed") return 500;
return 401;
}
/**
* Whether a failure means the workspace itself needs repair rather than the
* caller needing a fresh ticket. The UI turns these into an explicit
* "clone is incomplete" state instead of a generic sign-in error.
*/
export function isWorkspaceHandoffRepairReason(reason: WorkspaceHandoffExchangeFailureReason): boolean {
return reason === "unknown_user" || reason === "missing_membership" || reason === "user_mismatch";
}
export async function exchangeWorkspaceHandoffTicket(input: {
ticket: string | null | undefined;
key: string | null | undefined;
expected: {
instanceId: string | null;
executionWorkspaceId: string | null;
companyId: string | null;
origin: string | null;
};
deps: WorkspaceHandoffExchangeDeps;
now?: Date;
}): Promise<WorkspaceHandoffExchangeResult> {
const verification = verifyWorkspaceHandoffTicket({
ticket: input.ticket,
key: input.key,
expected: input.expected,
now: input.now,
});
if (!verification.ok) return { ok: false, reason: verification.reason, payload: null };
const payload = verification.payload;
const identity = await input.deps.findClonedIdentity({ userId: payload.sub, companyId: payload.cid });
if (!identity) return { ok: false, reason: "unknown_user", payload };
// The email is bound into the signature, so a mismatch means the clone drifted
// from the source instance (a reused id after a reseed), not a forged ticket.
if ((identity.email ?? "").trim().toLowerCase() !== payload.email.trim().toLowerCase()) {
return { ok: false, reason: "user_mismatch", payload };
}
if (!identity.hasActiveMembership) return { ok: false, reason: "missing_membership", payload };
// Reserve last: an identity failure must stay retryable after a repair, while a
// ticket that got far enough to mint a session is burned even if that mint fails.
const reserved = await input.deps.reserveNonce({
nonce: payload.jti,
expiresAt: new Date(payload.exp * 1000),
});
if (!reserved) return { ok: false, reason: "replayed", payload };
try {
await input.deps.createSession(identity.userId);
} catch {
return { ok: false, reason: "session_failed", payload };
}
return {
ok: true,
redirectTo: sanitizeWorkspaceHandoffRedirectPath(payload.next),
payload,
};
}
@@ -0,0 +1,222 @@
/**
* Better Auth plugin that exchanges a signed workspace login handoff ticket for
* an instance-scoped session (PAP-17572).
*
* Implemented as a plugin endpoint rather than a hand-rolled Express route so
* session creation and cookie signing go through Better Auth's own supported
* path (`internalAdapter.createSession` + `setSessionCookie`) instead of this
* repository reimplementing cookie signing.
*
* The endpoint is only registered when the process was handed a per-workspace
* verification key, so a normal control-plane instance exposes no extra surface.
*/
import { and, eq } from "drizzle-orm";
import { setSessionCookie } from "better-auth/cookies";
import { createAuthEndpoint } from "better-auth/api";
import type { Session, User } from "better-auth/types";
import type { Db } from "@paperclipai/db";
import { companyMemberships } from "@paperclipai/db";
import { logger } from "../middleware/logger.js";
import {
exchangeWorkspaceHandoffTicket,
isWorkspaceHandoffRepairReason,
type WorkspaceHandoffExchangeFailureReason,
} from "./workspace-login-handoff-exchange.js";
import {
WORKSPACE_HANDOFF_EXCHANGE_PATH,
WORKSPACE_HANDOFF_TICKET_QUERY_PARAM,
workspaceHandoffKeyFingerprint,
} from "./workspace-login-handoff.js";
/**
* Identity the guest expects, resolved from persisted configuration.
*
* Deliberately a callback: it is re-read per exchange so a hot-restarted guest
* cannot keep validating against a stale origin, and it keeps request headers
* out of the comparison entirely.
*/
export type WorkspaceHandoffExpectedIdentity = {
key: string | null;
instanceId: string | null;
executionWorkspaceId: string | null;
companyId: string | null;
origin: string | null;
};
/**
* The slice of Better Auth's endpoint context this exchange uses.
*
* Declared structurally rather than importing the library's deeply generic
* `GenericEndpointContext`, whose inference depends on the plugin's own option
* types. Narrowing here keeps the handler body fully type-checked; the single
* cast at the top of the handler is the only unchecked step.
*/
type WorkspaceHandoffEndpointContext = {
query?: Record<string, unknown>;
setHeader: (name: string, value: string) => void;
redirect: (url: string) => unknown;
error: (status: string, body?: Record<string, unknown>) => unknown;
context: {
internalAdapter: {
findUserById: (userId: string) => Promise<User | null>;
createSession: (userId: string) => Promise<Session | null>;
findVerificationValue: (identifier: string) => Promise<{ identifier: string } | null>;
reserveVerificationValue: (data: {
identifier: string;
value: string;
expiresAt: Date;
}) => Promise<boolean>;
};
};
};
/** Where a rejected ticket lands the browser: the labeled snapshot-local fallback. */
function buildFallbackRedirect(input: {
baseUrl: string | undefined;
next: string;
reason: WorkspaceHandoffExchangeFailureReason;
}): string {
const target = new URL("/auth", input.baseUrl ?? "http://localhost");
target.searchParams.set("next", input.next);
target.searchParams.set("workspaceHandoffError", input.reason);
return input.baseUrl ? target.toString() : `${target.pathname}${target.search}`;
}
export function workspaceLoginHandoffPlugin(deps: {
db: Db;
resolveExpectedIdentity: () => WorkspaceHandoffExpectedIdentity;
}) {
return {
id: "paperclip-workspace-login-handoff",
endpoints: {
exchangeWorkspaceLoginHandoff: createAuthEndpoint(
WORKSPACE_HANDOFF_EXCHANGE_PATH,
{ method: "GET", requireHeaders: true },
async (endpointContext) => {
const ctx = endpointContext as unknown as WorkspaceHandoffEndpointContext;
const expected = deps.resolveExpectedIdentity();
const rawTicket = ctx.query?.[WORKSPACE_HANDOFF_TICKET_QUERY_PARAM];
const ticket = typeof rawTicket === "string" ? rawTicket : null;
// `no-store` keeps the ticket-bearing response out of shared caches and
// `no-referrer` stops the ticket URL from leaking to the landing page.
ctx.setHeader("Cache-Control", "no-store");
ctx.setHeader("Referrer-Policy", "no-referrer");
const result = await exchangeWorkspaceHandoffTicket({
ticket,
key: expected.key,
expected: {
instanceId: expected.instanceId,
executionWorkspaceId: expected.executionWorkspaceId,
companyId: expected.companyId,
origin: expected.origin,
},
deps: {
findClonedIdentity: async ({ userId, companyId }) => {
const user = await ctx.context.internalAdapter.findUserById(userId);
if (!user) return null;
// Membership lives in Paperclip's own schema, so it is read
// through the app's `db` handle rather than the Better Auth
// adapter. Scoped to the ticket's company: an unscoped check
// would accept a clone where this user belongs to some *other*
// company and hand them a session with no access to the board
// they opened. Existence, not a count.
const activeMemberships = await deps.db
.select({ id: companyMemberships.id })
.from(companyMemberships)
.where(
and(
eq(companyMemberships.companyId, companyId),
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, userId),
eq(companyMemberships.status, "active"),
),
)
.limit(1)
.then((rows) => rows.length);
return {
userId: user.id,
email: user.email ?? null,
name: user.name ?? null,
hasActiveMembership: activeMemberships > 0,
};
},
reserveNonce: async ({ nonce, expiresAt }) => {
const identifier = `workspace-login-handoff:${nonce}`;
// Two layers, because each covers a case the other misses. The
// lookup catches an ordinary sequential replay on any storage
// backend; the reservation is a primary-key insert, so it is the
// one that survives two exchanges racing inside the TTL. Relying
// on the reservation alone would silently depend on the
// configured adapter enforcing id uniqueness.
if (await ctx.context.internalAdapter.findVerificationValue(identifier)) return false;
return await ctx.context.internalAdapter.reserveVerificationValue({
identifier,
value: "consumed",
// Keep the record slightly past the ticket's own expiry so a
// replay attempted at the very edge of the window still loses.
expiresAt: new Date(expiresAt.getTime() + 60_000),
});
},
createSession: async (userId) => {
const user = await ctx.context.internalAdapter.findUserById(userId);
if (!user) throw new Error("cloned user disappeared between verification and session creation");
const session = await ctx.context.internalAdapter.createSession(userId);
if (!session) throw new Error("Better Auth did not return a session");
await setSessionCookie(ctx as never, { session, user });
},
},
});
if (!result.ok) {
// Audit with the nonce and reason but never the ticket itself.
logger.warn(
{
reason: result.reason,
nonce: result.payload?.jti ?? null,
executionWorkspaceId: expected.executionWorkspaceId,
instanceId: expected.instanceId,
keyFingerprint: expected.key ? workspaceHandoffKeyFingerprint(expected.key) : null,
needsRepair: isWorkspaceHandoffRepairReason(result.reason),
},
"workspace login handoff rejected",
);
if (result.reason === "not_configured" || result.reason === "session_failed") {
throw ctx.error(result.reason === "not_configured" ? "SERVICE_UNAVAILABLE" : "INTERNAL_SERVER_ERROR", {
message: "Workspace login handoff is unavailable",
code: result.reason,
});
}
throw ctx.redirect(
buildFallbackRedirect({
baseUrl: expected.origin ?? undefined,
next: result.payload?.next ?? "/",
reason: result.reason,
}),
);
}
logger.info(
{
nonce: result.payload.jti,
userId: result.payload.sub,
executionWorkspaceId: expected.executionWorkspaceId,
instanceId: expected.instanceId,
issuerInstanceId: result.payload.iss,
},
"workspace login handoff accepted",
);
// An HTTP redirect (not a client-side navigation) is what keeps the
// ticket URL out of the browser's session history.
throw ctx.redirect(
expected.origin
? new URL(result.redirectTo, expected.origin).toString()
: result.redirectTo,
);
},
),
},
};
}
+406
View File
@@ -0,0 +1,406 @@
/**
* Signed workspace login handoff (PAP-17572).
*
* A board user opening a managed workspace must not have to know which cloned
* password is current. The authenticated main control plane mints a short-lived,
* single-use ticket; the isolated workspace verifies it against its own copy of
* the signing key and creates its own instance-scoped session.
*
* Threat model notes that drive the shape below:
*
* - **Nothing browser-supplied is trusted.** The audience is the origin the
* control plane published for that runtime and the workspace compares it
* against its *configured* public origin. `Host`, `X-Forwarded-Host`,
* `X-Forwarded-Proto` and Tailscale identity headers are never consulted, so
* a spoofed forwarded header cannot retarget or authorize a ticket.
* - **Keys are per workspace.** The control plane keeps the root secret and
* hands each guest only `HMAC(root, instanceId|executionWorkspaceId)`. A
* compromised workspace therefore cannot mint a ticket for a sibling
* workspace, and the guest never sees material that would let it do so.
* - **Replay is bounded twice.** A short expiry limits the window and the
* workspace records the nonce on first use, so a captured ticket cannot be
* exchanged a second time even inside that window.
*/
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
/** Ticket envelope version. Bumped whenever the signed payload shape changes. */
export const WORKSPACE_HANDOFF_TICKET_VERSION = "wh1";
/**
* Default ticket lifetime. Long enough for a click plus a cold guest response,
* short enough that a ticket leaked through a proxy log is worthless.
*/
export const WORKSPACE_HANDOFF_TICKET_TTL_SECONDS = 90;
/** Tolerated clock skew between the control plane and the isolated workspace. */
export const WORKSPACE_HANDOFF_CLOCK_SKEW_SECONDS = 30;
/** Path the guest exposes for the exchange, relative to the Better Auth mount. */
export const WORKSPACE_HANDOFF_EXCHANGE_PATH = "/workspace-handoff/exchange";
/** Query parameter carrying the ticket. Redacted from request logs by name. */
export const WORKSPACE_HANDOFF_TICKET_QUERY_PARAM = "ticket";
const HANDOFF_KEY_ENV = "PAPERCLIP_WORKSPACE_HANDOFF_KEY";
const HANDOFF_ROOT_SECRET_ENV = "PAPERCLIP_WORKSPACE_HANDOFF_SECRET";
const READINESS_TOKEN_ENV = "PAPERCLIP_WORKSPACE_READINESS_TOKEN";
const EXECUTION_WORKSPACE_ID_ENV = "PAPERCLIP_EXECUTION_WORKSPACE_ID";
const EXECUTION_WORKSPACE_COMPANY_ID_ENV = "PAPERCLIP_EXECUTION_WORKSPACE_COMPANY_ID";
export const WORKSPACE_HANDOFF_KEY_ENV_KEY = HANDOFF_KEY_ENV;
export const WORKSPACE_READINESS_TOKEN_ENV_KEY = READINESS_TOKEN_ENV;
export const WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY = EXECUTION_WORKSPACE_ID_ENV;
export const WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY = EXECUTION_WORKSPACE_COMPANY_ID_ENV;
export const WORKSPACE_READINESS_TOKEN_HEADER = "x-paperclip-workspace-readiness-token";
export const WORKSPACE_READINESS_USER_ID_HEADER = "x-paperclip-workspace-readiness-user-id";
export const WORKSPACE_READINESS_USER_EMAIL_HEADER = "x-paperclip-workspace-readiness-user-email";
export type WorkspaceHandoffTicketPayload = {
/** Envelope version. */
v: typeof WORKSPACE_HANDOFF_TICKET_VERSION;
/** Single-use nonce; the guest records it on first exchange. */
jti: string;
/** Cloned user id the guest must sign in. */
sub: string;
/** Expected email of that cloned user, compared case-insensitively. */
email: string;
/** Execution workspace the ticket is scoped to. */
ws: string;
/**
* Company whose board this workspace represents.
*
* Bound because "the cloned user has *some* active membership" is not the
* question worth asking: a multi-company clone can satisfy it while leaving the
* signed-in user with no access to the company they were opening, which lands
* them on an empty board with no explanation.
*/
cid: string;
/** Isolated instance id the ticket is scoped to. */
iid: string;
/** Origin the control plane published for this runtime. */
aud: string;
/** Issuing instance id, recorded for audit. */
iss: string;
/** Seconds since epoch. */
iat: number;
exp: number;
/** Relative path to land on after the exchange. */
next: string;
};
export type WorkspaceHandoffVerificationFailure =
| "not_configured"
| "malformed"
| "unsupported_version"
| "bad_signature"
| "expired"
| "not_yet_valid"
| "origin_mismatch"
| "instance_mismatch"
| "workspace_mismatch"
| "company_mismatch";
export type WorkspaceHandoffVerificationResult =
| { ok: true; payload: WorkspaceHandoffTicketPayload }
| { ok: false; reason: WorkspaceHandoffVerificationFailure };
function base64UrlEncode(value: Buffer): string {
return value.toString("base64url");
}
function base64UrlDecode(value: string): Buffer | null {
// `Buffer.from` silently drops invalid characters, so round-trip to prove the
// input really was canonical base64url before trusting the bytes.
if (!/^[A-Za-z0-9_-]+$/.test(value)) return null;
const decoded = Buffer.from(value, "base64url");
if (decoded.length === 0) return null;
return base64UrlEncode(decoded) === value ? decoded : null;
}
function sign(key: string, message: string): Buffer {
return createHmac("sha256", key).update(message).digest();
}
function constantTimeMatches(expected: Buffer, provided: Buffer): boolean {
if (expected.length !== provided.length) return false;
return timingSafeEqual(expected, provided);
}
/**
* Normalize an origin for comparison: scheme + host + explicit non-default port.
* Returns null when the input is not an absolute http(s) URL, so an unparseable
* or relative value can never accidentally compare equal.
*/
export function normalizeWorkspaceHandoffOrigin(value: string | null | undefined): string | null {
const trimmed = value?.trim();
if (!trimmed) return null;
let parsed: URL;
try {
parsed = new URL(trimmed);
} catch {
return null;
}
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return null;
return parsed.origin;
}
/**
* Reduce a caller-supplied landing target to a safe same-origin path.
*
* Anything absolute, protocol-relative, or backslash-escaped collapses to `/`.
* A handoff must never double as an open redirect.
*/
export function sanitizeWorkspaceHandoffRedirectPath(value: string | null | undefined): string {
const trimmed = value?.trim();
if (!trimmed) return "/";
if (!trimmed.startsWith("/")) return "/";
// `//host` and `/\host` are both browser-recognized protocol-relative forms.
if (trimmed.startsWith("//") || trimmed.startsWith("/\\")) return "/";
// Control characters and raw whitespace can split a `Location` header.
if (/[\u0000-\u0020\u007f]/.test(trimmed)) return "/";
return trimmed;
}
/**
* Derive the per-workspace signing key handed to one guest instance.
*
* Domain-separated from every other use of the root secret and bound to both
* identities, so a key only ever validates tickets for the exact workspace it
* was issued to.
*/
export function deriveWorkspaceHandoffKey(input: {
rootSecret: string;
instanceId: string;
executionWorkspaceId: string;
}): string {
return createHmac("sha256", input.rootSecret)
.update(`paperclip.workspace-login-handoff.${WORKSPACE_HANDOFF_TICKET_VERSION}\n`)
.update(`${input.instanceId}\n`)
.update(`${input.executionWorkspaceId}`)
.digest("hex");
}
/**
* Derive the bearer token the control plane presents to read a guest's
* protected readiness. Separate derivation from {@link deriveWorkspaceHandoffKey}
* so a probe token that leaks through an HTTP client log cannot sign tickets.
*/
export function deriveWorkspaceReadinessToken(input: {
rootSecret: string;
instanceId: string;
executionWorkspaceId: string;
}): string {
return createHmac("sha256", input.rootSecret)
.update(`paperclip.workspace-readiness-probe.${WORKSPACE_HANDOFF_TICKET_VERSION}\n`)
.update(`${input.instanceId}\n`)
.update(`${input.executionWorkspaceId}`)
.digest("hex");
}
/**
* Root secret for handoff key derivation, on the control-plane side only.
*
* A dedicated `PAPERCLIP_WORKSPACE_HANDOFF_SECRET` is preferred. When it is
* absent we derive dedicated key material from the instance's existing signing
* secret rather than disabling the feature: the alternative is that every
* already-deployed instance silently falls back to "remember the cloned
* password", which is the defect this work exists to remove. The derivation is
* one-way and domain-separated, so the handoff key is never equal to — and
* cannot be used to recover — the secret it came from.
*/
export function resolveWorkspaceHandoffRootSecret(
env: NodeJS.ProcessEnv = process.env,
): { secret: string; source: "dedicated" | "derived" } | null {
const dedicated = env[HANDOFF_ROOT_SECRET_ENV]?.trim();
if (dedicated) return { secret: dedicated, source: "dedicated" };
const fallback = env.BETTER_AUTH_SECRET?.trim() || env.PAPERCLIP_AGENT_JWT_SECRET?.trim();
if (!fallback) return null;
return {
secret: createHmac("sha256", fallback)
.update(`paperclip.workspace-login-handoff.root.${WORKSPACE_HANDOFF_TICKET_VERSION}`)
.digest("hex"),
source: "derived",
};
}
/** The verification key injected into this (guest) process, when present. */
export function resolveWorkspaceHandoffLocalKey(env: NodeJS.ProcessEnv = process.env): string | null {
return env[HANDOFF_KEY_ENV]?.trim() || null;
}
/** The readiness-probe token injected into this (guest) process, when present. */
export function resolveWorkspaceReadinessLocalToken(env: NodeJS.ProcessEnv = process.env): string | null {
return env[READINESS_TOKEN_ENV]?.trim() || null;
}
/** The execution workspace this (guest) process was provisioned for. */
export function resolveWorkspaceHandoffLocalWorkspaceId(env: NodeJS.ProcessEnv = process.env): string | null {
return env[EXECUTION_WORKSPACE_ID_ENV]?.trim() || null;
}
/** The company whose board this (guest) workspace represents. */
export function resolveWorkspaceHandoffLocalCompanyId(env: NodeJS.ProcessEnv = process.env): string | null {
return env[EXECUTION_WORKSPACE_COMPANY_ID_ENV]?.trim() || null;
}
export function issueWorkspaceHandoffTicket(input: {
key: string;
userId: string;
email: string;
executionWorkspaceId: string;
companyId: string;
instanceId: string;
origin: string;
issuerInstanceId: string;
next?: string | null;
ttlSeconds?: number;
now?: Date;
nonce?: string;
}): { ticket: string; payload: WorkspaceHandoffTicketPayload } {
const origin = normalizeWorkspaceHandoffOrigin(input.origin);
if (!origin) {
throw new Error("Workspace login handoff needs an absolute http(s) target origin");
}
const issuedAtMs = (input.now ?? new Date()).getTime();
const issuedAt = Math.floor(issuedAtMs / 1000);
const ttlSeconds = Math.max(1, Math.floor(input.ttlSeconds ?? WORKSPACE_HANDOFF_TICKET_TTL_SECONDS));
const payload: WorkspaceHandoffTicketPayload = {
v: WORKSPACE_HANDOFF_TICKET_VERSION,
jti: input.nonce ?? randomBytes(18).toString("base64url"),
sub: input.userId,
email: input.email,
ws: input.executionWorkspaceId,
cid: input.companyId,
iid: input.instanceId,
aud: origin,
iss: input.issuerInstanceId,
iat: issuedAt,
exp: issuedAt + ttlSeconds,
next: sanitizeWorkspaceHandoffRedirectPath(input.next),
};
const encodedPayload = base64UrlEncode(Buffer.from(JSON.stringify(payload), "utf8"));
const signedMessage = `${WORKSPACE_HANDOFF_TICKET_VERSION}.${encodedPayload}`;
const signature = base64UrlEncode(sign(input.key, signedMessage));
return { ticket: `${signedMessage}.${signature}`, payload };
}
/** Absolute exchange URL for a minted ticket. */
export function buildWorkspaceHandoffExchangeUrl(input: { origin: string; ticket: string }): string {
const origin = normalizeWorkspaceHandoffOrigin(input.origin);
if (!origin) throw new Error("Workspace login handoff needs an absolute http(s) target origin");
const url = new URL(`/api/auth${WORKSPACE_HANDOFF_EXCHANGE_PATH}`, origin);
url.searchParams.set(WORKSPACE_HANDOFF_TICKET_QUERY_PARAM, input.ticket);
return url.toString();
}
function isPayloadShape(value: unknown): value is WorkspaceHandoffTicketPayload {
if (!value || typeof value !== "object") return false;
const candidate = value as Record<string, unknown>;
return (
typeof candidate.jti === "string" && candidate.jti.length > 0
&& typeof candidate.sub === "string" && candidate.sub.length > 0
&& typeof candidate.email === "string" && candidate.email.length > 0
&& typeof candidate.ws === "string" && candidate.ws.length > 0
&& typeof candidate.cid === "string" && candidate.cid.length > 0
&& typeof candidate.iid === "string" && candidate.iid.length > 0
&& typeof candidate.aud === "string" && candidate.aud.length > 0
&& typeof candidate.iss === "string"
&& Number.isFinite(candidate.iat)
&& Number.isFinite(candidate.exp)
&& typeof candidate.next === "string"
);
}
/**
* Verify a ticket against this workspace's own key and identity.
*
* `expected` must come from persisted configuration. Passing header-derived
* values would reintroduce exactly the spoofing path this design closes.
*/
export function verifyWorkspaceHandoffTicket(input: {
ticket: string | null | undefined;
key: string | null | undefined;
expected: {
instanceId: string | null;
executionWorkspaceId: string | null;
companyId: string | null;
origin: string | null;
};
now?: Date;
clockSkewSeconds?: number;
}): WorkspaceHandoffVerificationResult {
const key = input.key?.trim();
if (!key) return { ok: false, reason: "not_configured" };
const expectedOrigin = normalizeWorkspaceHandoffOrigin(input.expected.origin);
const expectedInstanceId = input.expected.instanceId?.trim();
const expectedWorkspaceId = input.expected.executionWorkspaceId?.trim();
const expectedCompanyId = input.expected.companyId?.trim();
// Fail closed on unresolved local identity: without it the checks below would
// degrade into "any validly signed ticket wins".
if (!expectedOrigin) return { ok: false, reason: "origin_mismatch" };
if (!expectedInstanceId) return { ok: false, reason: "instance_mismatch" };
if (!expectedWorkspaceId) return { ok: false, reason: "workspace_mismatch" };
if (!expectedCompanyId) return { ok: false, reason: "company_mismatch" };
const ticket = input.ticket?.trim();
if (!ticket) return { ok: false, reason: "malformed" };
const segments = ticket.split(".");
if (segments.length !== 3) return { ok: false, reason: "malformed" };
const [version, encodedPayload, encodedSignature] = segments as [string, string, string];
if (version !== WORKSPACE_HANDOFF_TICKET_VERSION) return { ok: false, reason: "unsupported_version" };
const providedSignature = base64UrlDecode(encodedSignature);
if (!providedSignature) return { ok: false, reason: "malformed" };
const expectedSignature = sign(key, `${version}.${encodedPayload}`);
// Signature first: never parse attacker-controlled JSON we have not authenticated.
if (!constantTimeMatches(expectedSignature, providedSignature)) return { ok: false, reason: "bad_signature" };
const decodedPayload = base64UrlDecode(encodedPayload);
if (!decodedPayload) return { ok: false, reason: "malformed" };
let parsed: unknown;
try {
parsed = JSON.parse(decodedPayload.toString("utf8"));
} catch {
return { ok: false, reason: "malformed" };
}
if (!isPayloadShape(parsed)) return { ok: false, reason: "malformed" };
if (parsed.v !== WORKSPACE_HANDOFF_TICKET_VERSION) return { ok: false, reason: "unsupported_version" };
const nowSeconds = Math.floor((input.now ?? new Date()).getTime() / 1000);
const skew = Math.max(0, Math.floor(input.clockSkewSeconds ?? WORKSPACE_HANDOFF_CLOCK_SKEW_SECONDS));
if (parsed.exp + skew < nowSeconds) return { ok: false, reason: "expired" };
if (parsed.iat - skew > nowSeconds) return { ok: false, reason: "not_yet_valid" };
if (parsed.exp <= parsed.iat) return { ok: false, reason: "malformed" };
if (normalizeWorkspaceHandoffOrigin(parsed.aud) !== expectedOrigin) {
return { ok: false, reason: "origin_mismatch" };
}
if (parsed.iid !== expectedInstanceId) return { ok: false, reason: "instance_mismatch" };
if (parsed.ws !== expectedWorkspaceId) return { ok: false, reason: "workspace_mismatch" };
if (parsed.cid !== expectedCompanyId) return { ok: false, reason: "company_mismatch" };
return {
ok: true,
payload: { ...parsed, next: sanitizeWorkspaceHandoffRedirectPath(parsed.next) },
};
}
/** Verification-key identifier safe to log: proves which key, reveals no key bytes. */
export function workspaceHandoffKeyFingerprint(key: string): string {
return createHmac("sha256", "paperclip.workspace-login-handoff.fingerprint").update(key).digest("hex").slice(0, 12);
}
/**
* Strip the ticket from a URL or URL-ish string before it reaches a log sink.
* Applied by the request logger and by the guest's own audit records.
*/
export function redactWorkspaceHandoffTicket(value: string): string {
if (!value.includes(WORKSPACE_HANDOFF_TICKET_QUERY_PARAM)) return value;
return value.replace(
new RegExp(`([?&]${WORKSPACE_HANDOFF_TICKET_QUERY_PARAM}=)[^&#\\s]+`, "gi"),
"$1[redacted]",
);
}
+17
View File
@@ -69,6 +69,7 @@ export interface Config {
databaseBackupIntervalMinutes: number;
databaseBackupRetentionDays: number;
databaseBackupDir: string;
workspaceReaperCooldownDays: number;
serveUi: boolean;
uiDevMiddleware: boolean;
secretsProvider: SecretProvider;
@@ -265,6 +266,21 @@ export function loadConfig(): Config {
fileDatabaseBackup?.dir ??
resolveDefaultBackupDir(),
);
// The terminal-workspace reaper waits this many days after an issue tree
// becomes terminal before it archives the workspace. A person can reopen the
// work inside this window. A value of 0 disables the cooldown and restores
// immediate reaping. A negative or non-numeric value falls back to the
// default. The day granularity and the default of 7 obey the
// PAPERCLIP_DB_BACKUP_RETENTION_DAYS precedent above.
const workspaceReaperCooldownDaysEnv =
process.env.PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS?.trim();
const workspaceReaperCooldownDaysRaw = Number(workspaceReaperCooldownDaysEnv);
const workspaceReaperCooldownDays =
workspaceReaperCooldownDaysEnv
&& Number.isFinite(workspaceReaperCooldownDaysRaw)
&& workspaceReaperCooldownDaysRaw >= 0
? workspaceReaperCooldownDaysRaw
: 7;
const bindValidationErrors = validateConfiguredBindMode({
deploymentMode,
deploymentExposure,
@@ -307,6 +323,7 @@ export function loadConfig(): Config {
databaseBackupIntervalMinutes,
databaseBackupRetentionDays,
databaseBackupDir,
workspaceReaperCooldownDays,
serveUi:
process.env.SERVE_UI !== undefined
? process.env.SERVE_UI === "true"
+5
View File
@@ -1,6 +1,7 @@
import { existsSync, lstatSync, readFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { hasVerifiedWorktreeSeedManifest } from "./worktree-seed-manifest.js";
function parseEnvFile(contents: string): Record<string, string> {
const entries: Record<string, string> = {};
@@ -58,6 +59,10 @@ export function resolveWorktreeEnvFilePath(rootDir: string): string {
export function isWorktreeSeedPending(rootDir: string): boolean {
const markerDir = path.resolve(rootDir, ".paperclip");
const manifestPath = path.resolve(markerDir, "seed-manifest.json");
if (existsSync(manifestPath)) {
return !hasVerifiedWorktreeSeedManifest(manifestPath);
}
return existsSync(path.resolve(markerDir, "seed-pending"))
&& !existsSync(path.resolve(markerDir, "seed-complete"));
}
@@ -0,0 +1,92 @@
export type EmbeddedPostgresExitListener = (code: number | null, signal: NodeJS.Signals | null) => void;
export interface SupervisedEmbeddedPostgres {
start(): Promise<void>;
stop(): Promise<void>;
process?: { once(event: "exit", listener: EmbeddedPostgresExitListener): unknown };
}
export interface EmbeddedPostgresSupervisor {
current(): SupervisedEmbeddedPostgres;
shutdown(): Promise<void>;
waitForRecovery(): Promise<void>;
}
type Options = {
initialInstance: SupervisedEmbeddedPostgres;
createInstance: () => SupervisedEmbeddedPostgres;
beforeRestart?: (attempt: number) => Promise<void> | void;
restartDelaysMs?: number[];
delay?: (milliseconds: number) => Promise<void>;
onUnexpectedExit?: EmbeddedPostgresExitListener;
onRestartAttemptFailed?: (error: unknown, attempt: number) => void;
onRestarted?: (attempt: number) => void;
onRecoveryExhausted?: (error: unknown) => void;
};
const defaultDelay = (milliseconds: number) => new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
export function createEmbeddedPostgresSupervisor(options: Options): EmbeddedPostgresSupervisor {
const restartDelaysMs = options.restartDelaysMs ?? [0, 250, 1_000];
const wait = options.delay ?? defaultDelay;
let activeInstance = options.initialInstance;
let activeInstanceExited = false;
let shuttingDown = false;
let recoveryPromise: Promise<void> | null = null;
const recover = async () => {
let lastError: unknown = new Error("Embedded PostgreSQL exited unexpectedly");
for (let index = 0; index < restartDelaysMs.length; index += 1) {
if (shuttingDown) return;
const attempt = index + 1;
const delayMs = restartDelaysMs[index] ?? 0;
if (delayMs > 0) await wait(delayMs);
if (shuttingDown) return;
try {
await options.beforeRestart?.(attempt);
const replacement = options.createInstance();
await replacement.start();
if (shuttingDown) {
await replacement.stop();
return;
}
activeInstance = replacement;
activeInstanceExited = false;
monitor(replacement);
options.onRestarted?.(attempt);
return;
} catch (error) {
lastError = error;
options.onRestartAttemptFailed?.(error, attempt);
}
}
if (!shuttingDown) options.onRecoveryExhausted?.(lastError);
};
const monitor = (instance: SupervisedEmbeddedPostgres) => {
const child = instance.process;
if (!child) {
options.onRecoveryExhausted?.(new Error("Embedded PostgreSQL started without a child process to monitor"));
return;
}
child.once("exit", (code, signal) => {
if (activeInstance !== instance) return;
activeInstanceExited = true;
if (shuttingDown) return;
options.onUnexpectedExit?.(code, signal);
recoveryPromise = recover().finally(() => { recoveryPromise = null; });
});
};
monitor(activeInstance);
return {
current: () => activeInstance,
waitForRecovery: async () => { await recoveryPromise; },
shutdown: async () => {
if (shuttingDown) return;
shuttingDown = true;
await recoveryPromise;
if (!activeInstanceExited) await activeInstance.stop();
},
};
}
+85 -9
View File
@@ -72,6 +72,7 @@ import {
createCodexDeviceLoginReaper,
createProductionLoginSessionReaperRuntime,
} from "./services/codex-device-login-reaper.js";
import { createProductionSetupTokenReaper } from "./services/setup-token-reaper.js";
import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js";
import {
parseAdapterRegistryEnv,
@@ -96,6 +97,11 @@ import {
} from "./shutdown.js";
import { systemdNotify } from "./services/systemd-notify.js";
import { flushInFlightRunLogMirrors } from "./services/run-log-store.js";
import {
createEmbeddedPostgresSupervisor,
type EmbeddedPostgresSupervisor,
type SupervisedEmbeddedPostgres,
} from "./embedded-postgres-supervisor.js";
import type {
InstanceDatabaseBackupRunResult,
InstanceDatabaseBackupTrigger,
@@ -112,10 +118,8 @@ type BetterAuthSessionResult = {
user: BetterAuthSessionUser | null;
};
type EmbeddedPostgresInstance = {
type EmbeddedPostgresInstance = SupervisedEmbeddedPostgres & {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
@@ -326,6 +330,7 @@ export async function startServer(): Promise<StartedServer> {
let db;
let pluginMigrationDb;
let embeddedPostgres: EmbeddedPostgresInstance | null = null;
let embeddedPostgresSupervisor: EmbeddedPostgresSupervisor | null = null;
let embeddedPostgresStartedByThisProcess = false;
let migrationSummary: MigrationSummary = "skipped";
let activeDatabaseConnectionString: string;
@@ -450,7 +455,7 @@ export async function startServer(): Promise<StartedServer> {
}
port = detectedPort;
logger.info(`Using embedded PostgreSQL because no DATABASE_URL set (dataDir=${dataDir}, port=${port})`);
embeddedPostgres = new EmbeddedPostgres({
const createEmbeddedPostgres = () => new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
@@ -460,6 +465,7 @@ export async function startServer(): Promise<StartedServer> {
onLog: appendEmbeddedPostgresLog,
onError: appendEmbeddedPostgresLog,
});
embeddedPostgres = createEmbeddedPostgres();
if (!clusterAlreadyInitialized) {
try {
@@ -489,6 +495,36 @@ export async function startServer(): Promise<StartedServer> {
});
}
embeddedPostgresStartedByThisProcess = true;
embeddedPostgresSupervisor = createEmbeddedPostgresSupervisor({
initialInstance: embeddedPostgres,
createInstance: createEmbeddedPostgres,
beforeRestart: () => {
const runningPostgresPid = getRunningPid();
if (runningPostgresPid) {
throw new Error(`Refusing embedded PostgreSQL recovery because the data directory reports a live process (pid=${runningPostgresPid})`);
}
if (existsSync(postmasterPidFile)) rmSync(postmasterPidFile, { force: true });
},
onUnexpectedExit: (code, signal) => logger.error(
{ code, signal, recentLogs: logBuffer.getRecentLogs() },
"Embedded PostgreSQL exited unexpectedly; attempting recovery",
),
onRestartAttemptFailed: (err, attempt) => logger.error(
{ err, attempt, recentLogs: logBuffer.getRecentLogs() },
"Embedded PostgreSQL recovery attempt failed",
),
onRestarted: (attempt) => logger.info(
{ attempt, port },
"Embedded PostgreSQL recovered after unexpected exit",
),
onRecoveryExhausted: (err) => {
logger.fatal(
{ err, recentLogs: logBuffer.getRecentLogs() },
"Embedded PostgreSQL recovery exhausted; stopping the unhealthy server",
);
process.kill(process.pid, "SIGTERM");
},
});
}
}
@@ -1049,7 +1085,9 @@ export async function startServer(): Promise<StartedServer> {
const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, {
wakeup: heartbeat.wakeup,
});
const terminalWorkspaces = executionWorkspaceService(db as any);
const terminalWorkspaces = executionWorkspaceService(db as any, {
workspaceReaperCooldownDays: config.workspaceReaperCooldownDays,
});
const scheduleMergedPullRequestConfirmationSweep = () => {
if (heartbeatSchedulerStopped) return;
trackHeartbeatSchedulerWork(mergedPullRequestConfirmations
@@ -1081,7 +1119,8 @@ export async function startServer(): Promise<StartedServer> {
result.skippedActiveRun
+ result.skippedNonTerminalTree
+ result.skippedUndelivered
+ result.skippedRace;
+ result.skippedRace
+ result.skippedCooldown;
const nowMs = Date.now();
if (skipped > 0 && nowMs - lastTerminalWorkspaceSkipLogAt >= terminalWorkspaceSkipLogIntervalMs) {
lastTerminalWorkspaceSkipLogAt = nowMs;
@@ -1128,6 +1167,32 @@ export async function startServer(): Promise<StartedServer> {
}));
};
// The restart-safe cleanup backstop for the Claude setup-token login flow. It
// runs on startup and on the scheduler interval, so a sandbox lease survives a
// server restart and a release failure. It releases any lease whose login
// session is terminal, past its deadline, or already consumed.
const setupTokenReaper = createProductionSetupTokenReaper({
db: db as any,
environmentRuntime: environmentRuntimeService(db as any, { pluginWorkerManager }),
log: (line) => logger.info(line),
});
const logSetupTokenReaperResult = (
result: Awaited<ReturnType<typeof setupTokenReaper.sweep>>,
) => {
if (result.released > 0 || result.failed > 0) {
logger.info(result, "setup-token login reaper released leases");
}
};
const scheduleSetupTokenReaperSweep = () => {
if (heartbeatSchedulerStopped) return;
trackHeartbeatSchedulerWork(setupTokenReaper
.sweep()
.then(logSetupTokenReaperResult)
.catch((err) => {
logger.error({ err }, "setup-token login reaper sweep failed");
}));
};
const tools = toolAccessService(db as any, {
deploymentMode: config.deploymentMode,
deploymentExposure: config.deploymentExposure,
@@ -1265,6 +1330,15 @@ export async function startServer(): Promise<StartedServer> {
logger.error({ err }, "startup adapter login reaper sweep failed");
});
// Run the setup-token login reaper once at startup, so a login sandbox lease
// that outlived a server restart releases before timer ticks start.
await setupTokenReaper
.sweep()
.then(logSetupTokenReaperResult)
.catch((err) => {
logger.error({ err }, "startup setup-token login reaper sweep failed");
});
// Retry any orphan sandbox teardown left by a failed acquire before a server
// restart, so a leaked sandbox does not stay allocated across the restart.
await runEnvironmentLeaseCleanupSweep(0);
@@ -1328,6 +1402,7 @@ export async function startServer(): Promise<StartedServer> {
scheduleMergedPullRequestConfirmationSweep();
scheduleTerminalWorkspaceSweep();
scheduleAdapterLoginReaperSweep();
scheduleSetupTokenReaperSweep();
scheduleEnvironmentLeaseCleanupSweep();
if (heartbeatSchedulerStopped) return;
@@ -1632,8 +1707,9 @@ export async function startServer(): Promise<StartedServer> {
const appShutdown = (app as { locals?: { paperclipShutdown?: () => Promise<void> } }).locals
?.paperclipShutdown;
const embeddedPostgresToStop =
embeddedPostgres && embeddedPostgresStartedByThisProcess ? embeddedPostgres : null;
const stopEmbeddedPostgres = embeddedPostgres && embeddedPostgresStartedByThisProcess
? () => embeddedPostgresSupervisor?.shutdown() ?? embeddedPostgres!.stop()
: null;
// Await the ordered application teardown before the process exits. A live
// setup-token login session must stop and release its sandbox lease before
@@ -1642,7 +1718,7 @@ export async function startServer(): Promise<StartedServer> {
await finalizeServerShutdown({
signal,
shutdownAppServices: appShutdown,
stopEmbeddedPostgres: embeddedPostgresToStop ? () => embeddedPostgresToStop.stop() : null,
stopEmbeddedPostgres,
shutdownInstrumentation,
log: logger,
});
+8 -1
View File
@@ -90,6 +90,7 @@ export function errorHandler(
? err.details as Record<string, unknown>
: null;
const redactedSkillPolicyDenial = isRedactedSkillPolicyDenial(details);
const workspaceRepairPreconditionFailure = details?.code === "workspace_repair_precondition_failed";
const structuredConnectionError = new Set([
"user_authorization_required",
"grant_revoked",
@@ -113,13 +114,19 @@ export function errorHandler(
error: err.message,
...(typeof details?.code === "string" ? { code: details.code } : {}),
...(redactedSkillPolicyDenial && typeof details?.reason === "string" ? { reason: details.reason } : {}),
...(workspaceRepairPreconditionFailure && typeof details?.reason === "string" ? { reason: details.reason } : {}),
...(workspaceRepairPreconditionFailure && typeof details?.repairPhase === "string"
? { repairPhase: details.repairPhase }
: {}),
...(typeof details?.remediation === "string" || (structuredConnectionError && details?.remediation && typeof details.remediation === "object")
? { remediation: details.remediation }
: {}),
...(structuredConnectionError && details?.connection ? { connection: details.connection } : {}),
...(structuredConnectionError && details?.subject ? { subject: details.subject } : {}),
...(structuredConnectionError && typeof details?.grantId === "string" ? { grantId: details.grantId } : {}),
...(!redactedSkillPolicyDenial && err.details ? { details: err.details } : {}),
...(!redactedSkillPolicyDenial && !workspaceRepairPreconditionFailure && err.details
? { details: err.details }
: {}),
});
return;
}
+5 -2
View File
@@ -3,6 +3,7 @@ import { pinoHttp } from "pino-http";
import { HTTP_LOG_REDACT_PATHS } from "./http-log-redaction.js";
import { shouldSilenceHttpSuccessLog } from "./http-log-policy.js";
import { redactSensitive } from "./redact-sensitive.js";
import { redactWorkspaceHandoffTicket } from "../auth/workspace-login-handoff.js";
const sharedOpts = {
translateTime: "SYS:HH:MM:ss",
@@ -29,12 +30,14 @@ export const httpLogger = pinoHttp({
return "info";
},
customSuccessMessage(req, res) {
return `${req.method} ${req.url} ${res.statusCode}`;
// A workspace login handoff ticket is a bearer credential that rides in the
// query string, so the request line has to be redacted before it is logged.
return `${req.method} ${redactWorkspaceHandoffTicket(req.url ?? "")} ${res.statusCode}`;
},
customErrorMessage(req, res, err) {
const ctx = (res as any).__errorContext;
const errMsg = ctx?.error?.message || err?.message || (res as any).err?.message || "unknown error";
return `${req.method} ${req.url} ${res.statusCode} — ${errMsg}`;
return `${req.method} ${redactWorkspaceHandoffTicket(req.url ?? "")} ${res.statusCode} — ${errMsg}`;
},
customProps(req, res) {
if (res.statusCode >= 400) {
@@ -45,6 +45,10 @@ const SENSITIVE_KEYS = new Set<string>([
"browsercode",
"authorization_code",
"authorizationcode",
// The workspace login handoff ticket (PAP-17572). It is a signed bearer
// credential carried as a query parameter, so it must never reach a log line
// even though the exchange itself answers 302.
"ticket",
]);
const MAX_DEPTH = 6;
@@ -0,0 +1,107 @@
import type { Request, Response } from "express";
import type { z } from "zod";
/**
* The result of the shared adapter login start-route spine. The spine returns it
* when every guard passes. `ownerUserId` is the immutable session owner from the
* actor. `data` is the validated request body.
*/
export interface AdapterLoginStartResolution<TData> {
ownerUserId: string;
data: TData;
}
/**
* The injected steps of the shared adapter login start-route spine. The Codex
* device login start route and the Claude setup-token login start route both
* provide them. Each step holds the per-flow rule; the spine holds the shared
* order.
*/
export interface AdapterLoginStartSpineInput<TData extends { environmentId: string }> {
req: Request;
res: Response;
/**
* Derives the immutable session owner from the actor. It also runs the company
* access check. It throws a mapped HTTP error for a forbidden actor. The shared
* error handler maps the thrown error to the response.
*/
deriveOwner: () => Promise<string> | string;
/**
* The strict request schema. `.strict()` rejects an unknown field. The spine
* validates the request body against it before any session or lease side
* effect.
*/
requestSchema: z.ZodType<TData>;
/** The fixed 400 text for an invalid request. Each route keeps its own text. */
invalidRequestError: string;
/**
* Merges these fields into the request body before the parse. The Codex route
* injects the adapter type from the path, so the client body needs no adapter
* type field.
*/
requestOverrides?: Record<string, unknown>;
/**
* Runs an optional per-flow guard after the owner derivation and before the
* request validation. The Codex route checks the path adapter type here. It
* throws a mapped HTTP error to reject the request.
*/
guardBeforeValidate?: () => void | Promise<void>;
/**
* Runs an optional per-flow guard after the validation and before the sandbox
* check. The Claude route checks the body adapter type and the transport
* readiness here. It returns `true` when it sends a response; the spine then
* stops and returns null.
*/
guardAfterValidate?: (data: TData) => boolean | Promise<boolean>;
/**
* Checks the sandbox environment. It throws a mapped HTTP error for a missing,
* archived, non-sandbox, or foreign environment.
*/
assertSandbox: (data: TData) => Promise<void>;
}
/**
* Runs the shared start-route spine for an adapter login. It runs the same
* ordered steps for both start routes, and it runs every step before any session
* or lease side effect:
*
* 1. It derives the session owner and runs the company access check.
* 2. It runs the optional pre-validation guard.
* 3. It validates the request body with the strict schema. It sends the fixed
* 400 and returns null on a failure.
* 4. It runs the optional post-validation guard. It returns null when the guard
* sends a response.
* 5. It checks the sandbox environment.
*
* It returns the owner and the validated body when every step passes. It returns
* null when a step sends a response; the caller must then stop.
*/
export async function runAdapterLoginStartSpine<TData extends { environmentId: string }>(
input: AdapterLoginStartSpineInput<TData>,
): Promise<AdapterLoginStartResolution<TData> | null> {
const ownerUserId = await input.deriveOwner();
if (input.guardBeforeValidate) {
await input.guardBeforeValidate();
}
const body =
input.req.body && typeof input.req.body === "object"
? (input.req.body as Record<string, unknown>)
: {};
const candidate = input.requestOverrides ? { ...body, ...input.requestOverrides } : body;
const parsed = input.requestSchema.safeParse(candidate);
if (!parsed.success) {
input.res.status(400).json({ error: input.invalidRequestError });
return null;
}
const data = parsed.data;
if (input.guardAfterValidate) {
const stop = await input.guardAfterValidate(data);
if (stop) return null;
}
await input.assertSandbox(data);
return { ownerUserId, data };
}
+70
View File
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import type { AdapterLoginCapability, ServerAdapterModule } from "@paperclipai/adapter-utils";
import { buildAdapterCapabilities } from "./adapters.js";
// The adapter listing projects the safe scalar login fields to the client. The
// projection carries the panel mode, the sandbox transport, and the timeout
// policy. It never carries a function member or a secret. An adapter with no
// login capability projects no `login` object.
function makeAdapter(overrides: Partial<ServerAdapterModule> = {}): ServerAdapterModule {
return {
type: "vendor_local",
execute: async () => {
throw new Error("not used");
},
testEnvironment: async () => {
throw new Error("not used");
},
...overrides,
} as ServerAdapterModule;
}
const displayedCodeLogin: AdapterLoginCapability = {
panelMode: "displayed_code",
sandboxTransport: "streamed_exec",
timeoutPolicy: "caller_bounded",
getCommand: () => "vendor login",
parsePrompt: () => null,
};
describe("buildAdapterCapabilities login projection", () => {
it("projects the safe scalar login fields", () => {
const caps = buildAdapterCapabilities(makeAdapter({ loginCapability: displayedCodeLogin }));
expect(caps.login).toEqual({
panelMode: "displayed_code",
sandboxTransport: "streamed_exec",
timeoutPolicy: "caller_bounded",
});
});
it("omits the login object when the adapter declares no capability", () => {
const caps = buildAdapterCapabilities(makeAdapter());
expect(caps.login).toBeUndefined();
});
it("never projects the function members or a completion claim", () => {
const caps = buildAdapterCapabilities(
makeAdapter({
loginCapability: {
panelMode: "submitted_browser_code",
sandboxTransport: "pseudo_terminal",
timeoutPolicy: "fixed",
getCommand: () => "vendor setup-token",
parsePrompt: () => null,
captureCredential: () => null,
completionClaim: "storedSessionId",
},
}),
);
expect(caps.login).toEqual({
panelMode: "submitted_browser_code",
sandboxTransport: "pseudo_terminal",
timeoutPolicy: "fixed",
});
expect(caps.login).not.toHaveProperty("getCommand");
expect(caps.login).not.toHaveProperty("parsePrompt");
expect(caps.login).not.toHaveProperty("captureCredential");
expect(caps.login).not.toHaveProperty("completionClaim");
});
});
+38 -1
View File
@@ -41,6 +41,11 @@ import {
} from "../services/adapter-plugin-store.js";
import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
import type { ServerAdapterModule, AdapterConfigSchema } from "../adapters/types.js";
import type {
AdapterLoginPanelMode,
AdapterLoginSandboxTransport,
AdapterLoginTimeoutPolicy,
} from "@paperclipai/adapter-utils";
import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js";
import { logger } from "../middleware/logger.js";
import { forbidden } from "../errors.js";
@@ -79,6 +84,18 @@ interface AdapterInstallRequest {
version?: string;
}
/**
* The safe scalar login fields the adapter listing projects to the client. It
* carries only the panel mode, the sandbox transport, and the timeout policy.
* It carries no function member and no secret. The user interface reads it to
* pick the login flow and the login panel.
*/
interface AdapterLoginProjection {
panelMode: AdapterLoginPanelMode;
sandboxTransport: AdapterLoginSandboxTransport;
timeoutPolicy: AdapterLoginTimeoutPolicy;
}
interface AdapterCapabilities {
supportsInstructionsBundle: boolean;
supportsSkills: boolean;
@@ -86,6 +103,11 @@ interface AdapterCapabilities {
requiresMaterializedRuntimeSkills: boolean;
supportsModelProfiles: boolean;
supportsAcp: boolean;
/**
* The projected login capability. It is present only when the adapter
* declares an interactive login capability. It is absent otherwise.
*/
login?: AdapterLoginProjection;
}
interface AdapterInfo {
@@ -134,7 +156,13 @@ function readAdapterPackageVersionFromDisk(record: AdapterPluginRecord): string
}
}
function buildAdapterCapabilities(adapter: ServerAdapterModule): AdapterCapabilities {
/**
* Build the client capability view for one adapter. The login projection carries
* only the safe scalar fields; it drops the function members and the completion
* claim, so the response holds no secret and no code.
*/
export function buildAdapterCapabilities(adapter: ServerAdapterModule): AdapterCapabilities {
const login = adapter.loginCapability;
return {
supportsInstructionsBundle: adapter.supportsInstructionsBundle ?? false,
supportsSkills: Boolean(adapter.listSkills || adapter.syncSkills),
@@ -142,6 +170,15 @@ function buildAdapterCapabilities(adapter: ServerAdapterModule): AdapterCapabili
requiresMaterializedRuntimeSkills: adapter.requiresMaterializedRuntimeSkills ?? false,
supportsModelProfiles: Boolean(adapter.modelProfiles?.length || adapter.listModelProfiles),
supportsAcp: Boolean(adapter.acp),
...(login
? {
login: {
panelMode: login.panelMode,
sandboxTransport: login.sandboxTransport,
timeoutPolicy: login.timeoutPolicy,
},
}
: {}),
};
}
+112 -82
View File
@@ -29,6 +29,7 @@ import {
updateAgentSchema,
supportedEnvironmentDriversForAdapter,
LOW_TRUST_REVIEW_PRESET,
startAdapterAuthSessionRequestSchema,
startClaudeSetupTokenSessionRequestSchema,
submitBrowserCodeRequestSchema,
} from "@paperclipai/shared";
@@ -61,6 +62,7 @@ import {
import { badRequest, conflict, forbidden, HttpError, notFound, unprocessable } from "../errors.js";
import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js";
import { assertAuthenticated, assertBoard, assertCompanyAccess, assertInstanceAdmin, buildActorSecretContext, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js";
import { runAdapterLoginStartSpine } from "./adapter-login-route-spine.js";
import {
assertNoAgentHostWorkspaceCommandMutation,
collectAgentAdapterWorkspaceCommandPaths,
@@ -140,7 +142,6 @@ import {
} from "@paperclipai/adapter-codex-local/server";
import {
AdapterAuthSessionConflictError,
CODEX_DEVICE_LOGIN_ADAPTER_TYPE,
createCodexDeviceLoginService,
createDbAdapterAuthSessionStore,
createProductionLoginSessionRuntime,
@@ -362,12 +363,10 @@ export function agentRoutes(
companyId: string;
ownerUserId: string;
adapterType: string;
environmentId: string;
}): boolean =>
row.companyId === identity.companyId &&
row.ownerUserId === identity.ownerUserId &&
row.adapterType === identity.adapterType &&
row.environmentId === identity.environmentId;
row.adapterType === identity.adapterType;
const inMemorySetupTokenCleanupStore: SetupTokenCleanupStore = {
async record(record): Promise<void> {
setupTokenCleanupRows.set(record.sessionId, { ...record });
@@ -507,6 +506,7 @@ export function agentRoutes(
// after the fence; the lock spans the whole section.
const outcome = await adapterLoginStore.withCompanyAdapterPromotionLock(
context.companyId,
context.startedByUserId,
context.adapterType,
() =>
promoteDeviceLoginCredential({
@@ -1167,9 +1167,20 @@ export function agentRoutes(
return userId;
}
// The device-login flow supports only the Codex adapter. Reject any other type.
function assertCodexLoginAdapter(type: string): void {
if (type !== CODEX_DEVICE_LOGIN_ADAPTER_TYPE) {
// Read the interactive login capability the registry declares for an adapter
// type. Return null when the adapter declares no capability, so a guard fails
// closed on the absent case.
function getRegistryLoginCapability(type: string) {
return findActiveServerAdapter(type)?.loginCapability ?? null;
}
// The device-login route drives a login over the streamed exec channel. It
// serves any adapter whose registry login capability declares that transport.
// The guard reads the capability, not the adapter name, so a new adapter with
// the same transport passes with no code change. It rejects an adapter with no
// matching capability with a fixed 400.
function assertStreamedExecLoginAdapter(type: string): void {
if (getRegistryLoginCapability(type)?.sandboxTransport !== "streamed_exec") {
throw badRequest(`Adapter "${type}" does not support a device login.`);
}
}
@@ -1236,7 +1247,7 @@ export function agentRoutes(
const resolved = provider
? await resolvePluginSandboxProviderDriverByKey({ db, driverKey: provider })
: null;
if (!resolved?.driver.supportsSetupTokenLogin) {
if (!resolved?.driver.supportsLoginPty) {
throw unprocessable(SETUP_TOKEN_PROVIDER_UNSUPPORTED, {
code: SETUP_TOKEN_PROVIDER_UNSUPPORTED_CODE,
});
@@ -1251,19 +1262,17 @@ export function agentRoutes(
async function readOwnerLoginSession(
companyId: string,
adapterType: string,
sessionId: string,
publicSessionId: string,
requestingUserId: string,
): Promise<AdapterAuthSessionOwnerResponse | null> {
const row = await adapterLoginStore.get(sessionId);
if (
!row ||
row.companyId !== companyId ||
row.adapterType !== adapterType ||
row.startedByUserId !== requestingUserId
) {
// Read by the public session id, scoped to the company. The store predicate
// already carries the company id, so a foreign-company caller reads nothing
// and the internal row id never matches. Keep the adapter and owner checks.
const row = await adapterLoginStore.getByPublicId(publicSessionId, companyId);
if (!row || row.adapterType !== adapterType || row.startedByUserId !== requestingUserId) {
return null;
}
return adapterLoginService.readOwnerSession(sessionId, requestingUserId);
return adapterLoginService.readOwnerSession(publicSessionId, companyId, requestingUserId);
}
async function assertCanReadConfigurations(req: Request, companyId: string) {
@@ -2494,33 +2503,35 @@ export function agentRoutes(
async (req, res) => {
const companyId = req.params.companyId as string;
const type = req.params.type as string;
const startedByUserId = await assertCanManageAdapterLogin(req, companyId);
assertCodexLoginAdapter(type);
const environmentId =
typeof req.body?.environmentId === "string" && req.body.environmentId.trim().length > 0
? (req.body.environmentId as string)
: null;
if (!environmentId) {
throw badRequest("A sandbox environment is required to start a device login.");
}
const ttlSeconds =
typeof req.body?.ttlSeconds === "number" && Number.isFinite(req.body.ttlSeconds)
? (req.body.ttlSeconds as number)
: undefined;
// Reject a non-sandbox or inactive environment before the service starts.
await assertSandboxLoginEnvironment(companyId, environmentId);
// The shared start-route spine derives the owner, checks the path adapter
// type, validates the strict request schema, and checks the sandbox
// environment before any session or lease side effect. The client body
// carries no adapter type, so the spine injects the path type into the
// parse. The strict schema rejects an unknown field, a non-uuid
// environment id, and an out-of-range time-to-live with a fixed 400.
const resolved = await runAdapterLoginStartSpine({
req,
res,
deriveOwner: () => assertCanManageAdapterLogin(req, companyId),
guardBeforeValidate: () => assertStreamedExecLoginAdapter(type),
requestSchema: startAdapterAuthSessionRequestSchema,
invalidRequestError: "The device login start request is invalid.",
requestOverrides: { adapterType: type },
assertSandbox: (data) => assertSandboxLoginEnvironment(companyId, data.environmentId),
});
if (!resolved) return;
const { ownerUserId: startedByUserId, data } = resolved;
const controller = new AbortController();
let result: Awaited<ReturnType<typeof adapterLoginService.start>>;
try {
result = await adapterLoginService.start({
companyId,
environmentId,
environmentId: data.environmentId,
adapterType: type,
startedByUserId,
ttlSeconds,
ttlSeconds: data.ttlSeconds,
signal: controller.signal,
});
} catch (error) {
@@ -2556,7 +2567,7 @@ export function agentRoutes(
const type = req.params.type as string;
const sessionId = req.params.sessionId as string;
const ownerUserId = await assertCanManageAdapterLogin(req, companyId);
assertCodexLoginAdapter(type);
assertStreamedExecLoginAdapter(type);
const owner = await readOwnerLoginSession(companyId, type, sessionId, ownerUserId);
if (!owner) {
@@ -2576,7 +2587,7 @@ export function agentRoutes(
const type = req.params.type as string;
const sessionId = req.params.sessionId as string;
const ownerUserId = await assertCanManageAdapterLogin(req, companyId);
assertCodexLoginAdapter(type);
assertStreamedExecLoginAdapter(type);
// Scope the cancel to this company, adapter, and owner. A non-owner and a
// cross-company caller both receive a 404 and cannot cancel a session.
@@ -2589,7 +2600,7 @@ export function agentRoutes(
// even when this process does not own the in-flight run, so a cross-process
// cancel or a cancel after a restart does not leave the slot held until the
// expiry. The reaper deletes the sandbox and finalizes the terminal.
const cancelled = await adapterLoginService.cancelOwnerSession(sessionId, ownerUserId);
const cancelled = await adapterLoginService.cancelOwnerSession(sessionId, companyId, ownerUserId);
// Abort the in-flight run this process owns, so the local login stops at
// once instead of waiting for the reaper. A run in another process, or an
// already-terminal run, has no controller here.
@@ -4631,66 +4642,85 @@ export function agentRoutes(
router.post("/companies/:companyId/setup-token-login-sessions", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
const ownerUserId = deriveSetupTokenOwnerUserId(req);
res.setHeader("Cache-Control", "no-store");
// Parse the request with the shared strict validator before any session,
// lease, or pseudo-terminal side effect. `.strict()` rejects an unknown field,
// including a legacy `ttlSeconds`, with a fixed 400. The route echoes no input.
const parsed = startClaudeSetupTokenSessionRequestSchema.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({ error: "The Claude login start request is invalid." });
return;
}
const { environmentId, adapterType } = parsed.data;
const confirmedOverwrite: ClaudeSetupTokenOverwrite | null = parsed.data.overwrite ?? null;
// The company-and-environment login serves only the Claude adapter. A
// different adapter type gets a fixed 400.
if (adapterType !== CLAUDE_SETUP_TOKEN_ADAPTER_TYPE) {
res.status(400).json({ error: "Only the claude_local adapter supports a setup-token login." });
return;
}
if (!SETUP_TOKEN_LOGIN_TRANSPORT_READY) {
// The live login transport binds at a later call site. Fail closed with the
// fixed no-secret error until then.
res.status(503).json({ error: SETUP_TOKEN_START_FAILED });
return;
}
// Resolve the environment server-side to a live instance sandbox. It fails
// closed on a missing, archived, non-sandbox, or fake-provider environment, so
// the route never persists an empty or invalid environment scope. It runs
// before the session starts, so no store holds a rejected environment. It
// also rejects an environment that another company owns (see
// `assertSandboxLoginEnvironment`), so a login never runs in a foreign
// company sandbox. It also fails closed when the provider does not advertise
// the setup-token login capability, so an unsupported provider never reaches
// a session row, a lease, or a pseudo-terminal.
await assertSandboxLoginEnvironment(companyId, environmentId, {
requireSetupTokenLoginProvider: true,
// The shared start-route spine derives the owner, validates the strict
// request schema, runs the Claude-only guards, and checks the sandbox
// environment before any session, lease, or pseudo-terminal side effect.
//
// The owner step runs the company access check, derives the owner, and then
// sets `Cache-Control: no-store`, so a rejected member sees no cache header
// and every other response carries it. The strict schema rejects an unknown
// field, including a legacy `ttlSeconds`, with a fixed 400. The post-validate
// guard rejects a non-Claude adapter with a fixed 400 and fails closed with
// the fixed no-secret 503 until the live login transport binds. The sandbox
// check fails closed on a missing, archived, non-sandbox, fake-provider, or
// foreign environment, and on a provider without the setup-token login
// capability, so no rejected environment reaches a session row, a lease, or a
// pseudo-terminal.
const resolved = await runAdapterLoginStartSpine({
req,
res,
deriveOwner: () => {
assertCompanyAccess(req, companyId);
const ownerUserId = deriveSetupTokenOwnerUserId(req);
res.setHeader("Cache-Control", "no-store");
return ownerUserId;
},
requestSchema: startClaudeSetupTokenSessionRequestSchema,
invalidRequestError: "The Claude login start request is invalid.",
guardAfterValidate: (data) => {
// The setup-token route drives a login on a pseudo-terminal and records a
// stored session identifier on success. It serves any adapter whose
// registry login capability declares that transport and that claim. The
// guard reads the capability, not the adapter name, so a new adapter with
// the same capability passes with no code change. It rejects an adapter
// with no matching capability with a fixed 400.
const capability = getRegistryLoginCapability(data.adapterType);
if (
capability?.sandboxTransport !== "pseudo_terminal" ||
capability.completionClaim !== "storedSessionId"
) {
res.status(400).json({ error: "This adapter does not support a setup-token login." });
return true;
}
if (!SETUP_TOKEN_LOGIN_TRANSPORT_READY) {
res.status(503).json({ error: SETUP_TOKEN_START_FAILED });
return true;
}
return false;
},
assertSandbox: (data) =>
assertSandboxLoginEnvironment(companyId, data.environmentId, {
requireSetupTokenLoginProvider: true,
}),
});
if (!resolved) return;
const { ownerUserId, data } = resolved;
const { environmentId, adapterType } = data;
const confirmedOverwrite: ClaudeSetupTokenOverwrite | null = data.overwrite ?? null;
const scope: SetupTokenSessionScope = {
companyId,
ownerUserId,
targetAgentId: null,
adapterType,
environmentId,
confirmedOverwrite,
};
// Read the panel mode from the adapter capability. The guard already checked
// the capability, so it is present here. The client renders the panel from
// this value instead of a hard-coded mode.
const panelMode =
getRegistryLoginCapability(adapterType)?.panelMode ?? "submitted_browser_code";
try {
const started = await setupTokenLoginService.start(scope);
const descriptor = setupTokenLoginService.describeOwned(started.sessionId, scope);
// The start response carries the panel mode, so the client renders the
// browser-code panel. The full login URL rides only through the
// guarded prompt read, not the start response, so the prompt is null here.
// The client reads the prompt route for the login URL.
// correct panel. The full login URL rides only through the guarded prompt
// read, not the start response, so the prompt is null here. The client
// reads the prompt route for the login URL.
const body: ClaudeSetupTokenSessionOwnerResponse = {
...toClaudePublicResponse(descriptor),
panelMode: "submitted_browser_code",
panelMode,
prompt: null,
};
res.status(201).json(body);
+27 -1
View File
@@ -11,6 +11,7 @@ import {
resolveDeclaredSandboxCapabilities,
redactEnvironmentCustomImageSetupSession,
redactEnvironmentCustomImageTemplate,
relinkEnvironmentCustomImageTemplateSchema,
startEnvironmentCustomImageSetupSessionSchema,
type EnvironmentDeleteBlastRadius,
updateEnvironmentSchema,
@@ -741,7 +742,7 @@ export function environmentRoutes(
templateRefKind: driver.templateRefKind,
templateConfigBinding: driver.templateConfigBinding,
supportsTemplateDelete: driver.supportsTemplateDelete,
supportsSetupTokenLogin: driver.supportsSetupTokenLogin ?? false,
supportsLoginPty: driver.supportsLoginPty ?? false,
displayName: driver.displayName,
description: driver.description,
source: "plugin" as const,
@@ -952,6 +953,31 @@ export function environmentRoutes(
res.json(result);
});
router.post(
"/environments/:environmentId/custom-image-template/relink",
validate(relinkEnvironmentCustomImageTemplateSchema),
async (req, res) => {
assertCanAccessInstanceEnvironments(req);
const companyId = await resolveCustomImageCompanyId(req);
const actor = getActorInfo(req);
// The service classifies drift, re-stamps the fingerprint, and writes the
// activity row in one transaction. The route never classifies.
const result = await customImages.relinkActiveTemplate({
environmentId: req.params.environmentId as string,
confirmBootSourceDrift: req.body.confirmBootSourceDrift === true,
actor: {
actorType: actor.actorType,
actorId: actor.actorId,
agentId: actor.agentId,
runId: actor.runId,
agentApiKeyId: actor.agentApiKeyId,
},
companyId,
});
res.json(result);
},
);
router.delete("/environments/:environmentId/custom-image-template", async (req, res) => {
assertCanAccessInstanceEnvironments(req);
const companyId = await resolveCustomImageCompanyId(req);
+434 -16
View File
@@ -1,3 +1,6 @@
import { spawn } from "node:child_process";
import { accessSync, constants as fsConstants, existsSync, readFileSync } from "node:fs";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { Router, type Request, type Response } from "express";
import type { Db } from "@paperclipai/db";
@@ -11,6 +14,10 @@ import {
workspaceRuntimeControlTargetSchema,
} from "@paperclipai/shared";
import type { WorkspaceRuntimeDesiredState, WorkspaceRuntimeServiceStateMap } from "@paperclipai/shared";
import {
resolveCanonicalWorktreeSeedSource,
type CanonicalWorktreeSeedSource,
} from "@paperclipai/shared/worktree-seed-source";
import { validate } from "../middleware/validate.js";
import {
accessService,
@@ -45,9 +52,25 @@ import { appendWithCap } from "../adapters/utils.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { runExclusiveWorkspaceRuntimeControl } from "../services/workspace-operations.js";
import { resolveManagedWorkspaceInstanceId } from "../services/managed-workspace-identity.js";
import { isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
import {
issueWorkspaceLoginHandoff,
workspaceLoginHandoffFailureStatus,
} from "../services/workspace-login-handoff-issuer.js";
import { conflict, unprocessable } from "../errors.js";
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
function isReadableFile(filePath: string) {
try {
accessSync(filePath, fsConstants.R_OK);
return true;
} catch {
return false;
}
}
export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {
const router = Router();
const svc = executionWorkspaceService(db);
@@ -139,6 +162,87 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
res.json(readiness);
});
/**
* Mint a single-use workspace login handoff for the calling board user.
*
* Board-only: the ticket carries a user identity, so an agent key — which has
* no user to sign in — must never be able to mint one. Nothing in the request
* body influences what the ticket is bound to; only the landing path is taken
* from the caller, and it is reduced to a same-origin path before signing.
*/
router.post("/execution-workspaces/:id/login-handoff", async (req, res) => {
const id = req.params.id as string;
assertBoard(req);
const workspace = await getAccessibleResource(req, res, svc.getById(id), "Execution workspace not found");
if (!workspace) return;
// Opening a workspace board is a runtime-control-grade action: it hands the
// caller an authenticated session inside the cloned instance.
if (!(await assertRuntimeManageAllowed(req, res, workspace.companyId))) return;
const requestedNext = typeof (req.body as { next?: unknown } | null)?.next === "string"
? (req.body as { next: string }).next
: null;
const result = await issueWorkspaceLoginHandoff({
db,
companyId: workspace.companyId,
executionWorkspace: { id: workspace.id, cwd: workspace.cwd },
actor: {
userId: req.actor.userId ?? null,
userEmail: req.actor.userEmail ?? null,
source: req.actor.source ?? null,
},
next: requestedNext,
});
if (!result.ok) {
logger.warn(
{
executionWorkspaceId: workspace.id,
reason: result.failure.reason,
detail: "detail" in result.failure ? result.failure.detail : null,
},
"workspace login handoff was not issued",
);
res.status(workspaceLoginHandoffFailureStatus(result.failure)).json({
error: "workspace_login_handoff_unavailable",
// The UI turns this into the labeled snapshot-local credential fallback
// or a repair prompt, so the machine reason has to survive the boundary.
reason: result.failure.reason,
mode: "credentials",
...("detail" in result.failure ? { detail: result.failure.detail } : {}),
...("readiness" in result.failure ? { readiness: result.failure.readiness } : {}),
});
return;
}
await logActivity(db, {
companyId: workspace.companyId,
// Auditing issuance is a requirement of the handoff design: the nonce ties
// this record to the guest's own accept/reject record. The ticket itself is
// never recorded.
action: "workspace_login_handoff_issued",
entityType: "execution_workspace",
entityId: workspace.id,
actorType: "user",
actorId: result.issuance.userId,
details: {
nonce: result.issuance.nonce,
instanceId: result.issuance.instanceId,
origin: result.issuance.origin,
expiresAt: result.issuance.expiresAt,
},
}).catch((error) => {
logger.warn({ err: error, executionWorkspaceId: workspace.id }, "failed to audit workspace login handoff");
});
res.setHeader("Cache-Control", "no-store");
res.status(201).json({
url: result.issuance.url,
expiresAt: result.issuance.expiresAt,
mode: "handoff",
});
});
router.get("/execution-workspaces/:id/workspace-operations", async (req, res) => {
const id = req.params.id as string;
const workspace = await getAccessibleResource(req, res, svc.getById(id), "Execution workspace not found");
@@ -151,7 +255,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
async function handleExecutionWorkspaceRuntimeCommand(req: Request, res: Response) {
const id = req.params.id as string;
const action = String(req.params.action ?? "").trim().toLowerCase();
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "run") {
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "repair" && action !== "run") {
res.status(404).json({ error: "Workspace command action not found" });
return;
}
@@ -166,14 +270,6 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
sourceIssueId: existing.sourceIssueId,
});
// Recover any managed runtime-control operation this workspace was stranded with, then
// refuse only if one is genuinely still live. Authorization above still gates the caller,
// so recovery never widens who may control the workspace.
await workspaceOperationsSvc.assertRuntimeControlAvailable({
executionWorkspaceId: existing.id,
action,
});
const workspaceCwd = existing.cwd;
if (!workspaceCwd) {
res.status(422).json({ error: "Execution workspace needs a local path before Paperclip can run workspace commands" });
@@ -195,6 +291,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
and(
eq(projectWorkspaces.id, existing.projectWorkspaceId),
eq(projectWorkspaces.companyId, existing.companyId),
eq(projectWorkspaces.projectId, existing.projectId),
),
)
.then((rows) => rows[0] ?? null)
@@ -221,6 +318,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
const configuredServices = effectiveRuntimeConfig
? listConfiguredRuntimeServiceEntries({ workspaceRuntime: effectiveRuntimeConfig })
: [];
const repairRestartsRuntimeServices = action === "repair" && configuredServices.length > 0;
const workspaceCommand = effectiveRuntimeConfig
? findWorkspaceCommandDefinition(effectiveRuntimeConfig, target.workspaceCommandId ?? null)
: null;
@@ -267,6 +365,78 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
return;
}
let repairSeedSource: CanonicalWorktreeSeedSource | null = null;
let repairPreviousAttemptId: string | null = null;
let repairCliArgs: string[] | null = null;
if (action === "repair") {
const manifestPath = path.join(workspaceCwd, ".paperclip", "seed-manifest.json");
let manifest: {
attemptId?: unknown;
source?: { configPath?: unknown; instanceId?: unknown };
targetInstanceId?: unknown;
};
try {
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
} catch {
throw unprocessable("Workspace seed manifest is malformed; repair source identity cannot be trusted.", {
code: "workspace_repair_precondition_failed",
reason: "seed_manifest_malformed",
repairPhase: "precondition_validation",
});
}
try {
if (!projectWorkspace?.cwd) {
throw new Error("Workspace repair requires a registered base project workspace.");
}
const expectedTargetInstanceId = resolveManagedWorkspaceInstanceId(workspaceCwd);
if (!expectedTargetInstanceId) {
throw new Error("Workspace repair cannot resolve the registered target instance.");
}
repairSeedSource = resolveCanonicalWorktreeSeedSource({
registeredBaseWorkspaceCwd: projectWorkspace.cwd,
targetConfigPath: path.join(workspaceCwd, ".paperclip", "config.json"),
expectedTargetInstanceId,
manifestSource: manifest.source,
manifestTargetInstanceId: manifest.targetInstanceId,
});
repairPreviousAttemptId = typeof manifest.attemptId === "string" ? manifest.attemptId : null;
const baseWorkspaceCwd = repairSeedSource.baseWorkspaceCwd;
if (!baseWorkspaceCwd) {
throw new Error("Workspace repair source is not bound to a registered base project workspace.");
}
const cliRunner = path.join(baseWorkspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(baseWorkspaceCwd, "cli", "src", "index.ts");
const cliDist = path.join(baseWorkspaceCwd, "cli", "dist", "index.js");
repairCliArgs = isReadableFile(cliRunner) && isReadableFile(cliEntry)
? [cliRunner, cliEntry]
: isReadableFile(cliDist)
? [cliDist]
: null;
if (!repairCliArgs) {
throw new Error("Workspace repair cannot find a runnable Paperclip CLI in the base workspace.");
}
} catch (error) {
throw unprocessable(
error instanceof Error ? error.message : "Workspace repair source validation failed.",
{
code: "workspace_repair_precondition_failed",
reason: "source_registration_invalid",
repairPhase: "precondition_validation",
},
);
}
}
// This check can reconcile a stale operation row, so repair source validation must
// precede it. Invalid manifest diagnostics must fail before any operation or service
// mutation, not merely before the reseed child process is spawned.
await workspaceOperationsSvc.assertRuntimeControlAvailable({
executionWorkspaceId: existing.id,
action,
});
const actor = getActorInfo(req);
const recorder = workspaceOperationsSvc.createRecorder({
companyId: existing.companyId,
@@ -327,7 +497,11 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
}
const recordRuntimeControlOperation = () => recorder.recordOperation({
phase: action === "stop" ? "workspace_teardown" : "workspace_provision",
phase: action === "stop"
? "workspace_teardown"
: action === "repair"
? "workspace_repair"
: "workspace_provision",
command: workspaceCommand?.command ?? `workspace command ${action}`,
cwd: existing.cwd,
metadata: {
@@ -339,7 +513,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
runtimeServiceId: selectedRuntimeServiceId,
serviceIndex: selectedServiceIndex,
},
run: async () => {
run: async (reportProgress) => {
const ensureWorkspaceAvailable = async () =>
await ensurePersistedExecutionWorkspaceAvailable({
base: {
@@ -429,7 +603,242 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
else stderr = appendWithCap(stderr, chunk, WORKSPACE_CONTROL_OUTPUT_MAX_CHARS);
};
if (action === "stop" || action === "restart") {
if (action === "repair") {
type RepairPhase =
| "managed_stop"
| "precondition_validation"
| "target_backup"
| "full_reseed"
| "managed_restart"
| "readiness_validation";
const repairDiagnostics: Array<{
phase: RepairPhase;
status: "started" | "succeeded" | "failed";
at: string;
}> = [];
let repairPhase: RepairPhase = "managed_stop";
const repairPreconditionError = (
status: 409 | 422,
reason:
| "seed_manifest_malformed"
| "seed_manifest_instance_mismatch"
| "source_instance_unavailable"
| "paperclip_cli_unavailable",
message: string,
) => {
const details = {
code: "workspace_repair_precondition_failed",
reason,
repairPhase,
};
return status === 409
? conflict(message, details)
: unprocessable(message, details);
};
const reportRepairPhase = async (
phase: RepairPhase,
status: "started" | "succeeded" | "failed",
) => {
repairPhase = phase;
repairDiagnostics.push({ phase, status, at: new Date().toISOString() });
if (repairDiagnostics.length > 32) repairDiagnostics.splice(0, repairDiagnostics.length - 32);
await reportProgress({
metadata: {
repairPhase,
repairDiagnostics: [...repairDiagnostics],
databaseOnly: true,
worktreePreserved: true,
},
system: `Workspace repair ${phase}: ${status}.\n`,
});
};
try {
await reportRepairPhase("managed_stop", "started");
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId: existing.id,
workspaceCwd,
});
await reportRepairPhase("managed_stop", "succeeded");
if (!repairSeedSource?.baseWorkspaceCwd || !repairCliArgs) {
throw new Error("Workspace repair source preflight did not complete.");
}
const manifestPath = path.join(workspaceCwd, ".paperclip", "seed-manifest.json");
const sourceConfigPath = repairSeedSource.configPath;
const baseWorkspaceCwd = repairSeedSource.baseWorkspaceCwd;
await reportRepairPhase("target_backup", "started");
const child = spawn(process.execPath, [
...repairCliArgs,
"worktree",
"reseed",
"--from-config",
sourceConfigPath,
"--to",
workspaceCwd,
"--seed-mode",
"full",
"--yes",
"--backup-target",
], {
cwd: baseWorkspaceCwd,
env: {
...process.env,
PAPERCLIP_SEED_EXPECTED_COMPANY_ID: existing.companyId,
PAPERCLIP_WORKSPACE_BASE_CWD: baseWorkspaceCwd,
PAPERCLIP_PROJECT_WORKSPACE_ID: existing.projectWorkspaceId ?? "",
},
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout?.on("data", (chunk: Buffer) => {
void onLog("stdout", chunk.toString("utf8"));
});
child.stderr?.on("data", (chunk: Buffer) => {
void onLog("stderr", chunk.toString("utf8"));
});
let repairCommandTimedOut = false;
let repairCommandForceTimeout: NodeJS.Timeout | null = null;
const repairCommandTimeout = setTimeout(() => {
repairCommandTimedOut = true;
child.kill("SIGTERM");
repairCommandForceTimeout = setTimeout(() => child.kill("SIGKILL"), 5_000);
repairCommandForceTimeout.unref?.();
}, 20 * 60_000);
repairCommandTimeout.unref?.();
let reseedObserved = false;
const manifestPoll = setInterval(() => {
if (reseedObserved || !existsSync(manifestPath)) return;
try {
const current = JSON.parse(readFileSync(manifestPath, "utf8")) as {
attemptId?: unknown;
state?: unknown;
};
if (
typeof current.attemptId === "string"
&& current.attemptId !== repairPreviousAttemptId
&& (current.state === "pending" || current.state === "running" || current.state === "failed")
) {
reseedObserved = true;
}
} catch {
// The atomic manifest writer makes this unlikely; the terminal
// read below remains authoritative.
}
}, 100);
manifestPoll.unref?.();
const exitCode = await new Promise<number>((resolve, reject) => {
child.once("error", reject);
child.once("exit", (code) => resolve(code ?? 1));
}).finally(() => {
clearInterval(manifestPoll);
clearTimeout(repairCommandTimeout);
if (repairCommandForceTimeout) clearTimeout(repairCommandForceTimeout);
});
if (reseedObserved) {
await reportRepairPhase("target_backup", "succeeded");
await reportRepairPhase("full_reseed", "started");
}
if (exitCode !== 0) {
let seedFailurePhase: string | null = null;
try {
const failedManifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
phase?: unknown;
state?: unknown;
};
seedFailurePhase = failedManifest.state === "failed" && typeof failedManifest.phase === "string"
? failedManifest.phase
: null;
} catch {
// The outer repair phase remains exact when no seed phase was persisted.
}
await reportProgress({
metadata: { seedFailurePhase },
system: seedFailurePhase ? `Workspace seed failed during ${seedFailurePhase}.\n` : null,
});
throw new Error(
repairCommandTimedOut
? `Workspace database repair command timed out during ${repairPhase}.`
: `Workspace database repair command failed during ${repairPhase}.`,
);
}
if (!reseedObserved) {
await reportRepairPhase("target_backup", "succeeded");
await reportRepairPhase("full_reseed", "started");
}
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
if (!isVerifiedWorktreeSeedManifest(manifest)) {
throw new Error("Workspace reseed returned without a verified terminal manifest.");
}
const expectedInstanceId = resolveManagedWorkspaceInstanceId(workspaceCwd);
if (manifest.targetInstanceId !== expectedInstanceId) {
throw repairPreconditionError(
422,
"seed_manifest_instance_mismatch",
"Verified seed manifest belongs to a different workspace instance.",
);
}
resolveCanonicalWorktreeSeedSource({
registeredBaseWorkspaceCwd: baseWorkspaceCwd,
targetConfigPath: path.join(workspaceCwd, ".paperclip", "config.json"),
expectedTargetInstanceId: repairSeedSource.targetInstanceId,
manifestSource: manifest.source as { configPath?: unknown; instanceId?: unknown } | undefined,
manifestTargetInstanceId: manifest.targetInstanceId,
});
await reportRepairPhase("full_reseed", "succeeded");
await reportRepairPhase("managed_restart", "started");
let startedServices: Awaited<ReturnType<typeof startRuntimeServicesForWorkspaceControl>> = [];
if (repairRestartsRuntimeServices) {
const availableWorkspace = await ensureWorkspaceAvailable();
if (!availableWorkspace) {
throw new Error("Execution workspace needs a local path before Paperclip can restart it.");
}
startedServices = await startRuntimeServicesForWorkspaceControl({
db,
actor: {
id: actor.agentId ?? null,
name: actor.actorType === "user" ? "Board" : "Agent",
companyId: existing.companyId,
},
issue: existing.sourceIssueId
? { id: existing.sourceIssueId, identifier: null, title: existing.name }
: null,
workspace: availableWorkspace,
executionWorkspaceId: existing.id,
config: {
workspaceRuntime: effectiveRuntimeConfig,
runtimeProvisionCommand:
existing.config?.runtimeProvisionCommand
?? projectPolicy?.workspaceStrategy?.runtimeProvisionCommand
?? null,
},
adapterEnv: {},
onLog,
recorder,
});
}
runtimeServiceCount = startedServices.length;
await reportRepairPhase("managed_restart", "succeeded");
await reportRepairPhase("readiness_validation", "started");
if (
repairRestartsRuntimeServices
&& (
startedServices.length === 0
|| startedServices.some((service) => service.status !== "running" || service.healthStatus !== "healthy")
)
) {
throw new Error("Managed restart did not produce healthy runtime services.");
}
await reportRepairPhase("readiness_validation", "succeeded");
} catch (error) {
await reportRepairPhase(repairPhase, "failed");
throw error;
}
} else if (action === "stop" || action === "restart") {
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId: existing.id,
@@ -481,7 +890,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
throw error;
}
runtimeServiceCount = startedServices.length;
} else {
} else if (action !== "repair") {
runtimeServiceCount = selectedRuntimeServiceId ? Math.max(0, (existing.runtimeServices?.length ?? 1) - 1) : 0;
}
@@ -504,7 +913,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
config: { workspaceRuntime: effectiveRuntimeConfig },
currentDesiredState,
currentServiceStates: existing.config?.serviceStates ?? null,
action,
action: action === "repair"
? repairRestartsRuntimeServices ? "start" : "stop"
: action,
serviceIndex: selectedServiceIndex,
});
const metadata = mergeExecutionWorkspaceConfig(existing.metadata as Record<string, unknown> | null, {
@@ -522,12 +933,17 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
? "Stopped execution workspace runtime services.\n"
: action === "restart"
? "Restarted execution workspace runtime services.\n"
: "Started execution workspace runtime services.\n",
: action === "repair"
? repairRestartsRuntimeServices
? "Repaired the isolated workspace database and restarted healthy runtime services.\n"
: "Repaired the isolated workspace database; no managed runtime services were configured to restart.\n"
: "Started execution workspace runtime services.\n",
metadata: {
runtimeServiceCount,
workspaceCommandId: workspaceCommand?.id ?? target.workspaceCommandId ?? null,
runtimeServiceId: selectedRuntimeServiceId,
serviceIndex: selectedServiceIndex,
runtimeRestarted: action === "repair" ? repairRestartsRuntimeServices : undefined,
},
};
},
@@ -565,7 +981,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
// The operation is already terminal here. This also catches the recorder's own time
// budget expiring on a start that never settles, which is the one failure the inner
// handler cannot see — reconcile there too so no listener or desired state is left over.
if (action === "start" || action === "restart") await reconcileFailedRuntimeStart(error);
if (action === "start" || action === "restart" || action === "repair") {
await reconcileFailedRuntimeStart(error);
}
throw error;
}
},
+57 -4
View File
@@ -19,6 +19,13 @@ import {
type InspectDatabaseBackupHealthOptions,
} from "../services/database-backup-health.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { isManagedWorkspaceInstance, resolveWorkspaceReadiness } from "../services/workspace-readiness.js";
import {
resolveWorkspaceReadinessLocalToken,
WORKSPACE_READINESS_TOKEN_HEADER,
WORKSPACE_READINESS_USER_EMAIL_HEADER,
WORKSPACE_READINESS_USER_ID_HEADER,
} from "../auth/workspace-login-handoff.js";
import { serverVersion } from "../version.js";
function shouldExposeFullHealthDetails(
@@ -29,17 +36,33 @@ function shouldExposeFullHealthDetails(
return actorType === "board" || actorType === "agent";
}
function hasDevServerStatusToken(providedToken: string | undefined) {
const expectedToken = process.env.PAPERCLIP_DEV_SERVER_STATUS_TOKEN?.trim();
function matchesSharedToken(expectedToken: string | undefined | null, providedToken: string | undefined) {
const expectedValue = expectedToken?.trim();
const token = providedToken?.trim();
if (!expectedToken || !token) return false;
if (!expectedValue || !token) return false;
const expected = Buffer.from(expectedToken);
const expected = Buffer.from(expectedValue);
const provided = Buffer.from(token);
if (expected.length !== provided.length) return false;
return timingSafeEqual(expected, provided);
}
function hasDevServerStatusToken(providedToken: string | undefined) {
return matchesSharedToken(process.env.PAPERCLIP_DEV_SERVER_STATUS_TOKEN, providedToken);
}
/**
* Whether the caller may read this instance's workspace readiness.
*
* A managed workspace runs in `authenticated` mode, so its own control plane has
* no board session against it. The runtime injects a derived probe token into the
* guest and presents it here — the same shared-secret shape the dev-server
* supervisor already uses, and never a browser-supplied identity header.
*/
function hasWorkspaceReadinessToken(providedToken: string | undefined) {
return matchesSharedToken(resolveWorkspaceReadinessLocalToken(), providedToken);
}
function redactedDatabaseBackupWarning(warning: DatabaseBackupHealthWarning): DatabaseBackupHealthWarning {
const messages: Record<DatabaseBackupHealthWarning["code"], string> = {
database_backup_check_failed: "Database backup health check failed.",
@@ -147,6 +170,17 @@ export function healthRoutes(
const commit = serverInfo.git.available ? serverInfo.git.fullSha : null;
const exposeDevServerDetails =
exposeFullDetails || hasDevServerStatusToken(req.get("x-paperclip-dev-server-status-token"));
// Workspace readiness names the instance and execution workspace that
// answered, so it rides the protected responses only. Public health stays
// redacted: an anonymous caller still learns liveness and nothing else.
const exposeWorkspaceReadiness =
isManagedWorkspaceInstance()
&& (exposeFullDetails || hasWorkspaceReadinessToken(req.get(WORKSPACE_READINESS_TOKEN_HEADER)));
const requestedHandoffUserId = req.get(WORKSPACE_READINESS_USER_ID_HEADER)?.trim();
const requestedHandoffUserEmail = req.get(WORKSPACE_READINESS_USER_EMAIL_HEADER)?.trim();
const handoffSubject = requestedHandoffUserId && requestedHandoffUserEmail
? { userId: requestedHandoffUserId, email: requestedHandoffUserEmail }
: null;
if (!db) {
res.json(
@@ -173,6 +207,12 @@ export function healthRoutes(
await db.execute(sql`SELECT 1`);
} catch (error) {
logger.warn({ err: error }, "Health check database probe failed");
// Carry readiness on the unhealthy response too: the seed phase recorded on
// disk is exactly what tells an operator whether this is a half-finished
// restore or a database that died after being verified.
const workspace = exposeWorkspaceReadiness
? await resolveWorkspaceReadiness({ db, handoffSubject }).catch(() => null)
: null;
res.status(503).json({
status: "unhealthy",
version: serverVersion,
@@ -180,6 +220,7 @@ export function healthRoutes(
commit,
error: "database_unreachable",
...(exposeFullDetails ? { serverInfo } : {}),
...(workspace ? { workspace } : {}),
...(cloud ? { cloud } : {}),
});
return;
@@ -236,6 +277,13 @@ export function healthRoutes(
});
}
const workspaceReadiness = exposeWorkspaceReadiness
? await resolveWorkspaceReadiness({ db, handoffSubject }).catch((error) => {
logger.warn({ err: error }, "workspace readiness probe failed");
return null;
})
: null;
const databaseBackup = opts.databaseBackupHealth
? inspectDatabaseBackupHealth(opts.databaseBackupHealth)
: undefined;
@@ -254,6 +302,10 @@ export function healthRoutes(
...(redactedDatabaseBackup ? { databaseBackup: redactedDatabaseBackup } : {}),
...(redactedWarnings ? { warnings: redactedWarnings } : {}),
...(devServer ? { devServer } : {}),
// Token-authorized probe on an otherwise redacted response: the control
// plane needs readiness without a board session, and nothing else about
// this instance becomes visible.
...(workspaceReadiness ? { workspace: workspaceReadiness } : {}),
...(cloud ? { cloud } : {}),
});
return;
@@ -276,6 +328,7 @@ export function healthRoutes(
...(databaseBackup ? { databaseBackup } : {}),
...(warnings ? { warnings } : {}),
...(devServer ? { devServer } : {}),
...(workspaceReadiness ? { workspace: workspaceReadiness } : {}),
...(cloud ? { cloud } : {}),
});
});
+12 -10
View File
@@ -179,8 +179,8 @@ import {
} from "../services/onboarding-greeting.js";
import {
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
buildIssueBlockersResolvedWakeIdempotencyKey,
findExistingIssueBlockersResolvedWake,
buildIssueBlockersResolvedWakeStateKey,
findExistingIssueBlockersResolvedWakeForReadyState,
} from "../services/issue-dependency-wakeups.js";
import { assertEnvironmentSelectionForCompany } from "./environment-selection.js";
import {
@@ -10204,14 +10204,15 @@ export function issueRoutes(
source: string;
mutation: string;
}) => {
const idempotencyKey = buildIssueBlockersResolvedWakeIdempotencyKey({
const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: input.dependentIssueId,
resolvedBlockerIssueId: input.resolvedBlockerIssueId,
blockerIssueIds: input.blockerIssueIds,
});
try {
const existingWake = await findExistingIssueBlockersResolvedWake(db, {
const existingWake = await findExistingIssueBlockersResolvedWakeForReadyState(db, {
companyId: issue.companyId,
idempotencyKey,
dependentIssueId: input.dependentIssueId,
blockerIssueIds: input.blockerIssueIds,
});
if (existingWake) return;
} catch (err) {
@@ -12256,14 +12257,15 @@ export function issueRoutes(
resolvedBlockerIssueId: string;
blockerIssueIds: string[];
}) => {
const idempotencyKey = buildIssueBlockersResolvedWakeIdempotencyKey({
const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: input.dependentIssueId,
resolvedBlockerIssueId: input.resolvedBlockerIssueId,
blockerIssueIds: input.blockerIssueIds,
});
try {
const existingWake = await findExistingIssueBlockersResolvedWake(db, {
const existingWake = await findExistingIssueBlockersResolvedWakeForReadyState(db, {
companyId: currentIssue.companyId,
idempotencyKey,
dependentIssueId: input.dependentIssueId,
blockerIssueIds: input.blockerIssueIds,
});
if (existingWake) return;
} catch (err) {
+57
View File
@@ -112,6 +112,7 @@ import {
environmentCustomImageTerminalSessionTokenSchema,
environmentCustomImageTemplateSchema,
finishEnvironmentCustomImageSetupSessionSchema,
relinkEnvironmentCustomImageTemplateSchema,
updateEnvironmentSchema,
probeEnvironmentConfigSchema,
startEnvironmentCustomImageSetupSessionSchema,
@@ -635,6 +636,11 @@ const environmentCustomImageTemplateRollbackResultSchema = z.object({
supersededTemplate: environmentCustomImageTemplateSchema,
}).strict();
const environmentCustomImageTemplateRelinkResultSchema = z.object({
template: environmentCustomImageTemplateSchema,
classification: z.enum(["knob_only", "boot_source_drift", "unclassified"]),
}).strict();
const workTimelineQuerySchema = z.object({
from: z.string().optional(),
to: z.string().optional(),
@@ -804,6 +810,7 @@ const BOARD_ONLY_OPERATIONS = new Set([
"PATCH /api/companies/{companyId}/members/{memberId}/permissions",
"GET /api/companies/{companyId}/user-directory",
"POST /api/execution-workspaces/{id}/reconcile-branch",
"POST /api/execution-workspaces/{id}/login-handoff",
"GET /api/board-api-keys",
"POST /api/board-api-keys",
"DELETE /api/board-api-keys/{keyId}",
@@ -5323,6 +5330,36 @@ registry.registerPath({
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 422: r.unprocessable },
});
registry.registerPath({
method: "post",
path: "/api/execution-workspaces/{id}/login-handoff",
tags: ["execution-workspaces"],
summary: "Issue a single-use workspace login handoff",
description:
"Mints a short-lived, single-use ticket the isolated workspace exchanges for its own "
+ "instance-scoped session, so opening a managed workspace does not depend on a cloned "
+ "password. Board actors only. The response `url` must be navigated to, not stored: the "
+ "workspace answers it with a redirect so the ticket never enters browser history. A refusal "
+ "carries a machine `reason` and, where the control plane probed it, the workspace's own "
+ "readiness.",
request: {
params: z.object({ id: z.string() }),
body: jsonBody(
z.object({
next: z.string().optional().describe("Same-origin path to land on; anything else collapses to `/`."),
}),
),
},
responses: {
201: r.ok(),
401: r.unauthorized,
403: r.forbidden,
404: r.notFound,
409: r.conflict,
501: r.serverError,
},
});
registry.registerPath({
method: "post",
path: "/api/execution-workspaces/{id}/runtime-services/{action}",
@@ -5585,6 +5622,26 @@ registry.registerPath({
},
});
registry.registerPath({
method: "post",
path: "/api/environments/{environmentId}/custom-image-template/relink",
tags: ["environments"],
summary: "Relink a detached environment customImage template to the current config",
request: {
params: z.object({ environmentId: z.string() }),
query: environmentCustomImageCompanyQuerySchema,
body: jsonBody(relinkEnvironmentCustomImageTemplateSchema),
},
responses: {
200: r.ok(environmentCustomImageTemplateRelinkResultSchema),
400: r.badRequest,
401: r.unauthorized,
403: r.forbidden,
404: r.notFound,
409: r.conflict,
},
});
registry.registerPath({
method: "delete",
path: "/api/environments/{environmentId}/custom-image-template",
+59 -3
View File
@@ -105,6 +105,22 @@ const mockRunSecretRedactionRegistry = vi.hoisted(() => ({
const mockSecretService = vi.hoisted(() => ({
readClaudeOAuthUserSecretStatus: vi.fn(),
}));
// The registry lookup the start-route guard reads. A `beforeEach` sets the
// default so `claude_local` declares the setup-token capability. A test overrides
// it to prove the guard reads the capability, not the adapter name.
const mockFindActiveServerAdapter = vi.hoisted(() => vi.fn());
// The Claude setup-token login capability the registry declares for the built-in
// adapter. The guard requires the pseudo-terminal transport and the stored-session
// completion claim.
const CLAUDE_LOGIN_CAPABILITY = {
panelMode: "submitted_browser_code",
sandboxTransport: "pseudo_terminal",
timeoutPolicy: "fixed",
completionClaim: "storedSessionId",
getCommand: () => "",
parsePrompt: () => null,
} as const;
function registerModuleMocks(): void {
vi.doMock("../routes/authz.js", async () => vi.importActual("../routes/authz.js"));
@@ -160,11 +176,16 @@ function registerModuleMocks(): void {
syncInstructionsBundleConfigFromFilePath: vi.fn((_agent: unknown, config: unknown) => config),
workspaceOperationService: () => ({}),
}));
// The start-route guard reads the login capability from the registry. The
// `beforeEach` default declares the Claude setup-token capability for
// `claude_local` and no capability for any other type, so the guard passes for
// `claude_local` and fails closed for a different adapter through capability
// data alone.
vi.doMock("../adapters/index.js", () => ({
findServerAdapter: vi.fn(),
listAdapterModels: vi.fn(),
detectAdapterModel: vi.fn(),
findActiveServerAdapter: vi.fn(),
findActiveServerAdapter: mockFindActiveServerAdapter,
requireServerAdapter: vi.fn(),
}));
}
@@ -430,6 +451,12 @@ beforeEach(() => {
registerModuleMocks();
vi.clearAllMocks();
useOwner();
// The default registry declares the Claude setup-token capability for
// `claude_local` and no capability for any other type. A test overrides this to
// prove the guard reads the capability, not the adapter name.
mockFindActiveServerAdapter.mockImplementation((type: string) =>
type === "claude_local" ? { type, loginCapability: CLAUDE_LOGIN_CAPABILITY } : undefined,
);
// The default owner has no stored Claude value. A test overrides this to prove
// the status route returns the metadata for a present owner value.
mockSecretService.readClaudeOAuthUserSecretStatus.mockResolvedValue(null);
@@ -459,7 +486,7 @@ beforeEach(() => {
mockResolvePluginSandboxProviderDriverByKey.mockImplementation(
async ({ driverKey }: { driverKey: string }) =>
driverKey === "daytona"
? { plugin: { id: "plugin-daytona" }, driver: { supportsSetupTokenLogin: true } }
? { plugin: { id: "plugin-daytona" }, driver: { supportsLoginPty: true } }
: null,
);
});
@@ -651,7 +678,10 @@ describe("company-and-environment setup-token route — object-level authorizati
expect(transport.submittedCodes).toEqual([]);
});
it("rejects an adapter other than claude_local at start", async () => {
it("rejects an adapter whose login capability does not match the setup-token guard", async () => {
// The Codex adapter declares a streamed-exec device login, not a
// pseudo-terminal setup-token login. The guard reads the capability, so it
// rejects the adapter with a fixed 400 before any session.
const transport = buildTransport({ onSubmit: "pending" });
const { app } = await createApp({ transport });
@@ -664,6 +694,32 @@ describe("company-and-environment setup-token route — object-level authorizati
// holds no record.
expect(transport.records).toEqual([]);
});
it("starts a setup-token login for a third adapter that declares the capability", async () => {
// A third adapter, not the Claude adapter, declares the pseudo-terminal
// setup-token capability with the stored-session claim. The guard reads the
// capability, not the adapter name, so the adapter passes the guard and
// starts a session. This proves no adapter-name branch remains in the guard
// path. The start response reads the panel mode from the capability.
mockFindActiveServerAdapter.mockImplementation((type: string) =>
type === "gemini_local"
? {
type,
loginCapability: { ...CLAUDE_LOGIN_CAPABILITY, panelMode: "displayed_code" },
}
: undefined,
);
const transport = buildTransport({ onSubmit: "pending" });
const { app } = await createApp({ transport });
const res = await startCompanySession(app, {
environmentId: ENVIRONMENT_ID,
adapterType: "gemini_local",
});
expect(res.status, JSON.stringify(res.body)).toBe(201);
expect(res.body.panelMode).toBe("displayed_code");
expect(res.body.status).toBe("waiting_for_user");
});
});
describe("company-and-environment setup-token route — idempotent cancel", () => {
@@ -0,0 +1,81 @@
import { describe, expect, it } from "vitest";
import type { Environment } from "@paperclipai/shared";
import { buildLoginLeaseAcquireArgs } from "./adapter-login-lease.js";
// A minimal environment stand-in. The helper copies the reference without a
// read, so the test does not need a full environment row.
const ENVIRONMENT = { id: "env-1", name: "Sandbox", driver: "sandbox" } as unknown as Environment;
describe("buildLoginLeaseAcquireArgs", () => {
it("sets the fixed lease arguments that both login services share", () => {
const args = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT, adapterType: "codex" },
});
// A null issue, a null heartbeat run, and a null execution workspace disable
// lease reuse, so the login session always runs in a fresh sandbox.
expect(args.issueId).toBeNull();
expect(args.heartbeatRunId).toBeNull();
expect(args.persistedExecutionWorkspace).toBeNull();
// The helper applies the active custom-image template for both services.
expect(args.applyCustomImageTemplate).toBe(true);
});
it("passes the lease metadata through to the acquire arguments", () => {
const args = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT, adapterType: "claude" },
});
expect(args.companyId).toBe("co-1");
expect(args.environment).toBe(ENVIRONMENT);
expect(args.adapterType).toBe("claude");
});
it("defaults the adapter type to null when the caller omits it", () => {
const args = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT },
});
expect(args.adapterType).toBeNull();
});
it("passes the target agent through to the lease agent and defaults to null", () => {
const withAgent = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT },
targetAgentId: "agent-7",
});
expect(withAgent.agentId).toBe("agent-7");
const withoutAgent = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT },
});
expect(withoutAgent.agentId).toBeNull();
});
it("passes the company-binding assertion through and leaves it unset by default", () => {
const asserted = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT },
assertCompanyBinding: true,
});
expect(asserted.assertCompanyBinding).toBe(true);
const unset = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT },
});
expect(unset.assertCompanyBinding).toBeUndefined();
});
it("passes the requested expiry through and defaults to null", () => {
const deadline = new Date("2026-01-01T00:00:00.000Z");
const bounded = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT },
requestedExpiresAt: deadline,
});
expect(bounded.requestedExpiresAt).toBe(deadline);
const unbounded = buildLoginLeaseAcquireArgs({
metadata: { companyId: "co-1", environment: ENVIRONMENT },
});
expect(unbounded.requestedExpiresAt).toBeNull();
});
});
@@ -0,0 +1,72 @@
import type { Environment } from "@paperclipai/shared";
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
/**
* The argument object that the runtime `acquireRunLease` seam accepts. The type
* derives from the service method, so a change to the acquire input stays in
* sync with this helper.
*/
export type AcquireLoginLeaseArgs = Parameters<EnvironmentRuntimeService["acquireRunLease"]>[0];
/**
* The lease metadata that identifies the login sandbox. The helper copies these
* fields to the acquire arguments without a change.
*/
export interface LoginLeaseMetadata {
companyId: string;
environment: Environment;
/** The agent adapter type for this login (mixed-harness environments). */
adapterType?: string | null;
}
/**
* The options for one login lease acquire. The helper sets the fixed arguments
* and passes each option through to the acquire arguments.
*/
export interface BuildLoginLeaseAcquireArgsOptions {
/** The lease metadata that identifies the login sandbox. */
metadata: LoginLeaseMetadata;
/**
* The agent that owns the login. The codex flow passes null. The setup-token
* flow passes the target agent.
*/
targetAgentId?: string | null;
/**
* Re-check the environment company binding inside the lease insert
* transaction. The setup-token flow sets this to true.
*/
assertCompanyBinding?: boolean;
/**
* The latest time the acquired lease may stay active. The setup-token flow
* passes the session deadline. The codex flow passes nothing.
*/
requestedExpiresAt?: Date | null;
}
/**
* Build the fixed sandbox lease arguments for a login service. Both login
* services acquire a lease with a null issue, a null heartbeat run, and a null
* execution workspace, and both apply the active custom-image template. This
* helper sets those fixed arguments in one place and passes the caller options
* through without a change to the lease behavior.
*/
export function buildLoginLeaseAcquireArgs(
options: BuildLoginLeaseAcquireArgsOptions,
): AcquireLoginLeaseArgs {
return {
companyId: options.metadata.companyId,
environment: options.metadata.environment,
adapterType: options.metadata.adapterType ?? null,
agentId: options.targetAgentId ?? null,
// A null issue, a null heartbeat run, and a null execution workspace disable
// lease reuse, so the login session always runs in a fresh sandbox.
issueId: null,
heartbeatRunId: null,
persistedExecutionWorkspace: null,
// Apply the active custom-image template, so the sandbox binds to the
// trusted image and runtime identity.
applyCustomImageTemplate: true,
assertCompanyBinding: options.assertCompanyBinding,
requestedExpiresAt: options.requestedExpiresAt ?? null,
};
}
-1
View File
@@ -586,7 +586,6 @@ export function agentService(db: Db) {
companyId: input.companyId,
ownerUserId: ownerUserId ?? "",
adapterType: CLAUDE_LOCAL_ADAPTER_TYPE,
environmentId: input.environmentId ?? "",
});
if (!consumed) {
throw claudeOAuthClaimRejectedError();
@@ -1,8 +1,9 @@
import { inArray } from "drizzle-orm";
import { and, eq, inArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { adapterAuthSessions } from "@paperclipai/db";
import {
ADAPTER_AUTH_ACTIVE_STATUSES,
CODEX_DEVICE_LOGIN_ADAPTER_TYPE,
decodePendingTerminal,
LOGIN_LEASE_SESSION_TAG_KEY,
observeSandboxDelete,
@@ -166,6 +167,7 @@ export function createCodexDeviceLoginReaper(deps: CodexDeviceLoginReaperDeps) {
const pendingWrite = terminalCleanupWrite(false, "timed_out", null);
const claimed = await store.withCompanyAdapterPromotionLock(
row.companyId,
row.startedByUserId,
row.adapterType,
() =>
store.compareAndSetStatus({
@@ -313,10 +315,15 @@ export function createProductionLoginSessionReaperRuntime(
.selectDistinct({ environmentId: adapterAuthSessions.environmentId })
.from(adapterAuthSessions)
.where(
inArray(adapterAuthSessions.status, [
...ADAPTER_AUTH_ACTIVE_STATUSES,
"cleanup_pending",
]),
and(
// The shared table also holds the setup-token rows. Filter by the
// device-login adapter, so the orphan sweep reads only Codex rows.
eq(adapterAuthSessions.adapterType, CODEX_DEVICE_LOGIN_ADAPTER_TYPE),
inArray(adapterAuthSessions.status, [
...ADAPTER_AUTH_ACTIVE_STATUSES,
"cleanup_pending",
]),
),
);
const tagged: TaggedLease[] = [];
for (const { environmentId } of environmentRows) {
+117 -43
View File
@@ -21,6 +21,7 @@ import {
type SandboxLoginDriver,
} from "@paperclipai/adapter-codex-local/server";
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
import { buildLoginLeaseAcquireArgs } from "./adapter-login-lease.js";
import { environmentService } from "./environments.js";
// The login-session service. It creates a login session, acquires a fresh
@@ -104,11 +105,12 @@ export interface LoginSessionRuntime {
}
/** The per-session context the promotion seam needs. The service knows the
* session, the company, and the adapter, so the promotion resolves the company
* scope and the sole-active-owner check for this exact session. */
* session, the company, the owner, and the adapter, so the promotion resolves
* the company slot and the sole-active-owner check for this exact session. */
export interface CredentialPromotionContext {
sessionId: string;
companyId: string;
startedByUserId: string;
adapterType: AgentAdapterType;
}
@@ -210,6 +212,10 @@ export class AdapterAuthSessionConflictError extends Error {
export interface AdapterAuthSessionRow {
id: string;
/** The public, CSPRNG session identifier. The API returns and looks up this
* value. It never equals the internal primary-key `id`, so a caller cannot
* address a row by the internal id. */
publicSessionId: string;
companyId: string;
environmentId: string;
adapterType: AgentAdapterType;
@@ -227,6 +233,9 @@ export interface AdapterAuthSessionRow {
export interface InsertAdapterAuthSessionInput {
id: string;
/** The public, CSPRNG session identifier. The service builds it and returns it
* to the client; the store persists it in `public_session_id`. */
publicSessionId: string;
companyId: string;
environmentId: string;
adapterType: AgentAdapterType;
@@ -272,33 +281,44 @@ export interface AdapterAuthSessionStore {
/** A conditional status write. It returns true only when it changed one row. */
compareAndSetStatus(input: CompareAndSetAdapterAuthSessionStatusInput): Promise<boolean>;
get(sessionId: string): Promise<AdapterAuthSessionRow | null>;
/**
* Read a session by its public session id, scoped to the company and the Codex
* device-login adapter. The predicate carries the company id, so a query never
* keys on the public session id alone, and a foreign-company caller reads
* nothing. It never accepts the internal primary-key `id`.
*/
getByPublicId(publicSessionId: string, companyId: string): Promise<AdapterAuthSessionRow | null>;
/** Run `fn` while the process holds the promotion critical-section lock for the
* company and adapter. The reaper reclaims a stale `promoting` row inside this
* lock, so a reclaim never interleaves with a live credential write. */
* company, owner, and adapter slot. The reaper reclaims a stale `promoting`
* row inside this lock, so a reclaim never interleaves with a live credential
* write for the same slot. */
withCompanyAdapterPromotionLock<T>(
companyId: string,
startedByUserId: string,
adapterType: AgentAdapterType,
fn: () => Promise<T>,
): Promise<T>;
}
/** Build the advisory-lock key for the promotion critical section. The key is
* company-scoped and adapter-scoped, so two different company slots never
* contend. The credential-promotion path and the reaper reclaim both derive the
* key from this function, so they take the exact same lock. */
* scoped to the company, the owner, and the adapter, so two different slots
* never contend. The credential-promotion path and the reaper reclaim both
* derive the key from this function, so they take the exact same lock. */
export function adapterLoginPromotionLockKey(
companyId: string,
startedByUserId: string,
adapterType: AgentAdapterType,
): string {
return `paperclip:adapter-login-promotion:${companyId}:${adapterType}`;
return `paperclip:adapter-login-promotion:${companyId}:${startedByUserId}:${adapterType}`;
}
/**
* Run `fn` inside the promotion critical section for one company and adapter.
* Run `fn` inside the promotion critical section for one company, owner, and
* adapter slot.
*
* The function opens a database transaction and takes a transaction-scoped
* PostgreSQL advisory lock. The lock serializes the credential-promotion path
* against the reaper reclaim, so a reaper never releases the company slot while a
* against the reaper reclaim, so a reaper never releases the slot while a
* credential write runs, and a stale promotion never writes after the reaper
* reclaims the slot. The transaction holds the lock through `fn`, so the caller
* runs the ownership check and the credential write in one mutually-exclusive
@@ -308,10 +328,11 @@ export function adapterLoginPromotionLockKey(
export async function withAdapterLoginPromotionLock<T>(
db: Db,
companyId: string,
startedByUserId: string,
adapterType: AgentAdapterType,
fn: () => Promise<T>,
): Promise<T> {
const key = adapterLoginPromotionLockKey(companyId, adapterType);
const key = adapterLoginPromotionLockKey(companyId, startedByUserId, adapterType);
return db.transaction(async (tx) => {
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${key}, 0))`);
return fn();
@@ -354,10 +375,12 @@ export interface AdapterAuthReaperStore {
compareAndSetStatus(input: CompareAndSetAdapterAuthSessionStatusInput): Promise<boolean>;
get(sessionId: string): Promise<AdapterAuthSessionRow | null>;
/** Run `fn` while the process holds the promotion critical-section lock for the
* company and adapter. The reaper reclaims a stale `promoting` row inside this
* lock, so a reclaim never interleaves with a live credential write. */
* company, owner, and adapter slot. The reaper reclaims a stale `promoting`
* row inside this lock, so a reclaim never interleaves with a live credential
* write for the same slot. */
withCompanyAdapterPromotionLock<T>(
companyId: string,
startedByUserId: string,
adapterType: AgentAdapterType,
fn: () => Promise<T>,
): Promise<T>;
@@ -385,12 +408,16 @@ function isUniqueViolation(error: unknown): boolean {
function toRow(row: typeof adapterAuthSessions.$inferSelect): AdapterAuthSessionRow {
return {
id: row.id,
publicSessionId: row.publicSessionId,
companyId: row.companyId,
environmentId: row.environmentId,
adapterType: row.adapterType,
startedByUserId: row.startedByUserId,
providerLeaseId: row.providerLeaseId ?? null,
status: row.status,
// The unified `status` column type is the merged login-state union. A Codex
// device-login row only ever holds a Codex internal status, so narrow the
// read back to the internal union the service reasons about.
status: row.status as AdapterAuthSessionInternalStatus,
expiresAt: row.expiresAt ?? null,
promotionExpiresAt: row.promotionExpiresAt ?? null,
finishedAt: row.finishedAt ?? null,
@@ -427,6 +454,11 @@ export function createDbAdapterAuthSessionStore(
environmentId: input.environmentId,
adapterType: input.adapterType,
startedByUserId: input.startedByUserId,
// The unified table requires a unique public session id. The service
// builds it from a CSPRNG and returns it to the client, so the store
// persists that value here. It never uses the internal id, a timestamp,
// or a counter.
publicSessionId: input.publicSessionId,
status: "starting",
expiresAt: input.expiresAt,
createdAt: input.at,
@@ -477,6 +509,25 @@ export function createDbAdapterAuthSessionStore(
const row = rows[0];
return row ? toRow(row) : null;
},
async getByPublicId(publicSessionId, companyId) {
// The predicate carries the company id and the Codex device-login adapter,
// so a read never keys on the public session id alone. A foreign-company or
// foreign-adapter caller reads nothing. The internal primary-key `id` never
// matches, so a caller cannot address a row by the internal id.
const rows = await db
.select()
.from(adapterAuthSessions)
.where(
and(
eq(adapterAuthSessions.publicSessionId, publicSessionId),
eq(adapterAuthSessions.companyId, companyId),
eq(adapterAuthSessions.adapterType, CODEX_DEVICE_LOGIN_ADAPTER_TYPE),
),
)
.limit(1);
const row = rows[0];
return row ? toRow(row) : null;
},
async listExpiredActiveSessions(nowAt) {
// The partial index on the active statuses and the index on `expiresAt`
// both support this scan. The scan is bounded by the active-status set, so
@@ -489,6 +540,9 @@ export function createDbAdapterAuthSessionStore(
.from(adapterAuthSessions)
.where(
and(
// The shared table also holds the setup-token rows, so every reaper
// scan filters by the device-login adapter to reach only Codex rows.
eq(adapterAuthSessions.adapterType, CODEX_DEVICE_LOGIN_ADAPTER_TYPE),
inArray(adapterAuthSessions.status, [...ADAPTER_AUTH_ACTIVE_STATUSES]),
isNotNull(adapterAuthSessions.expiresAt),
lte(adapterAuthSessions.expiresAt, nowAt),
@@ -505,20 +559,30 @@ export function createDbAdapterAuthSessionStore(
const rows = await db
.select()
.from(adapterAuthSessions)
.where(eq(adapterAuthSessions.status, "cleanup_pending"));
.where(
and(
eq(adapterAuthSessions.adapterType, CODEX_DEVICE_LOGIN_ADAPTER_TYPE),
eq(adapterAuthSessions.status, "cleanup_pending"),
),
);
return rows.map(toRow);
},
async listLeaseReferences() {
const rows = await db
.select({ providerLeaseId: adapterAuthSessions.providerLeaseId })
.from(adapterAuthSessions)
.where(isNotNull(adapterAuthSessions.providerLeaseId));
.where(
and(
eq(adapterAuthSessions.adapterType, CODEX_DEVICE_LOGIN_ADAPTER_TYPE),
isNotNull(adapterAuthSessions.providerLeaseId),
),
);
return rows
.map((row) => row.providerLeaseId)
.filter((value): value is string => value != null);
},
async withCompanyAdapterPromotionLock(companyId, adapterType, fn) {
return withAdapterLoginPromotionLock(db, companyId, adapterType, fn);
async withCompanyAdapterPromotionLock(companyId, startedByUserId, adapterType, fn) {
return withAdapterLoginPromotionLock(db, companyId, startedByUserId, adapterType, fn);
},
};
}
@@ -644,6 +708,10 @@ export function createCodexDeviceLoginService(deps: CodexDeviceLoginServiceDeps)
input: StartCodexDeviceLoginInput,
): Promise<StartCodexDeviceLoginResult> {
const sessionId = randomUUID();
// The public session identifier the API returns and looks up. It is an
// independent CSPRNG value, so it never equals the internal `sessionId` and a
// caller cannot address the row by the internal id.
const publicSessionId = randomUUID();
const startedAt = now();
const ttlSeconds = input.ttlSeconds ?? CODEX_DEVICE_LOGIN_TIMEOUT_MS / 1000;
const expiresAt = new Date(startedAt.getTime() + ttlSeconds * 1000);
@@ -662,6 +730,7 @@ export function createCodexDeviceLoginService(deps: CodexDeviceLoginServiceDeps)
// acquires a lease for a losing start.
await store.insert({
id: sessionId,
publicSessionId,
companyId: input.companyId,
environmentId: input.environmentId,
adapterType: input.adapterType,
@@ -726,7 +795,8 @@ export function createCodexDeviceLoginService(deps: CodexDeviceLoginServiceDeps)
activity("lease_acquired");
const session: AdapterAuthSessionResponse = {
sessionId,
// The API identifier is the public session id, never the internal `sessionId`.
sessionId: publicSessionId,
environmentId: input.environmentId,
status: "starting",
expiresAt: expiresAt.toISOString(),
@@ -837,6 +907,7 @@ export function createCodexDeviceLoginService(deps: CodexDeviceLoginServiceDeps)
await promotion.promote(credential, {
sessionId,
companyId: input.companyId,
startedByUserId: input.startedByUserId,
adapterType: input.adapterType,
});
} catch {
@@ -972,24 +1043,28 @@ export function createCodexDeviceLoginService(deps: CodexDeviceLoginServiceDeps)
}
async function readOwnerSession(
sessionId: string,
publicSessionId: string,
companyId: string,
requestingUserId: string,
): Promise<AdapterAuthSessionOwnerResponse | null> {
const row = await store.get(sessionId);
// Look the row up by its public session id, scoped to the company. A
// foreign-company caller reads nothing, and the internal id never matches.
const row = await store.getByPublicId(publicSessionId, companyId);
if (!row) return null;
const isOwner = row.startedByUserId === requestingUserId;
const status = resolvePublicStatus(row);
// Deliver the one-time prompt to the owner principal exactly once. The read
// and the delete run with no await between them, so the first authorized
// owner read consumes the prompt and every later read returns null. This
// keeps the short-lived device code out of a repeated response.
// keeps the short-lived device code out of a repeated response. The prompt
// map keys on the internal id, so read it by `row.id`, not the public id.
let prompt: DeviceLoginPrompt | null = null;
if (isOwner) {
prompt = promptsBySession.get(sessionId) ?? null;
if (prompt) promptsBySession.delete(sessionId);
prompt = promptsBySession.get(row.id) ?? null;
if (prompt) promptsBySession.delete(row.id);
}
return {
sessionId: row.id,
sessionId: row.publicSessionId,
environmentId: row.environmentId,
status,
expiresAt: row.expiresAt?.toISOString() ?? null,
@@ -1011,14 +1086,18 @@ export function createCodexDeviceLoginService(deps: CodexDeviceLoginServiceDeps)
// row, so a cancel never interrupts an in-flight credential write; that
// promotion runs to its own terminal.
async function cancelOwnerSession(
sessionId: string,
publicSessionId: string,
companyId: string,
requestingUserId: string,
): Promise<AdapterAuthSessionOwnerResponse | null> {
const row = await store.get(sessionId);
// Look the row up by its public session id, scoped to the company. The status
// write keys on the internal `row.id`, so a cancel never addresses a row by
// the public id alone.
const row = await store.getByPublicId(publicSessionId, companyId);
if (!row || row.startedByUserId !== requestingUserId) return null;
const write = terminalCleanupWrite(false, "cancelled", null);
await store.compareAndSetStatus({
sessionId,
sessionId: row.id,
expectedStatuses: ["starting", "waiting_for_user"],
status: write.status,
at: now(),
@@ -1026,7 +1105,7 @@ export function createCodexDeviceLoginService(deps: CodexDeviceLoginServiceDeps)
finishedAt: now(),
promotionExpiresAt: null,
});
return readOwnerSession(sessionId, requestingUserId);
return readOwnerSession(publicSessionId, companyId, requestingUserId);
}
return { start, readOwnerSession, cancelOwnerSession };
@@ -1157,20 +1236,15 @@ export function createProductionLoginSessionRuntime(
if (!environment) {
throw new Error(`Environment "${input.environmentId}" is not found.`);
}
const record = await deps.environmentRuntime.acquireRunLease({
companyId: input.companyId,
environment,
issueId: null,
agentId: null,
// A null heartbeat run and a null execution workspace disable lease
// reuse, so the login session always runs in a fresh sandbox.
heartbeatRunId: null,
persistedExecutionWorkspace: null,
adapterType: input.adapterType,
// Apply the active custom-image template, so the sandbox binds to the
// trusted image and runtime identity.
applyCustomImageTemplate: true,
});
const record = await deps.environmentRuntime.acquireRunLease(
buildLoginLeaseAcquireArgs({
metadata: {
companyId: input.companyId,
environment,
adapterType: input.adapterType,
},
}),
);
// Tag the lease with the session identifier, so the reaper resolves an
// orphan lease that no live session references. The tag carries no secret.
await environmentsSvc.updateLeaseMetadata(record.lease.id, {
@@ -219,6 +219,313 @@ export function classifyEnvironmentCustomImageConfigChange(input: {
return "relinkable";
}
// --- Operator relink action (server-owned boot-source classification) --------
//
// A saved config change moves the fingerprint and detaches an otherwise-valid
// template. The relink action re-stamps the fingerprint, but only after the
// server decides the boot source did not change. To decide that without the
// capture-time config, the capture persists a small, server-owned snapshot of
// the boot-relevant fields. The relink then compares that snapshot to the
// current config. The comparison runs on the server; the client never
// classifies.
export const ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_METADATA_KEY = "bootRelevantConfig";
export const ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_VERSION = 1;
// A driver-declared identity path that fails canonicalization is never trusted
// and never persisted verbatim. It records this fixed marker instead, which
// forces the fail-closed `unclassified` result at relink time.
export const ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_UNRESOLVED_PATH = "[unresolved-identity-path]";
const BOOT_RELEVANT_CONFIG_MAX_PATH_DEPTH = 8;
const BOOT_RELEVANT_CONFIG_PATH_SEGMENT_RE = /^[A-Za-z_][A-Za-z0-9_-]*$/;
// Prototype keys have an identifier shape but never name an own config field.
// A driver that declares one is hostile or broken. Reject it so the path fails
// closed as an unresolved identity path.
const BOOT_RELEVANT_CONFIG_RESERVED_SEGMENTS = new Set(["__proto__", "prototype", "constructor"]);
export type EnvironmentCustomImageRelinkClassification =
| "knob_only"
| "boot_source_drift"
| "unclassified";
/**
* Server-owned snapshot of the boot-relevant config fields at capture time. It
* carries the field values that decide the boot source, with every secret-ref
* path and every unresolvable driver path excluded (path name only, no value).
*/
export interface EnvironmentCustomImageBootRelevantConfig {
version: typeof ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_VERSION;
provider: string;
bindingField: string;
values: Record<string, unknown>;
excludedPaths: string[];
}
export interface EnvironmentCustomImageDriftedPath {
path: string;
from?: unknown;
to?: unknown;
}
/**
* Canonicalizes a config path as a bounded dot-path. Each segment must be a
* plain identifier and the depth is bounded. An invalid, ambiguous, or
* unresolvable path returns `null`; the caller must never persist the raw
* driver-supplied string.
*/
export function canonicalizeEnvironmentCustomImageConfigPath(raw: unknown): string | null {
if (typeof raw !== "string") return null;
const trimmed = raw.trim();
if (!trimmed) return null;
const segments = trimmed.split(".");
if (segments.length === 0 || segments.length > BOOT_RELEVANT_CONFIG_MAX_PATH_DEPTH) return null;
for (const segment of segments) {
if (!BOOT_RELEVANT_CONFIG_PATH_SEGMENT_RE.test(segment)) return null;
if (BOOT_RELEVANT_CONFIG_RESERVED_SEGMENTS.has(segment)) return null;
}
return segments.join(".");
}
/**
* Reports whether a candidate path overlaps any secret-ref path by exact,
* ancestor, or descendant containment.
*/
function environmentCustomImageConfigPathOverlapsSecret(
candidate: string,
secretPaths: Iterable<string>,
): boolean {
for (const raw of secretPaths) {
const secret = canonicalizeEnvironmentCustomImageConfigPath(raw);
if (!secret) continue;
if (candidate === secret) return true;
if (candidate.startsWith(`${secret}.`)) return true;
if (secret.startsWith(`${candidate}.`)) return true;
}
return false;
}
/**
* The current boot-relevant path contract a provider driver declares. It names
* the config binding and the driver identity paths that decide the boot source
* now. A relink compares a persisted snapshot against this contract to detect a
* provider change since capture.
*/
export interface EnvironmentCustomImageBootRelevantContract {
binding: EnvironmentCustomImageRuntimeConfigBinding;
templateIdentityPaths?: Iterable<string>;
}
/**
* Computes the canonical candidate boot-relevant paths for a binding and its
* driver identity paths. Both the capture snapshot and the relink staleness
* check use this function, so they share one path set. `hasUnresolved` is true
* when any raw path fails canonicalization.
*/
export function environmentCustomImageBootRelevantCandidatePaths(
contract: EnvironmentCustomImageBootRelevantContract,
): { canonicalPaths: string[]; hasUnresolved: boolean } {
const canonicalPaths: string[] = [];
const seen = new Set<string>();
let hasUnresolved = false;
const rawCandidatePaths = [
contract.binding.field,
...contract.binding.unsetFields,
...ENVIRONMENT_CUSTOM_IMAGE_TEMPLATE_SOURCE_FIELDS,
...(contract.templateIdentityPaths ?? []),
];
for (const rawPath of rawCandidatePaths) {
const canonical = canonicalizeEnvironmentCustomImageConfigPath(rawPath);
if (!canonical) {
hasUnresolved = true;
continue;
}
if (seen.has(canonical)) continue;
seen.add(canonical);
canonicalPaths.push(canonical);
}
return { canonicalPaths, hasUnresolved };
}
/**
* Builds the capture-time boot-relevant snapshot from the parsed config only.
* Candidate paths are the runtime binding field, the binding unset fields, the
* standard boot-source fields, and every driver identity path. A secret-ref
* overlap or an unresolvable path is recorded in `excludedPaths` with no value.
*/
export function buildEnvironmentCustomImageBootRelevantConfig(input: {
config: SandboxEnvironmentConfig;
binding: EnvironmentCustomImageRuntimeConfigBinding;
templateIdentityPaths?: Iterable<string>;
secretRefExcludePaths?: Iterable<string>;
}): EnvironmentCustomImageBootRelevantConfig {
const config = input.config as Record<string, unknown>;
const secretPaths = [...(input.secretRefExcludePaths ?? [])];
// A null-prototype map is a second guard: a reserved segment assignment can
// never mutate the prototype, so a hostile path cannot vanish from `values`.
const values: Record<string, unknown> = Object.create(null);
const excludedPaths = new Set<string>();
const { canonicalPaths, hasUnresolved } = environmentCustomImageBootRelevantCandidatePaths({
binding: input.binding,
templateIdentityPaths: input.templateIdentityPaths,
});
if (hasUnresolved) {
excludedPaths.add(ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_UNRESOLVED_PATH);
}
for (const canonical of canonicalPaths) {
if (environmentCustomImageConfigPathOverlapsSecret(canonical, secretPaths)) {
excludedPaths.add(canonical);
continue;
}
const value = readConfigValueAtPath(config, canonical);
// An absent field is stored as `null` so that "absent then, absent now"
// compares equal at relink time.
values[canonical] = value === undefined ? null : value;
}
return {
version: ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_VERSION,
provider: input.config.provider,
bindingField: input.binding.field,
values,
excludedPaths: [...excludedPaths],
};
}
/**
* Reports whether a persisted snapshot no longer matches the current provider
* boot-relevant contract. A provider plugin can change its config binding or
* its identity paths after capture. When it does, a field that is boot-relevant
* now but absent from the snapshot would compare equal by absence and hide real
* drift. The check fails closed on:
*
* - a binding field that differs from the captured `bindingField`;
* - a current identity path the snapshot never covered (neither a value nor an
* excluded path);
* - a current contract that itself carries an unresolvable identity path.
*/
export function environmentCustomImageBootRelevantSnapshotIsStale(input: {
bootRelevantConfig: EnvironmentCustomImageBootRelevantConfig;
currentContract: EnvironmentCustomImageBootRelevantContract;
}): boolean {
const boot = input.bootRelevantConfig;
if (input.currentContract.binding.field !== boot.bindingField) return true;
const { canonicalPaths, hasUnresolved } = environmentCustomImageBootRelevantCandidatePaths(
input.currentContract,
);
if (hasUnresolved) return true;
const covered = new Set<string>([
...Object.keys(boot.values),
...boot.excludedPaths,
]);
return canonicalPaths.some((path) => !covered.has(path));
}
/**
* Reads and validates the persisted boot-relevant snapshot. Returns `null` for
* a legacy template with no snapshot or for a malformed shape; both classify as
* `unclassified` (fail closed).
*/
export function readEnvironmentCustomImageBootRelevantConfig(
metadata: Record<string, unknown> | null | undefined,
): EnvironmentCustomImageBootRelevantConfig | null {
const raw = metadata?.[ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_METADATA_KEY];
if (!isRecord(raw)) return null;
if (raw.version !== ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_VERSION) return null;
if (typeof raw.provider !== "string" || typeof raw.bindingField !== "string") return null;
if (!isRecord(raw.values)) return null;
if (!Array.isArray(raw.excludedPaths)) return null;
const excludedPaths = raw.excludedPaths.filter(
(entry): entry is string => typeof entry === "string",
);
return {
version: ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_VERSION,
provider: raw.provider,
bindingField: raw.bindingField,
values: raw.values,
excludedPaths,
};
}
/**
* Classifies the drift between the capture-time boot-relevant snapshot and the
* current parsed config.
*
* - `knob_only`: the snapshot is present, no path was excluded, and every value
* still matches. The fingerprint can be re-stamped without confirmation.
* - `boot_source_drift`: a value-bearing path differs. The operator must
* confirm before the re-stamp.
* - `unclassified`: no snapshot, an excluded path, a provider mismatch, or a
* snapshot that no longer matches the current provider contract. The server
* cannot verify the boot source; the operator must confirm (fail closed).
*
* `driftedPaths` carries raw `from`/`to` values only for value-bearing paths
* that passed containment at capture. Excluded paths carry the path name only.
*
* `currentContract` is the current provider boot-relevant contract. When the
* caller cannot resolve the driver, it passes `null`; the server then cannot
* verify the boot source and the result is `unclassified` (fail closed).
*/
export function classifyEnvironmentCustomImageBootRelevantDrift(input: {
bootRelevantConfig: EnvironmentCustomImageBootRelevantConfig | null;
currentConfig: SandboxEnvironmentConfig;
currentContract: EnvironmentCustomImageBootRelevantContract | null;
}): {
classification: EnvironmentCustomImageRelinkClassification;
driftedPaths: EnvironmentCustomImageDriftedPath[];
} {
const boot = input.bootRelevantConfig;
if (!boot) return { classification: "unclassified", driftedPaths: [] };
// Without the current provider contract the boot source cannot be verified
// against a possible identity-path change; fail closed.
if (!input.currentContract) return { classification: "unclassified", driftedPaths: [] };
// A provider change since capture (new binding or new identity path) makes
// the persisted snapshot untrustworthy; fail closed.
if (environmentCustomImageBootRelevantSnapshotIsStale({
bootRelevantConfig: boot,
currentContract: input.currentContract,
})) {
return { classification: "unclassified", driftedPaths: [] };
}
const current = input.currentConfig as Record<string, unknown>;
const driftedPaths: EnvironmentCustomImageDriftedPath[] = [];
let hasValueDrift = false;
for (const [path, capturedValue] of Object.entries(boot.values)) {
const currentValue = readConfigValueAtPath(current, path);
if (stableStringify(capturedValue ?? null) !== stableStringify(currentValue ?? null)) {
hasValueDrift = true;
driftedPaths.push({ path, from: capturedValue ?? null, to: currentValue ?? null });
}
}
for (const path of boot.excludedPaths) {
driftedPaths.push({ path });
}
const providerMismatch = boot.provider !== input.currentConfig.provider;
if (boot.excludedPaths.length > 0 || providerMismatch) {
return { classification: "unclassified", driftedPaths };
}
if (hasValueDrift) {
return { classification: "boot_source_drift", driftedPaths };
}
return { classification: "knob_only", driftedPaths: [] };
}
/**
* Removes the server-only boot-relevant snapshot from a template's metadata.
* The snapshot exists to classify a later relink and holds raw boot-source
* config values. It must never reach an API response. The runtime keeps it in
* the persisted row and reads it straight from the row at relink time.
*/
export function stripInternalEnvironmentCustomImageTemplateMetadata(
metadata: Record<string, unknown> | null | undefined,
): Record<string, unknown> | null {
if (!metadata) return null;
if (!(ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_METADATA_KEY in metadata)) return metadata;
const rest = { ...metadata };
delete rest[ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_METADATA_KEY];
return rest;
}
export function environmentCustomImageTemplateFromRow(row: TemplateRow): EnvironmentCustomImageTemplate {
return {
id: row.id,
@@ -234,7 +541,9 @@ export function environmentCustomImageTemplateFromRow(row: TemplateRow): Environ
capturedAt: row.capturedAt ?? null,
lastUsedAt: row.lastUsedAt ?? null,
supersededByTemplateId: row.supersededByTemplateId ?? null,
metadata: row.metadata ?? null,
// The boot-relevant snapshot is server-internal; keep it out of every
// template response. The relink path reads it from the persisted row.
metadata: stripInternalEnvironmentCustomImageTemplateMetadata(row.metadata),
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
+179 -10
View File
@@ -39,7 +39,11 @@ import {
import { environmentService } from "./environments.js";
import {
ENVIRONMENT_CUSTOM_IMAGE_CONFIG_FINGERPRINT_EXCLUDED_PATHS,
ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_METADATA_KEY,
classifyEnvironmentCustomImageConfigChange,
classifyEnvironmentCustomImageBootRelevantDrift,
buildEnvironmentCustomImageBootRelevantConfig,
readEnvironmentCustomImageBootRelevantConfig,
fingerprintEnvironmentSandboxProviderConfig,
ENVIRONMENT_CUSTOM_IMAGE_RUNTIME_CONFIG_BINDING_METADATA_KEY,
defaultEnvironmentCustomImageRuntimeConfigBinding,
@@ -47,7 +51,9 @@ import {
normalizeEnvironmentCustomImageRuntimeConfigBinding,
environmentCustomImageTemplateFromRow,
readEnvironmentCustomImageTemplateKind as readTemplateKind,
type EnvironmentCustomImageRelinkClassification,
} from "./environment-custom-image-runtime.js";
import { logActivity } from "./activity-log.js";
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
const ACTIVE_SETUP_STATUSES = ["starting", "waiting_for_user", "capturing"] as const;
@@ -230,10 +236,10 @@ function sourceTemplateFromConfig(
return { sourceTemplateRef: null, sourceTemplateKind: null };
}
async function resolveActiveTemplate(
async function resolveActiveTemplateRow(
db: Db,
input: { environmentId: string; provider?: string | null },
): Promise<EnvironmentCustomImageTemplate | null> {
): Promise<typeof environmentCustomImageTemplates.$inferSelect | null> {
const conditions = [
eq(environmentCustomImageTemplates.environmentId, input.environmentId),
eq(environmentCustomImageTemplates.status, "active"),
@@ -241,12 +247,19 @@ async function resolveActiveTemplate(
if (input.provider) {
conditions.push(eq(environmentCustomImageTemplates.provider, input.provider));
}
const row = await db
return db
.select()
.from(environmentCustomImageTemplates)
.where(and(...conditions))
.orderBy(desc(environmentCustomImageTemplates.capturedAt), desc(environmentCustomImageTemplates.createdAt))
.then((rows) => rows[0] ?? null);
}
async function resolveActiveTemplate(
db: Db,
input: { environmentId: string; provider?: string | null },
): Promise<EnvironmentCustomImageTemplate | null> {
const row = await resolveActiveTemplateRow(db, input);
return row ? environmentCustomImageTemplateFromRow(row) : null;
}
@@ -803,11 +816,14 @@ export function environmentCustomImageService(
const captured = await callProviderCapture({ session, previousTemplate: currentActive });
const environment = await requireEnvironment(session.environmentId);
const parsed = parseEnvironmentDriverConfig(environment);
const captureSecretRefExcludePaths = parsed.driver === "sandbox"
? [...await resolveSandboxProviderSecretRefPaths(db, parsed.config.provider)]
: [];
const baseFingerprint = parsed.driver === "sandbox"
? fingerprintEnvironmentSandboxProviderConfig(parsed.config, {
excludePaths: [
...ENVIRONMENT_CUSTOM_IMAGE_CONFIG_FINGERPRINT_EXCLUDED_PATHS,
...await resolveSandboxProviderSecretRefPaths(db, parsed.config.provider),
...captureSecretRefExcludePaths,
],
})
: null;
@@ -821,6 +837,32 @@ export function environmentCustomImageService(
templateRefKind: captured.templateKind,
templateConfigBinding: provider.driver.templateConfigBinding,
});
// Server-owned boot-relevant snapshot from the parsed config only. It
// lets a later relink decide, without the capture-time config, whether
// the boot source changed. Secret-ref paths carry no value.
const bootRelevantConfig = parsed.driver === "sandbox"
? buildEnvironmentCustomImageBootRelevantConfig({
config: parsed.config,
binding: runtimeConfigBinding,
templateIdentityPaths: provider.driver.templateIdentityPaths ?? [],
secretRefExcludePaths: captureSecretRefExcludePaths,
})
: null;
const baseTemplateMetadata = normalizeProviderMetadata({
...(captured.metadata ?? {}),
...(input.metadata ? { userMetadata: input.metadata } : {}),
...persistedSetupMetadata(session.metadata),
[ENVIRONMENT_CUSTOM_IMAGE_RUNTIME_CONFIG_BINDING_METADATA_KEY]: runtimeConfigBinding,
}) ?? {};
// The boot-relevant snapshot bypasses the provider-metadata redactor:
// its config field names (for example `apiUrl`) would otherwise be
// redacted by key and the relink comparison could never match.
const templateMetadata = bootRelevantConfig
? {
...baseTemplateMetadata,
[ENVIRONMENT_CUSTOM_IMAGE_BOOT_RELEVANT_CONFIG_METADATA_KEY]: bootRelevantConfig,
}
: baseTemplateMetadata;
const now = input.now ?? new Date();
const templateRow = await db.transaction(async (tx) => {
const templateId = randomUUID();
@@ -851,12 +893,7 @@ export function environmentCustomImageService(
createdByUserId: session.startedByUserId,
createdByAgentId: session.startedByAgentId,
capturedAt: now,
metadata: normalizeProviderMetadata({
...(captured.metadata ?? {}),
...(input.metadata ? { userMetadata: input.metadata } : {}),
...persistedSetupMetadata(session.metadata),
[ENVIRONMENT_CUSTOM_IMAGE_RUNTIME_CONFIG_BINDING_METADATA_KEY]: runtimeConfigBinding,
}),
metadata: templateMetadata,
createdAt: now,
updatedAt: now,
})
@@ -999,6 +1036,138 @@ export function environmentCustomImageService(
};
},
/**
* Re-stamps the active template's source fingerprint so a detached-but-valid
* template applies again, without a sandbox boot or a new provider snapshot.
* The server classifies the drift between the capture-time boot-relevant
* snapshot and the current config. A `boot_source_drift` or `unclassified`
* result needs the `confirmBootSourceDrift` flag; the client never
* classifies. The re-stamp UPDATE and the activity row run in one
* transaction. Zero updated rows abort with a conflict and write no activity
* row.
*/
relinkActiveTemplate: async (input: {
environmentId: string;
confirmBootSourceDrift?: boolean;
actor: {
actorType: "agent" | "user" | "system" | "plugin";
actorId: string;
agentId?: string | null;
runId?: string | null;
agentApiKeyId?: string | null;
};
companyId: string;
now?: Date;
}): Promise<{
template: EnvironmentCustomImageTemplate;
classification: EnvironmentCustomImageRelinkClassification;
}> => {
const environment = await requireEnvironment(input.environmentId);
const parsed = parseEnvironmentDriverConfig(environment);
if (parsed.driver !== "sandbox") {
throw unprocessable("Environment customImage relink is only supported for sandbox environments.");
}
const activeRow = await resolveActiveTemplateRow(db, {
environmentId: input.environmentId,
provider: parsed.config.provider,
});
if (!activeRow) throw notFound("Active environment customImage template not found");
const active = environmentCustomImageTemplateFromRow(activeRow);
const secretRefExcludePaths = parsed.config.provider === "fake"
? []
: [...await resolveSandboxProviderSecretRefPaths(db, parsed.config.provider)];
// Resolve the current provider contract so the classifier can reject a
// snapshot captured against a different binding or identity-path set. A
// driver that no longer resolves fails closed (null contract).
const resolvedDriver = await resolvePluginSandboxProviderDriverByKey({
db,
driverKey: active.provider,
});
const currentContract = resolvedDriver
? {
binding: templateConfigBindingFromDriver({
templateRefKind: active.templateKind,
templateConfigBinding: resolvedDriver.driver.templateConfigBinding,
}),
templateIdentityPaths: resolvedDriver.driver.templateIdentityPaths ?? [],
}
: null;
// The persisted snapshot is server-internal; read it from the row, not the
// sanitized template response.
const drift = classifyEnvironmentCustomImageBootRelevantDrift({
bootRelevantConfig: readEnvironmentCustomImageBootRelevantConfig(activeRow.metadata),
currentConfig: parsed.config,
currentContract,
});
const confirmBootSourceDrift = input.confirmBootSourceDrift === true;
if (drift.classification !== "knob_only" && !confirmBootSourceDrift) {
throw conflict(
drift.classification === "boot_source_drift"
? "The base image changed since this template was captured. Confirm the relink to keep the captured snapshot."
: "The server cannot verify the boot source for this template. Confirm the relink to keep the captured snapshot.",
{
classification: drift.classification,
driftedPaths: drift.driftedPaths,
},
);
}
const nextFingerprint = fingerprintEnvironmentSandboxProviderConfig(parsed.config, {
excludePaths: [
...ENVIRONMENT_CUSTOM_IMAGE_CONFIG_FINGERPRINT_EXCLUDED_PATHS,
...secretRefExcludePaths,
],
});
const now = input.now ?? new Date();
// Activity details carry the sanitized classification and canonical path
// names only. They never carry a fingerprint or a config value.
const activityDetails = {
environmentId: input.environmentId,
templateId: active.id,
provider: active.provider,
classification: drift.classification,
driftedPaths: drift.driftedPaths.map((entry) => entry.path),
confirmBootSourceDrift,
};
const row = await db.transaction(async (tx) => {
const updated = await tx
.update(environmentCustomImageTemplates)
.set({ sourceEnvironmentConfigFingerprint: nextFingerprint, updatedAt: now })
.where(and(
eq(environmentCustomImageTemplates.id, active.id),
eq(environmentCustomImageTemplates.status, "active"),
eq(environmentCustomImageTemplates.environmentId, input.environmentId),
eq(environmentCustomImageTemplates.provider, active.provider),
))
.returning()
.then((rows) => rows[0] ?? null);
if (!updated) {
throw conflict("Active environment customImage template changed before relink; retry.");
}
await logActivity(tx as unknown as Db, {
companyId: input.companyId,
actorType: input.actor.actorType,
actorId: input.actor.actorId,
agentId: input.actor.agentId ?? null,
runId: input.actor.runId ?? null,
agentApiKeyId: input.actor.agentApiKeyId ?? null,
action: "environment.custom_image_template.relinked",
entityType: "environment",
entityId: input.environmentId,
details: activityDetails,
});
return updated;
});
return {
template: environmentCustomImageTemplateFromRow(row),
classification: drift.classification,
};
},
rollbackTemplate: async (input: {
environmentId: string;
now?: Date;
@@ -317,6 +317,12 @@ export async function resolveEnvironmentExecutionTarget(input: {
// here. The client falls back to the chunked upload path when this is
// false.
supportsSingleStreamStdinProgress: false,
// Carry the verified concurrent-sync opt-in to the sync client. The
// client copies it onto the native path and ignores it on the base64
// fallback, which always permits concurrency. A null snapshot or a
// provider that never opted in keeps it false, so an unverified
// provider never permits concurrent sync operations.
allowConcurrentSyncOperations: effectiveCapabilities?.concurrentSyncOperations === true,
execute: async (commandInput) => {
// Record true start and stop timestamps around the provider await,
// so the exec span and the result carry a real wall time.
@@ -499,6 +505,21 @@ export async function resolveEnvironmentExecutionTarget(input: {
}),
}
: {}),
// Expose the duplex channel only when the effective snapshot grants
// the opt-in `duplexCommandStream` capability. A null snapshot
// (resolution failed or the snapshot is not resolvable) leaves the
// member undefined, so the caller keeps the file bridge. This mirrors
// the syncIn/syncOut gate above and fails closed.
...(effectiveCapabilities?.duplexCommandStream
? {
openDuplexChannel: (channelInput) =>
input.environmentRuntime!.openDuplexChannel({
environment: input.environment as Environment,
lease: input.lease!,
command: channelInput.command,
}),
}
: {}),
}
: undefined,
};
@@ -45,6 +45,7 @@ import {
import { buildWorkspaceRealizationRequest } from "./workspace-realization.js";
import { executionWorkspaceService } from "./execution-workspaces.js";
import { logActivity } from "./activity-log.js";
import { logger } from "../middleware/logger.js";
import { parseObject } from "../adapters/utils.js";
import type { RealizedExecutionWorkspace } from "./workspace-runtime.js";
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
@@ -420,7 +421,29 @@ export function environmentRunOrchestrator(
(typeof lease.metadata?.remoteCwd === "string" && lease.metadata.remoteCwd.trim().length > 0
? lease.metadata.remoteCwd.trim()
: executionWorkspace.cwd);
if (provisionCommand && environment.driver !== "local") {
// The host `provisionCommand` runs on the host worktree during the
// `workspace_provision` step, before the run reaches the environment.
// A `sandbox`-driver environment does not receive the repo tree here. The
// sandbox driver `realizeWorkspace` step only creates the remote folder.
// The adapter uploads the provisioned tree later, in its `stage.sync` step.
// So the host command must not run inside the still-empty sandbox; it fails
// there (exit 127). Skip the step for `sandbox`, and keep the existing skip
// for `local`. Keep the step for `ssh`, which runs the command on the
// remote host that shares the workspace path.
const driverSkipsHostProvision =
environment.driver === "local" || environment.driver === "sandbox";
if (provisionCommand && environment.driver === "sandbox") {
logger.info(
{
environmentId: environment.id,
driver: environment.driver,
issueId,
heartbeatRunId,
},
"Skip host provisionCommand for sandbox-driver environment; the adapter stage.sync step delivers the provisioned tree",
);
}
if (provisionCommand && !driverSkipsHostProvision) {
try {
const provisionResult = await environmentRuntime.execute({
environment,
+143 -7
View File
@@ -14,6 +14,9 @@ import type {
} from "@paperclipai/shared";
import { resolveDeclaredSandboxCapabilities } from "@paperclipai/shared";
import type { EffectiveSandboxCapabilities } from "@paperclipai/adapter-utils/execution-target";
import type {
CommandManagedDuplexChannel,
} from "@paperclipai/adapter-utils/command-managed-runtime";
import type {
PluginEnvironmentAcquireLeaseParams,
PluginEnvironmentExecuteResult,
@@ -52,7 +55,12 @@ import {
sandboxConfigFromLeaseMetadataLoose,
} from "./sandbox-provider-runtime.js";
import { pluginRegistryService } from "./plugin-registry.js";
import type { ExecuteLogSink, PluginWorkerManager } from "./plugin-worker-manager.js";
import type {
ExecuteLogSink,
PluginWorkerManager,
DuplexChannelHostSession,
DuplexChannelOpenInput as WorkerManagerDuplexChannelOpenInput,
} from "./plugin-worker-manager.js";
import {
REUSABLE_LEASE_WORKER_METHODS,
destroyPluginEnvironmentLease,
@@ -78,6 +86,12 @@ import { logger } from "../middleware/logger.js";
// this constant and the plain provider identifiers only.
const SANDBOX_ORPHAN_CLEANUP_WRITE_ERROR_KIND = "sandbox_orphan_cleanup_write_failed";
// The fixed non-secret refusal a duplex channel open returns when the lease does
// not grant the opt-in `duplexCommandStream` capability. The service resolves the
// exact lease capability snapshot and throws this before it reaches any driver.
const DUPLEX_CHANNEL_CAPABILITY_DENIED =
"Sandbox lease does not grant the duplex command stream capability.";
// ---------------------------------------------------------------------------
// Sandbox capability contract — one normalizer for both branches
// ---------------------------------------------------------------------------
@@ -89,6 +103,8 @@ export const SANDBOX_CAPABILITY_KEYS = [
"persistentProcessSessions",
"independentControlCommands",
"incrementalSessionOutput",
"concurrentSyncOperations",
"duplexCommandStream",
] as const;
export type SandboxCapabilityKey = (typeof SANDBOX_CAPABILITY_KEYS)[number];
@@ -104,6 +120,8 @@ export type SandboxCapabilityKey = (typeof SANDBOX_CAPABILITY_KEYS)[number];
*/
const SANDBOX_CAPABILITY_OPT_IN_KEYS: ReadonlySet<SandboxCapabilityKey> = new Set([
"incrementalSessionOutput",
"concurrentSyncOperations",
"duplexCommandStream",
]);
/**
@@ -134,6 +152,17 @@ const SANDBOX_CAPABILITY_OPT_IN_KEYS: ReadonlySet<SandboxCapabilityKey> = new Se
* provider tails the session log through it. The verified verb is necessary
* but not sufficient: this key is opt-in, so the declaration is the real gate
* (see {@link SANDBOX_CAPABILITY_OPT_IN_KEYS}).
* - `concurrentSyncOperations` requires BOTH sync verbs, because parallel
* bidirectional transfer runs an inbound and an outbound transfer at the same
* time. The two verbs are separate required groups, so a provider that
* verifies only one direction cannot get the capability. The verbs are
* necessary but not sufficient: this key is opt-in, so the declaration is the
* real gate (see {@link SANDBOX_CAPABILITY_OPT_IN_KEYS}).
* - `duplexCommandStream` requires `duplexChannelOpen`, the worker verb that
* opens the persistent duplex channel. The verified verb is necessary but not
* sufficient: this key is opt-in, so the declaration is the real gate. A
* provider that does not implement the duplex open verb resolves `false` and
* keeps the file bridge.
*/
const SANDBOX_CAPABILITY_PREREQUISITE_METHODS: Record<SandboxCapabilityKey, readonly (readonly string[])[]> = {
// Reusable leases require ALL reuse verbs. Each verb is its own required
@@ -147,6 +176,8 @@ const SANDBOX_CAPABILITY_PREREQUISITE_METHODS: Record<SandboxCapabilityKey, read
persistentProcessSessions: [["environmentExecute"]],
independentControlCommands: [["environmentExecute"]],
incrementalSessionOutput: [["environmentExecute"]],
concurrentSyncOperations: [["environmentSyncIn"], ["environmentSyncOut"]],
duplexCommandStream: [["duplexChannelOpen"]],
};
function capabilityIsVerified(
@@ -230,6 +261,8 @@ export function resolveEffectiveSandboxCapabilities(input: {
persistentProcessSessions: resolve("persistentProcessSessions"),
independentControlCommands: resolve("independentControlCommands"),
incrementalSessionOutput: resolve("incrementalSessionOutput"),
concurrentSyncOperations: resolve("concurrentSyncOperations"),
duplexCommandStream: resolve("duplexCommandStream"),
};
}
@@ -244,9 +277,9 @@ export function resolveEffectiveSandboxCapabilities(input: {
* which falls back for a `job` backend or a `nativeFileSyncUnsupported` lease).
* - `configResolutionFailed` marks that the runtime could not resolve the
* provider config. A provider whose config cannot be resolved is untrusted, so
* the runtime fails closed and narrows `persistentProcessSessions` and
* `incrementalSessionOutput` to false. An empty config alone does not fail
* closed; only a resolution error does.
* the runtime fails closed and narrows `persistentProcessSessions`,
* `incrementalSessionOutput`, and `duplexCommandStream` to false. An empty
* config alone does not fail closed; only a resolution error does.
*/
export function buildSandboxCapabilityNarrowing(input: {
leasePolicy?: EnvironmentLease["leasePolicy"] | null;
@@ -265,11 +298,15 @@ export function buildSandboxCapabilityNarrowing(input: {
if (input.configResolutionFailed === true) {
// The runtime could not resolve the provider config, so it fails closed and
// denies persistent process sessions and incremental session output. The
// session-output streaming gate reads `incrementalSessionOutput`, so it must
// narrow with the persistent-session gate to keep the fail-closed behavior.
// denies persistent process sessions, incremental session output, and the
// duplex command stream. The session-output streaming gate reads
// `incrementalSessionOutput`, so it must narrow with the persistent-session
// gate to keep the fail-closed behavior. The duplex command stream opens a
// host-owned bidirectional channel, so an untrusted provider must not keep
// it either.
narrowing.persistentProcessSessions = false;
narrowing.incrementalSessionOutput = false;
narrowing.duplexCommandStream = false;
}
return narrowing;
@@ -460,6 +497,11 @@ export interface EnvironmentDriverSyncInput extends EnvironmentDriverLeaseInput
operations: PluginSyncOperation[];
}
export interface EnvironmentDriverOpenDuplexChannelInput extends EnvironmentDriverLeaseInput {
/** The command line the sandbox runs as the duplex channel child process. */
command: string;
}
export interface EnvironmentRuntimeDriver {
readonly driver: string;
acquireRunLease(input: EnvironmentDriverAcquireInput): Promise<EnvironmentLease>;
@@ -476,6 +518,15 @@ export interface EnvironmentRuntimeDriver {
*/
syncIn?(input: EnvironmentDriverSyncInput): Promise<PluginEnvironmentSyncResult>;
syncOut?(input: EnvironmentDriverSyncInput): Promise<PluginEnvironmentSyncResult>;
/**
* Optional persistent duplex channel. Present only for a plugin-backed sandbox
* driver whose lease grants the `duplexCommandStream` capability. The driver
* opens the host-owned duplex route on the plugin worker and adapts it to the
* cross-layer {@link CommandManagedDuplexChannel}. Other drivers omit it.
*/
openDuplexChannel?(
input: EnvironmentDriverOpenDuplexChannelInput,
): Promise<CommandManagedDuplexChannel>;
/** True when the lease's plugin worker advertises both sync verbs. */
supportsSync?(input: EnvironmentDriverLeaseInput): boolean;
/**
@@ -983,6 +1034,39 @@ function createSshEnvironmentDriver(db: Db): EnvironmentRuntimeDriver {
};
}
/**
* Adapt the worker manager's duplex host session to the cross-layer channel. The
* two shapes differ in the exit and stop members: the host session resolves the
* exit one time from `wait()`, so the channel bridges it to a one-time `onExit`
* listener; `kill()` maps to `stop()`. The `write`, `onData`, and `close`
* members map one to one.
*/
function adaptDuplexChannelHostSession(
session: DuplexChannelHostSession,
): CommandManagedDuplexChannel {
return {
write(data: string): void {
session.write(data);
},
onData(listener: (chunk: string) => void): void {
session.onData(listener);
},
onExit(listener: (exit: { exitCode: number | null }) => void): void {
// `wait()` resolves one time with the exit and never rejects, so a single
// `then` bridges it to the one-time exit listener.
void session.wait().then((exit) => {
listener(exit);
});
},
stop(): void {
session.kill();
},
close(): Promise<void> {
return session.close();
},
};
}
function createSandboxEnvironmentDriver(
db: Db,
options: {
@@ -2359,6 +2443,37 @@ function createSandboxEnvironmentDriver(
return await callPluginEnvironmentSync("environmentSyncOut", input);
},
async openDuplexChannel(input) {
// Plugin-backed sandbox providers only: open the host-owned duplex route on
// the plugin worker. The lease scope mirrors the sandbox execute path — the
// provider driver key, the company, the environment, and the provider lease
// id — so the route binds to the same worker session the runner streams.
if (!input.lease.metadata?.sandboxProviderPlugin || !pluginWorkerManager) {
throw new Error("Sandbox driver does not support duplex channels for this lease.");
}
const pluginId = readString(input.lease.metadata?.pluginId);
const providerKey = readString(input.lease.metadata?.provider);
const providerLeaseId = readString(input.lease.providerLeaseId);
if (!pluginId || !providerKey || !providerLeaseId) {
throw new Error(
"Sandbox duplex channel needs a plugin id, a provider key, and a provider lease id on the lease.",
);
}
const worker = pluginWorkerManager.getWorker(pluginId);
if (!worker) {
throw new Error(`Plugin worker "${pluginId}" is not running for the duplex channel.`);
}
const managerInput: WorkerManagerDuplexChannelOpenInput = {
driverKey: providerKey,
companyId: input.lease.companyId,
environmentId: input.environment.id,
providerLeaseId,
command: input.command,
};
const session = await worker.openDuplexChannel(managerInput);
return adaptDuplexChannelHostSession(session);
},
async effectiveSandboxCapabilities(input) {
const metadata = input.lease.metadata ?? {};
const providerKey =
@@ -3310,6 +3425,27 @@ export function environmentRuntimeService(
}
return await driver.syncOut(input);
},
async openDuplexChannel(
input: EnvironmentDriverOpenDuplexChannelInput,
): Promise<CommandManagedDuplexChannel> {
const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment));
if (!driver.openDuplexChannel) {
throw new Error(`Environment driver "${driver.driver}" does not support duplex channels.`);
}
// Centralize the duplex channel authorization here. Resolve the exact lease
// capability snapshot and refuse unless the effective snapshot grants the
// opt-in `duplexCommandStream` capability. This gate runs before the driver
// call, so an unauthorized lease never reaches the worker. The
// execution-target member gate stays as defense in depth. A driver that
// cannot resolve the snapshot fails closed with the fixed refusal.
const effective =
(await driver.effectiveSandboxCapabilities?.(input)) ?? null;
if (effective?.duplexCommandStream !== true) {
throw new Error(DUPLEX_CHANNEL_CAPABILITY_DENIED);
}
return await driver.openDuplexChannel(input);
},
};
}
@@ -67,6 +67,24 @@ type RuntimeServiceReadDb = Pick<Db, "select">;
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
const execFileAsync = promisify(execFile);
const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]);
// Return the timestamp when an issue became terminal. A `done` issue uses
// `completedAt`. A `cancelled` issue uses `cancelledAt`. The reaper cooldown
// measures the age of the terminal transition from this timestamp. Fall back to
// `updatedAt` when the terminal timestamp is null, so an old issue that lacks a
// recorded transition time still gates the cooldown. Return null for a
// non-terminal issue.
function issueTerminalTimestamp(issue: {
status: string;
completedAt: Date | null;
cancelledAt: Date | null;
updatedAt: Date;
}): Date | null {
if (issue.status === "done") return issue.completedAt ?? issue.updatedAt;
if (issue.status === "cancelled") return issue.cancelledAt ?? issue.updatedAt;
return null;
}
const WORKSPACE_BRANCH_INCOHERENCE_REASON = "git_worktree_branch_incoherence";
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
export const ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON = "issue_terminal";
@@ -208,6 +226,10 @@ export type ExecutionWorkspaceServiceOptions = {
resolvePullRequestDetails?: PullRequestMergeDetailsResolver;
now?: () => Date;
beforeTerminalWorkspaceCleanup?: (workspace: ExecutionWorkspaceRow) => Promise<void>;
// The terminal-workspace reaper waits this many days after an issue tree
// becomes terminal before it archives the workspace. A value of 0 disables
// the cooldown. The default is 7 days.
workspaceReaperCooldownDays?: number;
};
function parseGitHubRepository(repoUrl: string | null) {
@@ -1230,6 +1252,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
const recoveryActionsSvc = issueRecoveryActionService(db);
const resolvePullRequestDetails = opts.resolvePullRequestDetails ?? createPullRequestMergeDetailsResolver(db);
const now = opts.now ?? (() => new Date());
// The reaper waits this long after an issue tree becomes terminal before it
// archives the workspace. A value of 0 disables the cooldown, so the reaper
// archives a terminal workspace on the same sweep. A negative value also
// disables the cooldown.
const workspaceReaperCooldownMs = Math.max(
0,
(opts.workspaceReaperCooldownDays ?? 7) * 24 * 60 * 60 * 1000,
);
const pullRequestStateCache = new Map<
string,
{
@@ -1268,6 +1298,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
.select({
id: issues.id,
status: issues.status,
completedAt: issues.completedAt,
cancelledAt: issues.cancelledAt,
updatedAt: issues.updatedAt,
})
.from(issues)
.where(and(
@@ -1323,6 +1356,17 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
const sourceIssue = issueTree.find((issue) => issue.id === workspace.sourceIssueId) ?? null;
const sourceIssueTerminal = Boolean(sourceIssue && TERMINAL_ISSUE_STATUSES.has(sourceIssue.status));
const subtreeTerminal = Boolean(sourceIssue && issueTree.every((issue) => TERMINAL_ISSUE_STATUSES.has(issue.status)));
// The cooldown anchor is the most recent terminal timestamp across the whole
// issue tree. The reaper compares it against the cooldown window. A null
// anchor means no issue in the tree is terminal yet, so the cooldown never
// applies (the terminal-tree gates above already block the archive).
let cooldownAnchor: Date | null = null;
for (const issue of issueTree) {
const terminalAt = issueTerminalTimestamp(issue);
if (terminalAt && (!cooldownAnchor || terminalAt.getTime() > cooldownAnchor.getTime())) {
cooldownAnchor = terminalAt;
}
}
let mergedPullRequest = false;
let pullRequestStateUnknown = false;
const workspaceHeadSha = git?.repoRoot && git.workspacePath
@@ -1386,6 +1430,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}),
sourceIssueTerminal,
subtreeTerminal,
cooldownAnchor,
workspaceDirty: Boolean(git?.hasDirtyTrackedFiles || git?.hasUntrackedFiles),
workspaceHeadSha,
};
@@ -2173,6 +2218,13 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
.where(eq(executionWorkspaces.id, id))
.then((rows) => rows[0] ?? null);
if (!row) return null;
const { refreshPersistedRuntimeServiceHealth } = await import("./workspace-runtime.js");
await refreshPersistedRuntimeServiceHealth({
db,
companyId: row.companyId,
executionWorkspaceId: row.id,
projectWorkspaceId: row.projectWorkspaceId,
});
const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, row.companyId, [row]);
return hydrateWorkspace(
row,
@@ -2460,6 +2512,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
skippedUndelivered: 0,
skippedRace: 0,
skippedReopened: 0,
skippedCooldown: 0,
clearedStaleReopenPending: 0,
};
}
@@ -2523,6 +2576,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
skippedUndelivered: 0,
skippedRace: 0,
skippedReopened: 0,
skippedCooldown: 0,
clearedStaleReopenPending: 0,
};
@@ -2564,6 +2618,23 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
result.skippedUndelivered += 1;
continue;
}
// Hold the archive during the cooldown window. The anchor is the most
// recent terminal timestamp across the issue tree. A person can reopen
// the work inside this window. A cooldown of 0 disables the check, so the
// reaper archives the workspace on the same sweep. The archive statement
// below re-checks the same cutoff under the lifecycle lock, so the loop
// check and the guarded statement agree.
const cooldownCutoff = workspaceReaperCooldownMs > 0
? new Date(now().getTime() - workspaceReaperCooldownMs)
: null;
if (
cooldownCutoff
&& assessment.cooldownAnchor
&& assessment.cooldownAnchor.getTime() > cooldownCutoff.getTime()
) {
result.skippedCooldown += 1;
continue;
}
if (reopenPending) {
const pendingSince = readMetadataReopenPendingConsumptionSince(
workspace.metadata as Record<string, unknown> | null,
@@ -2689,6 +2760,34 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
)
SELECT 1 FROM issue_tree WHERE status NOT IN ('done', 'cancelled')
)`,
// Re-check the cooldown under the lifecycle lock. This predicate
// matches the loop check above: block the archive when any issue in
// the tree became terminal after the cutoff. The tree walk mirrors
// the terminal-tree walk above. A null cutoff means the cooldown is
// disabled, so this predicate drops out of the guard.
cooldownCutoff
? sql<boolean>`NOT EXISTS (
WITH RECURSIVE cooldown_tree(id, status, completed_at, cancelled_at, updated_at) AS (
SELECT root.id, root.status, root.completed_at, root.cancelled_at, root.updated_at
FROM ${issues} root
WHERE root.company_id = ${workspace.companyId}
AND root.id = ${workspace.sourceIssueId}
UNION ALL
SELECT child.id, child.status, child.completed_at, child.cancelled_at, child.updated_at
FROM ${issues} child
JOIN cooldown_tree parent ON child.parent_id = parent.id
WHERE child.company_id = ${workspace.companyId}
)
SELECT 1 FROM cooldown_tree
WHERE COALESCE(
CASE
WHEN status = 'done' THEN completed_at
WHEN status = 'cancelled' THEN cancelled_at
END,
updated_at
) > ${cooldownCutoff.toISOString()}::timestamptz
)`
: undefined,
))
.returning()
.then((rows) => rows[0] ?? null);
+12
View File
@@ -2891,6 +2891,12 @@ export type ReferencedProjectFailureReason = "authorization" | "resolution" | "s
export interface ReferencedProjectFailure {
projectId: string;
reason: ReferencedProjectFailureReason;
/**
* The failure message, when the layer that dropped the project produced one. A `staging` failure
* carries the remote extract or sync error here, so a reader of the run log learns why the project
* dropped. An `authorization` or `resolution` drop omits this field.
*/
error?: string;
}
export interface ResolvedRunReferencedProjects {
@@ -3235,6 +3241,8 @@ export interface ReferencedProjectRunObservability {
referenced_project_failures: Array<{
project_id: string;
reason: ReferencedProjectFailureReason;
/** The failure message for a `staging` drop; absent for an `authorization` or `resolution` drop. */
error?: string;
}>;
}
@@ -3257,6 +3265,9 @@ export function buildReferencedProjectRunObservability(input: {
referenced_project_failures: input.failures.map((failure) => ({
project_id: failure.projectId,
reason: failure.reason,
// Carry the error only when the layer produced one, so an authorization or resolution drop
// stays a two-field entry and a staging drop names its reason.
...(failure.error !== undefined ? { error: failure.error } : {}),
})),
};
}
@@ -16301,6 +16312,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
failures: referencedProjectStagingFailures.map((failure) => ({
projectId: failure.projectId,
reason: "staging" as const,
error: failure.error,
})),
});
logger.info(
+93 -26
View File
@@ -1,9 +1,14 @@
import { and, eq, inArray } from "drizzle-orm";
import { and, eq, inArray, or } from "drizzle-orm";
import { createHash } from "node:crypto";
import type { Db } from "@paperclipai/db";
import { agentWakeupRequests } from "@paperclipai/db";
export const ISSUE_BLOCKERS_RESOLVED_WAKE_REASON = "issue_blockers_resolved";
// A wake counts as "already delivered or in flight for the current ready state"
// for these statuses. The level-triggered state key uses this full set so that
// one wake for a ready state suppresses further wakes for the SAME state. This
// bounds reconciliation: after one wake, later passes find the completed row.
const IDEMPOTENT_DEPENDENCY_WAKE_STATUSES = [
"queued",
"deferred_issue_execution",
@@ -11,6 +16,23 @@ const IDEMPOTENT_DEPENDENCY_WAKE_STATUSES = [
"completed",
] as const;
// A wake counts as "still in flight" for these statuses. The `completed` status
// is not in this set on purpose. Dependency readiness is level-triggered, so a
// historical completed per-edge wake must never suppress a new wake for the
// current ready state. The dedup uses this set only for the legacy per-edge key,
// to avoid a duplicate while an old-format wake is still queued or claimed.
const IN_FLIGHT_DEPENDENCY_WAKE_STATUSES = [
"queued",
"deferred_issue_execution",
"claimed",
] as const;
/**
* Legacy per-edge idempotency key. One key encodes a single resolved blocker
* edge `issue_blockers_resolved:{dependentIssueId}:{resolvedBlockerIssueId}`.
* The dedup keeps this format only to read wake rows written before the
* level-triggered state key existed.
*/
export function buildIssueBlockersResolvedWakeIdempotencyKey(input: {
dependentIssueId: string;
resolvedBlockerIssueId: string;
@@ -22,36 +44,82 @@ export function buildIssueBlockersResolvedWakeIdempotencyKey(input: {
].join(":");
}
export async function findExistingIssueBlockersResolvedWake(
db: Db,
input: {
companyId: string;
idempotencyKey: string;
},
) {
return db
.select({ id: agentWakeupRequests.id, status: agentWakeupRequests.status })
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, input.companyId),
eq(agentWakeupRequests.idempotencyKey, input.idempotencyKey),
inArray(agentWakeupRequests.status, [...IDEMPOTENT_DEPENDENCY_WAKE_STATUSES]),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
/**
* Level-triggered idempotency key. One key encodes the full set of blockers that
* defines the current dependency-ready state. Two wakes for the same ready state
* share the key. A wake for an earlier partial state has a different blocker set,
* so it produces a different key and never suppresses the current wake. All three
* emit paths (route-time, finalize-time, periodic backstop) use this key so they
* share one idempotency rule.
*/
export function buildIssueBlockersResolvedWakeStateKey(input: {
dependentIssueId: string;
blockerIssueIds: string[];
}) {
const sortedBlockerIssueIds = [...new Set(input.blockerIssueIds.filter(Boolean))].sort();
const digest = createHash("sha256")
.update(sortedBlockerIssueIds.join(","))
.digest("hex")
.slice(0, 32);
return [
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
"state",
input.dependentIssueId,
String(sortedBlockerIssueIds.length),
digest,
].join(":");
}
export async function findExistingIssueBlockersResolvedWakeForAnyKey(
/**
* Find a wake that already covers the current dependency-ready state of the
* dependent issue. The check is level-triggered:
*
* - The state key matches a wake in any idempotent status (including
* `completed`). This suppresses a duplicate wake for the SAME ready state and
* bounds reconciliation.
* - Each legacy per-edge key matches only a wake that is still in flight
* (`queued`, `deferred_issue_execution`, `claimed`). This prevents a duplicate
* wake while an old-format wake is still pending after a deploy, but it never
* lets a historical completed per-edge wake strand the issue.
*
* Returns the first matching wake or `null`.
*/
export async function findExistingIssueBlockersResolvedWakeForReadyState(
db: Db,
input: {
companyId: string;
idempotencyKeys: string[];
dependentIssueId: string;
blockerIssueIds: string[];
},
) {
const idempotencyKeys = [...new Set(input.idempotencyKeys.filter(Boolean))];
if (idempotencyKeys.length === 0) return null;
const stateKey = buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: input.dependentIssueId,
blockerIssueIds: input.blockerIssueIds,
});
const legacyKeys = [
...new Set(
input.blockerIssueIds
.filter(Boolean)
.map((resolvedBlockerIssueId) =>
buildIssueBlockersResolvedWakeIdempotencyKey({
dependentIssueId: input.dependentIssueId,
resolvedBlockerIssueId,
}),
),
),
];
const stateMatch = and(
eq(agentWakeupRequests.idempotencyKey, stateKey),
inArray(agentWakeupRequests.status, [...IDEMPOTENT_DEPENDENCY_WAKE_STATUSES]),
);
const legacyMatch =
legacyKeys.length > 0
? and(
inArray(agentWakeupRequests.idempotencyKey, legacyKeys),
inArray(agentWakeupRequests.status, [...IN_FLIGHT_DEPENDENCY_WAKE_STATUSES]),
)
: null;
return db
.select({
@@ -63,8 +131,7 @@ export async function findExistingIssueBlockersResolvedWakeForAnyKey(
.where(
and(
eq(agentWakeupRequests.companyId, input.companyId),
inArray(agentWakeupRequests.idempotencyKey, idempotencyKeys),
inArray(agentWakeupRequests.status, [...IDEMPOTENT_DEPENDENCY_WAKE_STATUSES]),
legacyMatch ? or(stateMatch, legacyMatch) : stateMatch,
),
)
.limit(1)
+149 -23
View File
@@ -1,5 +1,6 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import fsSync from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
@@ -63,6 +64,32 @@ function getRuntimeServicesDir() {
return path.resolve(resolvePaperclipInstanceRoot(), "runtime-services");
}
function getRuntimeServiceLogsDir() {
return path.resolve(resolvePaperclipInstanceRoot(), "runtime-service-logs");
}
export function resolveLocalServiceLogPath(serviceKey: string) {
if (!/^[a-z0-9._-]+$/.test(serviceKey)) {
throw new Error("Invalid local service key for log path");
}
return path.resolve(getRuntimeServiceLogsDir(), `${serviceKey}.log`);
}
/**
* Open a managed service's durable append-only output file.
*
* The returned descriptor is intended to be passed directly to spawn(). The
* child receives its own duplicate, so the caller can close this handle as soon
* as spawn returns without tying the service's stdio lifetime to Paperclip's.
*/
export async function openLocalServiceLogFile(serviceKey: string) {
await fs.mkdir(getRuntimeServiceLogsDir(), { recursive: true });
const logPath = resolveLocalServiceLogPath(serviceKey);
const handle = await fs.open(logPath, "a+", 0o600);
const startOffset = (await handle.stat()).size;
return { handle, logPath, startOffset };
}
function getRuntimeServiceRegistryPath(serviceKey: string) {
return path.resolve(getRuntimeServicesDir(), `${serviceKey}.json`);
}
@@ -230,10 +257,91 @@ export function isProcessGroupAlive(processGroupId: number | null | undefined) {
if (typeof processGroupId !== "number" || !Number.isInteger(processGroupId) || processGroupId <= 0) return false;
try {
process.kill(-processGroupId, 0);
return true;
} catch {
return false;
}
if (process.platform === "linux") {
const liveMember = readLinuxProcessGroupActivity(processGroupId);
if (liveMember !== null) return liveMember;
}
return true;
}
function readLinuxProcessGroupActivity(processGroupId: number): boolean | null {
let entries: fsSync.Dirent[];
try {
entries = fsSync.readdirSync("/proc", { withFileTypes: true });
} catch {
return null;
}
let foundMember = false;
for (const entry of entries) {
if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue;
try {
const stat = fsSync.readFileSync(`/proc/${entry.name}/stat`, "utf8");
const commandEnd = stat.lastIndexOf(")");
if (commandEnd < 0) continue;
const fields = stat.slice(commandEnd + 1).trim().split(/\s+/);
const state = fields[0];
const memberProcessGroupId = Number.parseInt(fields[2] ?? "", 10);
if (memberProcessGroupId !== processGroupId) continue;
foundMember = true;
if (state !== "Z" && state !== "X") return true;
} catch {
// The process can exit while /proc is scanned.
}
}
// kill(-pgid, 0) also succeeds for a group that contains only zombies. Such
// processes cannot run or own a listener and are waiting only for their
// parent to reap them, so termination is complete for service-control use.
return foundMember ? false : null;
}
function tokenizeCommandLine(value: string) {
return value.match(/"(?:\\.|[^"\\])*"|'[^']*'|\S+/g) ?? [];
}
function normalizeCommandToken(value: string) {
const unquoted = value.replace(/^["']|["']$/g, "");
const basename = path.basename(unquoted.replace(/\\/g, "/"));
const launcher = basename.replace(/\.(?:cjs|mjs|js|cmd|exe)$/i, "");
return /^(?:bun|node|nodejs|npm|npx|pnpm|yarn)$/i.test(launcher) ? launcher : unquoted;
}
/**
* Compare a configured service command with the argv exposed by the OS.
*
* Package-manager launchers commonly replace `pnpm dev` with
* `node /path/to/pnpm.cjs dev` after the shell starts. A literal substring
* check rejects that surviving process even though the executable and all
* configured arguments are still present. Normalize executable paths and
* script extensions, then require the configured argv to remain contiguous.
*/
export function doesLocalServiceCommandLineMatch(input: {
commandLine: string;
recordedCommand: string;
serviceName: string;
}) {
const normalize = (value: string) => value.replace(/["']/g, "").replace(/\s+/g, " ").trim();
const normalizedCommandLine = normalize(input.commandLine);
const normalizedRecordedCommand = normalize(input.recordedCommand);
if (
normalizedCommandLine.includes(normalizedRecordedCommand)
|| normalizedCommandLine.includes(input.serviceName)
) {
return true;
}
const actualTokens = tokenizeCommandLine(input.commandLine).map(normalizeCommandToken);
const recordedTokens = tokenizeCommandLine(input.recordedCommand).map(normalizeCommandToken);
if (recordedTokens.length === 0 || recordedTokens.length > actualTokens.length) return false;
return actualTokens.some((_, start) => recordedTokens.every(
(token, offset) => actualTokens[start + offset] === token,
));
}
async function isLikelyMatchingCommand(record: LocalServiceRegistryRecord) {
@@ -242,10 +350,11 @@ async function isLikelyMatchingCommand(record: LocalServiceRegistryRecord) {
const { stdout } = await execFileAsync("ps", ["-o", "command=", "-p", String(record.pid)]);
const commandLine = stdout.trim();
if (!commandLine) return false;
const normalize = (value: string) => value.replace(/["']/g, "").replace(/\s+/g, " ").trim();
const normalizedCommandLine = normalize(commandLine);
const normalizedRecordedCommand = normalize(record.command);
return normalizedCommandLine.includes(normalizedRecordedCommand) || normalizedCommandLine.includes(record.serviceName);
return doesLocalServiceCommandLineMatch({
commandLine,
recordedCommand: record.command,
serviceName: record.serviceName,
});
} catch {
return true;
}
@@ -396,11 +505,34 @@ export async function touchLocalServiceRegistryRecord(
}
export async function terminateLocalService(
record: Pick<LocalServiceRegistryRecord, "pid" | "processGroupId">,
opts?: { signal?: NodeJS.Signals; forceAfterMs?: number },
record: Pick<LocalServiceRegistryRecord, "pid" | "processGroupId"> &
Partial<Pick<LocalServiceRegistryRecord, "port">>,
opts?: { signal?: NodeJS.Signals; forceAfterMs?: number; verifyAfterMs?: number },
) {
const signal = opts?.signal ?? "SIGTERM";
const targetProcessGroup = process.platform !== "win32" && record.processGroupId && record.processGroupId > 0;
const targetIsGone = async () => {
const targetAlive = targetProcessGroup
? isProcessGroupAlive(record.processGroupId)
: isPidAlive(record.pid);
if (targetAlive) return false;
if (!record.port) return true;
const portOwnerPid = await readLocalServicePortOwner(record.port);
if (!portOwnerPid) return true;
const ownerProcessId = targetProcessGroup ? record.processGroupId! : record.pid;
return !(await isLocalServiceProcessOwnedBy(portOwnerPid, ownerProcessId));
};
const waitUntilGone = async (timeoutMs: number) => {
const deadline = Date.now() + timeoutMs;
do {
if (await targetIsGone()) return true;
await delay(100);
} while (Date.now() < deadline);
return await targetIsGone();
};
try {
if (targetProcessGroup) {
process.kill(-record.processGroupId!, signal);
@@ -408,24 +540,10 @@ export async function terminateLocalService(
process.kill(record.pid, signal);
}
} catch {
return;
if (await targetIsGone()) return;
}
const deadline = Date.now() + (opts?.forceAfterMs ?? 2_000);
while (Date.now() < deadline) {
const targetAlive = targetProcessGroup
? isProcessGroupAlive(record.processGroupId)
: isPidAlive(record.pid);
if (!targetAlive) {
return;
}
await delay(100);
}
const stillAlive = targetProcessGroup
? isProcessGroupAlive(record.processGroupId)
: isPidAlive(record.pid);
if (!stillAlive) return;
if (await waitUntilGone(opts?.forceAfterMs ?? 2_000)) return;
try {
if (targetProcessGroup) {
process.kill(-record.processGroupId!, "SIGKILL");
@@ -435,6 +553,14 @@ export async function terminateLocalService(
} catch {
// Ignore cleanup races.
}
if (await waitUntilGone(opts?.verifyAfterMs ?? 2_000)) return;
const target = targetProcessGroup
? `process group ${record.processGroupId}`
: `process ${record.pid}`;
const listener = record.port ? ` and listener on port ${record.port}` : "";
throw new Error(`Failed to terminate local service ${target}${listener}`);
}
export async function readLocalServicePortOwner(port: number) {
@@ -0,0 +1,386 @@
/**
* Control-plane half of the workspace readiness/handoff contract (PAP-17572).
*
* Resolves the identity and derived key material for one managed workspace,
* injects them into the guest process, and probes the guest's *protected*
* readiness so `running / healthy` can never be published for a workspace whose
* clone, identity, or login handoff disagrees with what was expected.
*
* Identity always comes from the worktree's own recorded instance pointer, never
* from a branch name or basename heuristic, and an unresolvable identity fails
* closed (no key injected, no readiness claimed).
*/
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { authUsers, companyMemberships, type Db } from "@paperclipai/db";
import type { WorkspaceReadiness, WorkspaceReadinessProbeResult } from "@paperclipai/shared";
import {
deriveWorkspaceHandoffKey,
deriveWorkspaceReadinessToken,
resolveWorkspaceHandoffRootSecret,
WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY,
WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY,
WORKSPACE_HANDOFF_KEY_ENV_KEY,
WORKSPACE_READINESS_TOKEN_ENV_KEY,
WORKSPACE_READINESS_TOKEN_HEADER,
WORKSPACE_READINESS_USER_EMAIL_HEADER,
WORKSPACE_READINESS_USER_ID_HEADER,
} from "../auth/workspace-login-handoff.js";
import { logger } from "../middleware/logger.js";
import {
deriveWorktreeInstanceId,
readWorktreeInstanceId,
} from "./workspace-instance-cleanup.js";
/**
* Per-probe budget. Matched to the semantic transport probe it replaces, so
* upgrading a runtime health check to the readiness contract cannot make a
* managed start or a reuse decision slower than it was.
*/
export const WORKSPACE_READINESS_PROBE_TIMEOUT_MS = 2_000;
export type ManagedWorkspaceIdentity = {
instanceId: string;
executionWorkspaceId: string;
companyId: string;
handoffKey: string;
readinessToken: string;
/** Whether the root secret was configured explicitly or derived. */
secretSource: "dedicated" | "derived";
};
/**
* Resolve the isolated instance id a worktree actually runs as.
*
* The worktree's `.paperclip/.env` pointer is authoritative because it is what
* the guest process itself loads. The path-derived id is only a fallback for a
* worktree provisioned before the pointer existed, and the two agree by
* construction for anything Paperclip provisioned.
*/
export function resolveManagedWorkspaceInstanceId(workspaceCwd: string): string | null {
const recorded = readWorktreeInstanceId(workspaceCwd);
if (recorded) return recorded;
const derived = deriveWorktreeInstanceId(workspaceCwd);
return derived || null;
}
export function resolveManagedWorkspaceIdentity(input: {
workspaceCwd: string | null | undefined;
executionWorkspaceId: string | null | undefined;
companyId: string | null | undefined;
env?: NodeJS.ProcessEnv;
}): ManagedWorkspaceIdentity | null {
const workspaceCwd = input.workspaceCwd?.trim();
const executionWorkspaceId = input.executionWorkspaceId?.trim();
const companyId = input.companyId?.trim();
if (!workspaceCwd || !executionWorkspaceId || !companyId) return null;
const root = resolveWorkspaceHandoffRootSecret(input.env ?? process.env);
if (!root) return null;
const instanceId = resolveManagedWorkspaceInstanceId(path.resolve(workspaceCwd));
if (!instanceId) return null;
return {
instanceId,
executionWorkspaceId,
companyId,
handoffKey: deriveWorkspaceHandoffKey({ rootSecret: root.secret, instanceId, executionWorkspaceId }),
readinessToken: deriveWorkspaceReadinessToken({ rootSecret: root.secret, instanceId, executionWorkspaceId }),
secretSource: root.source,
};
}
/**
* Env injected into a managed workspace's guest process.
*
* Only the *derived* per-workspace values cross the boundary: the guest never
* receives the root secret, so a compromised workspace cannot mint a ticket for
* a sibling workspace.
*/
export function buildManagedWorkspaceGuestEnv(identity: ManagedWorkspaceIdentity): Record<string, string> {
return {
[WORKSPACE_HANDOFF_KEY_ENV_KEY]: identity.handoffKey,
[WORKSPACE_READINESS_TOKEN_ENV_KEY]: identity.readinessToken,
[WORKSPACE_EXECUTION_WORKSPACE_ID_ENV_KEY]: identity.executionWorkspaceId,
[WORKSPACE_EXECUTION_WORKSPACE_COMPANY_ID_ENV_KEY]: identity.companyId,
};
}
function isWorkspaceReadinessShape(value: unknown): value is WorkspaceReadiness {
if (!value || typeof value !== "object") return false;
const candidate = value as Record<string, unknown>;
return (
typeof candidate.state === "string"
&& typeof candidate.databaseReady === "boolean"
&& typeof candidate.cloneDataReady === "boolean"
&& typeof candidate.authHandoffReady === "boolean"
&& (typeof candidate.authHandoffUserId === "string" || candidate.authHandoffUserId === null)
&& typeof candidate.seedState === "string"
&& (typeof candidate.companyId === "string" || candidate.companyId === null)
);
}
/**
* Read a managed workspace's protected readiness over loopback.
*
* Failure modes are distinguished so callers can tell "not reachable" from
* "reachable but serving somebody else's clone" — the second is what made a
* relocated port able to masquerade as healthy.
*/
export async function probeManagedWorkspaceReadiness(input: {
healthUrl: string;
identity: ManagedWorkspaceIdentity;
fetchImpl?: typeof fetch;
timeoutMs?: number;
handoffSubject?: { userId: string; email: string } | null;
}): Promise<WorkspaceReadinessProbeResult> {
const fetchImpl = input.fetchImpl ?? fetch;
let response: Response;
try {
response = await fetchImpl(input.healthUrl, {
headers: {
[WORKSPACE_READINESS_TOKEN_HEADER]: input.identity.readinessToken,
...(input.handoffSubject
? {
[WORKSPACE_READINESS_USER_ID_HEADER]: input.handoffSubject.userId,
[WORKSPACE_READINESS_USER_EMAIL_HEADER]: input.handoffSubject.email,
}
: {}),
},
signal: AbortSignal.timeout(input.timeoutMs ?? WORKSPACE_READINESS_PROBE_TIMEOUT_MS),
});
} catch (error) {
return {
ok: false,
reason: "unreachable",
readiness: null,
detail: error instanceof Error ? error.message : String(error),
};
}
const payload = await response.json().catch(() => null) as
| { status?: unknown; workspace?: unknown }
| null;
if (!response.ok) {
const readiness = isWorkspaceReadinessShape(payload?.workspace) ? payload.workspace : null;
return { ok: false, reason: "http_error", readiness, detail: `HTTP ${response.status}` };
}
if (payload?.status !== "ok") {
return { ok: false, reason: "unhealthy_payload", readiness: null, detail: String(payload?.status ?? "missing") };
}
if (
payload.workspace
&& typeof payload.workspace === "object"
&& !("companyId" in payload.workspace)
) {
// A guest that implements readiness but predates the company binding is not
// a legacy transport-only guest. Treat the partial contract as an identity
// disagreement so auto compatibility mode cannot publish an unopenable clone.
return {
ok: false,
reason: "identity_mismatch",
readiness: null,
detail: `expected company ${input.identity.companyId}, got missing`,
};
}
if (
input.handoffSubject
&& payload.workspace
&& typeof payload.workspace === "object"
&& !("authHandoffUserId" in payload.workspace)
) {
return {
ok: false,
reason: "identity_mismatch",
readiness: null,
detail: `expected handoff user ${input.handoffSubject.userId}, got missing`,
};
}
if (!isWorkspaceReadinessShape(payload.workspace)) {
// Either the token was rejected or the guest predates this contract. Both
// mean the control plane cannot prove user readiness, so neither may publish.
return { ok: false, reason: "readiness_missing", readiness: null, detail: null };
}
const readiness = payload.workspace;
if (
readiness.instanceId !== input.identity.instanceId
|| readiness.executionWorkspaceId !== input.identity.executionWorkspaceId
|| readiness.companyId !== input.identity.companyId
) {
return {
ok: false,
reason: "identity_mismatch",
readiness,
detail: `expected ${input.identity.instanceId}/${input.identity.executionWorkspaceId}/${input.identity.companyId}, got ${readiness.instanceId}/${readiness.executionWorkspaceId}/${readiness.companyId}`,
};
}
if (!readiness.databaseReady || !readiness.cloneDataReady || !readiness.authHandoffReady) {
return { ok: false, reason: "not_ready", readiness, detail: readiness.failurePhase };
}
if (input.handoffSubject && readiness.authHandoffUserId !== input.handoffSubject.userId) {
return {
ok: false,
reason: "identity_mismatch",
readiness,
detail: `expected handoff user ${input.handoffSubject.userId}, got ${readiness.authHandoffUserId}`,
};
}
return { ok: true, readiness };
}
/** Current board identities that must survive the clone for it to publish ready. */
export async function listManagedWorkspaceHandoffSubjects(
db: Db,
companyId: string,
): Promise<Array<{ userId: string; email: string }>> {
const rows = await db
.select({ userId: authUsers.id, email: authUsers.email })
.from(authUsers)
.innerJoin(
companyMemberships,
and(
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, authUsers.id),
eq(companyMemberships.companyId, companyId),
eq(companyMemberships.status, "active"),
),
);
return rows.flatMap((row) => {
const userId = row.userId?.trim();
const email = row.email?.trim();
return userId && email ? [{ userId, email }] : [];
});
}
/** Prove that every current board member can use the advertised handoff. */
export async function probeManagedWorkspaceHandoffSubjects(input: {
db: Db;
healthUrl: string;
identity: ManagedWorkspaceIdentity;
fetchImpl?: typeof fetch;
}): Promise<WorkspaceReadinessProbeResult> {
const subjects = await listManagedWorkspaceHandoffSubjects(input.db, input.identity.companyId);
if (subjects.length === 0) {
return {
ok: false,
reason: "not_ready",
readiness: null,
detail: "no active board user is eligible for workspace login handoff",
};
}
const results = await Promise.all(subjects.map((handoffSubject) => probeManagedWorkspaceReadiness({
healthUrl: input.healthUrl,
identity: input.identity,
fetchImpl: input.fetchImpl,
handoffSubject,
})));
return results.find((result) => !result.ok) ?? results[0]!;
}
/**
* How strictly a managed start is gated on the guest's protected readiness.
*
* - `auto` (default): a guest that *reports* readiness must satisfy it, and a
* guest serving a different instance/workspace is always rejected. A guest
* that reports no readiness block at all — an older checkout, which a managed
* worktree can legitimately be — falls back to the semantic transport check
* and is logged. Failing that case closed would turn an upgrade lag into an
* outage of every existing workspace, which is a worse reliability outcome
* than the defect this gate exists to fix. Such a workspace is still not
* silently "usable": it has no handoff plugin, so `Open workspace` reports the
* snapshot-local credential fallback rather than pretending to be ready.
* - `strict`: a missing readiness block also blocks publication. Intended for
* deployments that have finished rolling the new guest build everywhere.
*/
export type WorkspaceReadinessGateMode = "auto" | "strict";
export function resolveWorkspaceReadinessGateMode(
env: NodeJS.ProcessEnv = process.env,
): WorkspaceReadinessGateMode {
return env.PAPERCLIP_WORKSPACE_READINESS_GATE?.trim().toLowerCase() === "strict" ? "strict" : "auto";
}
/**
* Whether a rejected probe must stop the runtime from being published.
*
* Everything except "this guest does not implement the contract" is a real
* disagreement about the clone and always blocks.
*/
export function shouldBlockPublicationOnReadiness(
result: Extract<WorkspaceReadinessProbeResult, { ok: false }>,
mode: WorkspaceReadinessGateMode = resolveWorkspaceReadinessGateMode(),
): boolean {
if (result.reason === "readiness_missing") return mode === "strict";
return true;
}
/**
* How long a freshly started workspace may take to satisfy readiness.
*
* Deliberately short. The transport readiness probe has already required a
* `status: ok` response from this guest by the time the gate runs, so the only
* thing left to absorb is a one-off blip — not a cold start. A longer budget
* would add latency to every managed start to no benefit.
*/
export const WORKSPACE_READINESS_GATE_TIMEOUT_MS = 3_000;
const WORKSPACE_READINESS_GATE_INTERVAL_MS = 250;
/**
* Poll readiness until it passes or the gate budget runs out.
*
* `identity_mismatch` and `readiness_missing` are terminal and return
* immediately: another instance owning the port, or a guest that does not
* implement the contract, are not conditions that polling can change.
*/
export async function waitForManagedWorkspaceReadiness(input: {
healthUrl: string;
identity: ManagedWorkspaceIdentity;
fetchImpl?: typeof fetch;
timeoutMs?: number;
now?: () => number;
sleep?: (ms: number) => Promise<void>;
}): Promise<WorkspaceReadinessProbeResult> {
const now = input.now ?? Date.now;
const sleep = input.sleep ?? ((ms: number) => new Promise<void>((resolve) => { setTimeout(resolve, ms); }));
const deadline = now() + (input.timeoutMs ?? WORKSPACE_READINESS_GATE_TIMEOUT_MS);
let last: WorkspaceReadinessProbeResult = {
ok: false,
reason: "unreachable",
readiness: null,
detail: "readiness gate did not run",
};
for (;;) {
last = await probeManagedWorkspaceReadiness({
healthUrl: input.healthUrl,
identity: input.identity,
fetchImpl: input.fetchImpl,
});
if (last.ok || last.reason === "identity_mismatch" || last.reason === "readiness_missing") return last;
if (now() >= deadline) return last;
await sleep(WORKSPACE_READINESS_GATE_INTERVAL_MS);
}
}
/** Log a rejected probe once, with the machine reason kept intact for operators. */
export function logManagedWorkspaceReadinessRejection(input: {
executionWorkspaceId: string;
healthUrl: string;
result: Extract<WorkspaceReadinessProbeResult, { ok: false }>;
}) {
logger.warn(
{
executionWorkspaceId: input.executionWorkspaceId,
healthUrl: input.healthUrl,
reason: input.result.reason,
detail: input.result.detail,
seedState: input.result.readiness?.seedState ?? null,
failurePhase: input.result.readiness?.failurePhase ?? null,
},
"managed workspace readiness probe rejected the runtime",
);
}
@@ -2,9 +2,22 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import type { NativeExecutionInputV1 } from "@paperclipai/paperclip-runner";
type BackendFactoryOptions = {
runnerInstanceId?: string;
onSpawn?: (meta: {
pid: number;
processGroupId: number | null;
startedAt: string;
}) => Promise<void>;
};
const state = vi.hoisted(() => ({
execute: vi.fn(),
createBackend: vi.fn(() => ({ kind: "test" })),
createBackend: vi.fn(
(_input: NativeExecutionInputV1, _options: BackendFactoryOptions) => ({
kind: "test",
}),
),
cancel: vi.fn(),
release: null as null | (() => void),
}));
@@ -78,7 +78,7 @@ export interface ReadyPluginEnvironmentDriver {
templateRefKind?: PluginEnvironmentDriverDeclaration["templateRefKind"];
templateConfigBinding?: PluginEnvironmentDriverDeclaration["templateConfigBinding"];
supportsTemplateDelete?: PluginEnvironmentDriverDeclaration["supportsTemplateDelete"];
supportsSetupTokenLogin?: PluginEnvironmentDriverDeclaration["supportsSetupTokenLogin"];
supportsLoginPty?: PluginEnvironmentDriverDeclaration["supportsLoginPty"];
}
export function pluginDriverProviderKey(config: Pick<PluginEnvironmentConfig, "pluginKey" | "driverKey">): string {
@@ -257,7 +257,7 @@ export async function listReadyPluginEnvironmentDrivers(input: {
templateRefKind: driver.templateRefKind,
templateConfigBinding: driver.templateConfigBinding,
supportsTemplateDelete: driver.supportsTemplateDelete,
supportsSetupTokenLogin: driver.supportsSetupTokenLogin,
supportsLoginPty: driver.supportsLoginPty,
})),
);
}
+11 -2
View File
@@ -545,6 +545,7 @@ const PROVIDER_SPAN_ATTR_ALLOWLIST: ReadonlySet<string> = new Set<string>([
SPAN_ATTRS.packWallMs,
SPAN_ATTRS.transferWallMs,
SPAN_ATTRS.transferGuardCount,
SPAN_ATTRS.transferDirection,
]);
/** The subset of allowed keys that carry a finite number. */
@@ -557,11 +558,15 @@ const PROVIDER_SPAN_NUMERIC_ATTRS: ReadonlySet<string> = new Set<string>([
/** The closed value set for the `outcome` attribute. */
const KNOWN_SPAN_OUTCOMES: ReadonlySet<string> = new Set(["ok", "skipped", "failed"]);
/** The closed value set for the `transfer.direction` attribute. */
const KNOWN_TRANSFER_DIRECTIONS: ReadonlySet<string> = new Set(["inbound", "outbound"]);
/**
* Re-clamp the worker-sent attributes at the trust boundary. Drop every key that
* is not on the allowlist. Re-map `provider` through `normalizeProviderFamily`,
* bound `outcome` to its closed set, and keep a numeric attribute only when it
* is a finite number. The result holds only bounded, low-cardinality values.
* bound `outcome` and `transfer.direction` each to its closed set, and keep a
* numeric attribute only when it is a finite number. The result holds only
* bounded, low-cardinality values.
*/
export function clampProviderSpanAttributes(
raw: Record<string, unknown> | undefined,
@@ -578,6 +583,10 @@ export function clampProviderSpanAttributes(
if (typeof value === "string" && KNOWN_SPAN_OUTCOMES.has(value)) clamped[key] = value;
continue;
}
if (key === SPAN_ATTRS.transferDirection) {
if (typeof value === "string" && KNOWN_TRANSFER_DIRECTIONS.has(value)) clamped[key] = value;
continue;
}
if (PROVIDER_SPAN_NUMERIC_ATTRS.has(key)) {
if (typeof value === "number" && Number.isFinite(value)) clamped[key] = value;
continue;
+564 -17
View File
@@ -39,6 +39,8 @@ import {
JsonRpcCallError,
SETUP_TOKEN_PTY_OUTPUT_NOTIFICATION,
SETUP_TOKEN_PTY_EXIT_NOTIFICATION,
DUPLEX_CHANNEL_DATA_NOTIFICATION,
DUPLEX_CHANNEL_EXIT_NOTIFICATION,
} from "@paperclipai/plugin-sdk";
import type {
JsonRpcId,
@@ -156,6 +158,60 @@ const SETUP_TOKEN_PTY_ROUTE_BUSY = "SETUP_TOKEN_PTY_ROUTE_BUSY";
/** The fixed non-secret error a failed open returns. */
const SETUP_TOKEN_PTY_OPEN_FAILED = "SETUP_TOKEN_PTY_OPEN_FAILED";
// Bounds and timeouts for the generic duplex channel route. The route mirrors the
// login pseudo-terminal route, but it carries no command allowlist and adds seven
// explicit bounds the pseudo-terminal route lacks. Each bound ends the route when
// it passes the limit, so a faulty or hostile worker cannot flood the host.
/** The default maximum characters for one duplex channel data notification. */
const MAX_DUPLEX_CHANNEL_CHUNK_CHARS = 1_000_000;
/**
* The default maximum cumulative characters the host buffers for one duplex
* channel route before a data listener attaches. A worker that streams data
* before the consumer binds cannot grow the host buffer without limit.
*/
const MAX_DUPLEX_CHANNEL_PRE_BIND_CHARS = 8 * 1024 * 1024;
/**
* The default maximum number of data frames the host buffers for one duplex
* channel route before a data listener attaches.
*/
const MAX_DUPLEX_CHANNEL_PRE_BIND_FRAMES = 10_000;
/**
* The default maximum number of in-flight host→worker requests for one duplex
* channel route. A worker that never replies cannot make the host hold an
* unbounded number of pending requests.
*/
const MAX_DUPLEX_CHANNEL_PENDING_REQUESTS = 256;
/** The default maximum characters for one host→worker duplex channel write. */
const MAX_DUPLEX_CHANNEL_WRITE_CHARS = 1_000_000;
/**
* The default maximum number of protocol errors for one duplex channel route.
* A protocol error is one malformed or mismatched data frame. The route ends
* when the count passes this budget, so a flood of bad frames bounds the route.
*/
const MAX_DUPLEX_CHANNEL_PROTOCOL_ERRORS = 100;
/**
* The default maximum cumulative bytes the host forwards for one duplex channel
* route over its whole life. The host counts the bytes of every inbound chunk,
* before and after a data listener attaches. The route ends when the count
* passes this cap, so an active route with a bound listener cannot stream an
* unbounded number of bytes.
*/
const MAX_DUPLEX_CHANNEL_TOTAL_DATA_BYTES = 256 * 1024 * 1024;
/**
* The default maximum lifetime for one duplex channel route, in milliseconds.
* The host starts a timer when the route opens and ends the route when the
* timer expires, so a route cannot live without limit.
*/
const MAX_DUPLEX_CHANNEL_DURATION_MS = 60 * 60 * 1000;
/** The default open timeout for one duplex channel route, in milliseconds. */
const DUPLEX_CHANNEL_OPEN_TIMEOUT_MS = 30_000;
/** The default close timeout for one duplex channel route, in milliseconds. */
const DUPLEX_CHANNEL_CLOSE_TIMEOUT_MS = 10_000;
/** The fixed non-secret error a rejected second duplex channel open returns. */
const DUPLEX_CHANNEL_ROUTE_BUSY = "DUPLEX_CHANNEL_ROUTE_BUSY";
/** The fixed non-secret error a failed duplex channel open returns. */
const DUPLEX_CHANNEL_OPEN_FAILED = "DUPLEX_CHANNEL_OPEN_FAILED";
/** Minimum time between two dropped-`execute.log` debug records. The router
* rate-limits the record so a flood of dropped chunks writes at most one line
* per window with a running count. */
@@ -322,6 +378,36 @@ export interface WorkerStartOptions {
/** The close timeout for one login pseudo-terminal route, in milliseconds. */
closeTimeoutMs?: number;
};
/**
* Bounds and timeouts for the generic duplex channel route. The defaults bound
* one data notification, the pre-bind buffer, the in-flight request count, one
* host→worker write, the protocol-error budget, and the open and close
* timeouts. A test overrides them to exercise each bound without huge inputs or
* long waits.
*/
duplexChannelLimits?: {
/** Max characters for one duplex channel data notification. */
maxChunkChars?: number;
/** Max cumulative characters the host buffers before a data listener attaches. */
maxPreBindBufferedChars?: number;
/** Max number of data frames the host buffers before a data listener attaches. */
maxPreBindBufferedFrames?: number;
/** Max number of in-flight host→worker requests for one route. */
maxPendingRequests?: number;
/** Max characters for one host→worker duplex channel write. */
maxWriteChars?: number;
/** Max number of protocol errors for one route before the route ends. */
maxProtocolErrors?: number;
/** Max cumulative bytes the host forwards for one route over its whole life. */
maxTotalDataBytes?: number;
/** The maximum lifetime for one route, in milliseconds. */
maxDurationMs?: number;
/** The open timeout for one duplex channel route, in milliseconds. */
openTimeoutMs?: number;
/** The close timeout for one duplex channel route, in milliseconds. */
closeTimeoutMs?: number;
};
}
/**
@@ -393,6 +479,38 @@ export interface SetupTokenPtyHostSession {
close(): Promise<void>;
}
/**
* The input the manager needs to open one generic duplex channel. The manager
* mints the host route identifier. The caller supplies the sandbox scope, the
* provider lease id, and the command. The duplex channel carries no command
* allowlist, so the caller owns the command.
*/
export interface DuplexChannelOpenInput {
driverKey: string;
companyId: string;
environmentId: string;
providerLeaseId: string;
command: string;
}
/**
* One live duplex channel the manager hands to a caller. The shape matches the
* login pseudo-terminal session, so a caller consumes one live bidirectional
* stream with the same methods.
*/
export interface DuplexChannelHostSession {
/** Registers the one data listener. The session streams each raw chunk in order. */
onData(listener: (chunk: string) => void): void;
/** Writes raw input bytes to the channel. */
write(data: string): void;
/** Resolves with the child exit code when the command ends or the route ends. */
wait(): Promise<{ exitCode: number | null }>;
/** Stops the child process. Safe to call more than one time. */
kill(): void;
/** Closes the route and releases the channel. Safe to call more than one time. */
close(): Promise<void>;
}
/**
* Host-owned route for one active execute call. The host mints the invocation
* id and stores the exact company id and log sink here. A worker never selects
@@ -484,6 +602,17 @@ export interface PluginWorkerHandle {
input: SetupTokenPtyOpenInput,
): Promise<SetupTokenPtyHostSession>;
/**
* Open one generic duplex channel on this worker. The manager mints the host
* route identifier, reserves the route, drives the open, binds the worker
* session identifier one time, and returns a session a caller drives. It
* permits one active duplex channel per worker. It enforces five explicit
* bounds and ends the route when a bound passes its limit.
*/
openDuplexChannel(
input: DuplexChannelOpenInput,
): Promise<DuplexChannelHostSession>;
/**
* Authorize the set of companies this worker may act on from proactive
* (non-invocation) context. Replaces any previously-authorized set. See the
@@ -667,6 +796,34 @@ export function createPluginWorkerHandle(
const setupTokenPtyCloseTimeoutMs =
options.setupTokenPtyLimits?.closeTimeoutMs ?? SETUP_TOKEN_PTY_CLOSE_TIMEOUT_MS;
// Bounds and timeouts for the generic duplex channel route. A caller (a test)
// can lower them to exercise each bound and the terminalize paths.
const maxDuplexChannelChunkChars =
options.duplexChannelLimits?.maxChunkChars ?? MAX_DUPLEX_CHANNEL_CHUNK_CHARS;
const maxDuplexChannelPreBindChars =
options.duplexChannelLimits?.maxPreBindBufferedChars ??
MAX_DUPLEX_CHANNEL_PRE_BIND_CHARS;
const maxDuplexChannelPreBindFrames =
options.duplexChannelLimits?.maxPreBindBufferedFrames ??
MAX_DUPLEX_CHANNEL_PRE_BIND_FRAMES;
const maxDuplexChannelPendingRequests =
options.duplexChannelLimits?.maxPendingRequests ??
MAX_DUPLEX_CHANNEL_PENDING_REQUESTS;
const maxDuplexChannelWriteChars =
options.duplexChannelLimits?.maxWriteChars ?? MAX_DUPLEX_CHANNEL_WRITE_CHARS;
const maxDuplexChannelProtocolErrors =
options.duplexChannelLimits?.maxProtocolErrors ??
MAX_DUPLEX_CHANNEL_PROTOCOL_ERRORS;
const maxDuplexChannelTotalDataBytes =
options.duplexChannelLimits?.maxTotalDataBytes ??
MAX_DUPLEX_CHANNEL_TOTAL_DATA_BYTES;
const maxDuplexChannelDurationMs =
options.duplexChannelLimits?.maxDurationMs ?? MAX_DUPLEX_CHANNEL_DURATION_MS;
const duplexChannelOpenTimeoutMs =
options.duplexChannelLimits?.openTimeoutMs ?? DUPLEX_CHANNEL_OPEN_TIMEOUT_MS;
const duplexChannelCloseTimeoutMs =
options.duplexChannelLimits?.closeTimeoutMs ?? DUPLEX_CHANNEL_CLOSE_TIMEOUT_MS;
// ------------------------------------------------------------------
// Proactive company scopes (LOOA-629)
// ------------------------------------------------------------------
@@ -1039,7 +1196,42 @@ export function createPluginWorkerHandle(
// verifies a close acknowledgement bound to that identifier; it retires the
// worker on an unconfirmed close.
type SetupTokenPtyRouteState = "reserved" | "opening" | "open" | "closed";
// A single-consumer route state. The login pseudo-terminal route and the
// generic duplex channel route share it.
type RouteState = "reserved" | "opening" | "open" | "closed";
// Shared route-binding helpers. The login pseudo-terminal route and the duplex
// channel route both use them, so the two routes bind and settle one way.
// Settle the route wait exactly once. Replace the settler with a no-op, so a
// later exit or terminalize never settles the wait a second time.
function settleRouteWait(
route: { settleWait: (value: { exitCode: number | null }) => void },
value: { exitCode: number | null },
): void {
const settle = route.settleWait;
route.settleWait = () => {};
settle(value);
}
// Read the worker session identifier from an open reply, but only when the
// route can still bind. Return null for a malformed reply, or for a route that
// already left `opening` or terminalized. A late or a duplicate reply never
// binds, revives, or reopens a route.
function readBindableWorkerSessionId(
route: { state: RouteState; terminalized: boolean },
openResult: unknown,
): string | null {
const workerSessionId = readNonEmptyString(
isRecord(openResult) ? openResult.workerSessionId : null,
);
if (!workerSessionId || route.state !== "opening" || route.terminalized) {
return null;
}
return workerSessionId;
}
type SetupTokenPtyRouteState = RouteState;
interface SetupTokenPtyRoute {
hostRouteId: string;
state: SetupTokenPtyRouteState;
@@ -1054,15 +1246,6 @@ export function createPluginWorkerHandle(
// blocks a second open until the manager confirms the first route's close.
let setupTokenPtyRoute: SetupTokenPtyRoute | null = null;
function settleSetupTokenPtyWait(
route: SetupTokenPtyRoute,
value: { exitCode: number | null },
): void {
const settle = route.settleWait;
route.settleWait = () => {};
settle(value);
}
// Close the worker terminal by the host route identifier and verify the bound
// acknowledgement. Return true only when the worker returns an acknowledgement
// that carries the exact host route identifier. An absent, malformed,
@@ -1092,7 +1275,7 @@ export function createPluginWorkerHandle(
route.buffered = [];
// A terminalized route reports a null exit code, which the runner treats as a
// failure.
settleSetupTokenPtyWait(route, { exitCode: null });
settleRouteWait(route, { exitCode: null });
const confirmed = await closeSetupTokenPtyTerminal(route.hostRouteId);
if (setupTokenPtyRoute === route) setupTokenPtyRoute = null;
if (!confirmed) {
@@ -1144,7 +1327,7 @@ export function createPluginWorkerHandle(
const workerSessionId = readNonEmptyString(params.workerSessionId);
if (!workerSessionId || workerSessionId !== route.workerSessionId) return;
const exitCode = typeof params.exitCode === "number" ? params.exitCode : null;
settleSetupTokenPtyWait(route, { exitCode });
settleRouteWait(route, { exitCode });
}
// Close the one route on a worker exit. The worker is gone, so the manager
@@ -1158,7 +1341,7 @@ export function createPluginWorkerHandle(
route.state = "closed";
route.listener = null;
route.buffered = [];
settleSetupTokenPtyWait(route, { exitCode: null });
settleRouteWait(route, { exitCode: null });
}
// Open one live login pseudo-terminal route. Reserve the route
@@ -1219,10 +1402,8 @@ export function createPluginWorkerHandle(
throw err instanceof Error ? err : new Error(SETUP_TOKEN_PTY_OPEN_FAILED);
}
const workerSessionId = readNonEmptyString(
isRecord(openResult) ? openResult.workerSessionId : null,
);
if (!workerSessionId || route.state !== "opening" || route.terminalized) {
const workerSessionId = readBindableWorkerSessionId(route, openResult);
if (!workerSessionId) {
// A malformed reply, or a route that already left `opening`. A late or a
// duplicate reply never binds, revives, or reopens a route.
await terminalizeSetupTokenPtyRoute(route);
@@ -1268,6 +1449,345 @@ export function createPluginWorkerHandle(
};
}
// -----------------------------------------------------------------------
// Host-owned generic duplex channel route
// -----------------------------------------------------------------------
// The duplex channel route mirrors the login pseudo-terminal route model. The
// host owns the route identifier, binds the worker session identifier one time
// on a valid open reply, and keys the close on the host route identifier. It
// rejects a late or a duplicate open reply, and it retires the worker on an
// unconfirmed close. The duplex channel carries no command allowlist, so the
// caller owns the command.
//
// The route adds seven explicit bounds the pseudo-terminal route lacks. Each
// bound ends the route when it passes its limit:
// 1. pre-bind buffered bytes — the cumulative characters the host buffers
// before a data listener attaches;
// 2. pre-bind buffered frame count — the number of data frames the host
// buffers before a data listener attaches;
// 3. pending request count — the number of in-flight host→worker requests;
// 4. host→worker write size — the characters for one write;
// 5. protocol error rate — the count of malformed or mismatched data frames;
// 6. total data bytes — the cumulative inbound bytes over the whole life,
// counted before and after a data listener attaches;
// 7. route lifetime — the milliseconds from the open to the terminal end.
interface DuplexChannelRoute {
hostRouteId: string;
state: RouteState;
workerSessionId: string | null;
listener: ((chunk: string) => void) | null;
buffered: string[];
bufferedChars: number;
pendingRequests: number;
protocolErrors: number;
totalDataBytes: number;
lifetimeTimer: ReturnType<typeof setTimeout> | null;
terminalized: boolean;
settleWait: (value: { exitCode: number | null }) => void;
}
// At most one active duplex channel per worker. A non-null route blocks a
// second open until the manager confirms the first route's close.
let duplexChannelRoute: DuplexChannelRoute | null = null;
// Close the worker channel by the host route identifier and verify the bound
// acknowledgement. Return true only when the worker returns an acknowledgement
// that carries the exact host route identifier. An absent, malformed,
// mismatched, or timed-out acknowledgement returns false, so the caller fails
// closed.
async function closeDuplexChannelTerminal(hostRouteId: string): Promise<boolean> {
try {
const ack = await callInternal(
"duplexChannelClose",
{ hostRouteId },
duplexChannelCloseTimeoutMs,
);
return isRecord(ack) && readNonEmptyString(ack.hostRouteId) === hostRouteId;
} catch {
return false;
}
}
// Terminalize the route exactly once. Resolve the wait, close the worker
// channel by the host route identifier, and free the per-worker slot only
// after the close resolves. Retire the worker when the close is unconfirmed.
// Clear the route lifetime timer one time. Every terminal path and the
// worker-exit path calls this, so a timer never fires after the route ends.
function clearDuplexChannelLifetimeTimer(route: DuplexChannelRoute): void {
if (route.lifetimeTimer) {
clearTimeout(route.lifetimeTimer);
route.lifetimeTimer = null;
}
}
async function terminalizeDuplexChannelRoute(route: DuplexChannelRoute): Promise<void> {
if (route.terminalized) return;
route.terminalized = true;
route.state = "closed";
route.listener = null;
route.buffered = [];
route.bufferedChars = 0;
clearDuplexChannelLifetimeTimer(route);
// A terminalized route reports a null exit code, which the caller treats as a
// failure.
settleRouteWait(route, { exitCode: null });
const confirmed = await closeDuplexChannelTerminal(route.hostRouteId);
if (duplexChannelRoute === route) duplexChannelRoute = null;
if (!confirmed) {
// The worker did not acknowledge the close, so the host cannot prove the
// channel is gone. Fail closed: retire the worker before any reuse.
log.error(
{ pluginId },
"duplex channel close not acknowledged; retiring worker",
);
void killProcess();
}
}
// Count one protocol error for the route. End the route when the count passes
// the per-route budget, so a flood of malformed or mismatched frames bounds the
// route.
function recordDuplexChannelProtocolError(route: DuplexChannelRoute): void {
route.protocolErrors += 1;
if (route.protocolErrors > maxDuplexChannelProtocolErrors) {
void terminalizeDuplexChannelRoute(route);
}
}
// Deliver one duplex channel chunk to the bound listener in isolation. A
// listener that throws must not escape the worker stdout notification handler
// or the buffered replay, so a throw here breaks neither the notification
// dispatch loop nor the pre-bind drain. The manager catches the error and logs
// it without the raw bytes. This mirrors the `execute.log` delivery isolation.
function deliverDuplexChannelChunk(
listener: (chunk: string) => void,
chunk: string,
): void {
try {
listener(chunk);
} catch (err) {
log.error(
{ err: err instanceof Error ? err.message : String(err) },
"duplex channel data delivery threw",
);
}
}
// Route one duplex channel data notification to the per-session listener.
// Deliver only while the route is `open` and the notification carries the exact
// bound worker session identifier and a valid chunk. Count a mismatched or
// malformed frame as a protocol error. End the route at once when one chunk is
// larger than the per-chunk limit or when the cumulative bytes pass the total
// cap. Buffer a valid frame under the pre-bind bounds when no listener has
// attached yet. Never log the raw bytes.
function routeDuplexChannelData(notification: JsonRpcNotification): void {
const route = duplexChannelRoute;
if (!route || route.state !== "open") return;
const params = isRecord(notification.params) ? notification.params : {};
const workerSessionId = readNonEmptyString(params.workerSessionId);
const chunk = params.chunk;
if (
!workerSessionId ||
workerSessionId !== route.workerSessionId ||
typeof chunk !== "string" ||
chunk.length === 0
) {
// A late, unknown, malformed, or mismatched frame. Drop it and count one
// protocol error.
recordDuplexChannelProtocolError(route);
return;
}
if (chunk.length > maxDuplexChannelChunkChars) {
// One inbound chunk is larger than the per-chunk limit. End the route at
// once. Do not count the chunk as a protocol error.
void terminalizeDuplexChannelRoute(route);
return;
}
// Count the bytes of the chunk. Enforce the cumulative total-byte cap before
// and after a listener attaches. End the route when the cap is exceeded, so a
// bound listener cannot receive data past the cap.
const chunkBytes = Buffer.byteLength(chunk);
if (route.totalDataBytes + chunkBytes > maxDuplexChannelTotalDataBytes) {
void terminalizeDuplexChannelRoute(route);
return;
}
route.totalDataBytes += chunkBytes;
if (route.listener) {
deliverDuplexChannelChunk(route.listener, chunk);
return;
}
// No listener attached yet. Buffer the frame under the pre-bind bounds. End
// the route when the cumulative bytes or the frame count passes the bound.
if (
route.buffered.length + 1 > maxDuplexChannelPreBindFrames ||
route.bufferedChars + chunk.length > maxDuplexChannelPreBindChars
) {
void terminalizeDuplexChannelRoute(route);
return;
}
route.buffered.push(chunk);
route.bufferedChars += chunk.length;
}
// Route one duplex channel exit notification to the wait. Resolve only while
// the route is `open` and the notification carries the exact bound worker
// session identifier.
function routeDuplexChannelExit(notification: JsonRpcNotification): void {
const route = duplexChannelRoute;
if (!route || route.state !== "open") return;
const params = isRecord(notification.params) ? notification.params : {};
const workerSessionId = readNonEmptyString(params.workerSessionId);
if (!workerSessionId || workerSessionId !== route.workerSessionId) return;
const exitCode = typeof params.exitCode === "number" ? params.exitCode : null;
settleRouteWait(route, { exitCode });
}
// Close the one route on a worker exit. The worker is gone, so the manager
// resolves the wait with the fixed non-secret exit and clears the route one
// time. The pending channel calls reject through `rejectAllPending`.
function closeDuplexChannelRouteOnWorkerExit(): void {
const route = duplexChannelRoute;
if (!route) return;
duplexChannelRoute = null;
route.terminalized = true;
route.state = "closed";
route.listener = null;
route.buffered = [];
route.bufferedChars = 0;
clearDuplexChannelLifetimeTimer(route);
settleRouteWait(route, { exitCode: null });
}
// Open one live generic duplex channel route. Reserve the route before the open
// call, bind the worker session identifier one time on the first successful
// open reply, and return a session a caller drives. Terminalize the route on
// every open failure path.
async function openDuplexChannel(
input: DuplexChannelOpenInput,
): Promise<DuplexChannelHostSession> {
if (duplexChannelRoute) {
// A route for this worker is not yet closed and confirmed. Reject the
// second open with one fixed non-secret error before it reaches the worker.
throw new Error(DUPLEX_CHANNEL_ROUTE_BUSY);
}
const hostRouteId = randomUUID();
let settleWait: (value: { exitCode: number | null }) => void = () => {};
const waitPromise = new Promise<{ exitCode: number | null }>((resolve) => {
settleWait = resolve;
});
const route: DuplexChannelRoute = {
hostRouteId,
state: "reserved",
workerSessionId: null,
listener: null,
buffered: [],
bufferedChars: 0,
pendingRequests: 0,
protocolErrors: 0,
totalDataBytes: 0,
lifetimeTimer: null,
terminalized: false,
settleWait,
};
duplexChannelRoute = route;
route.state = "opening";
let openResult: HostToWorkerMethods["duplexChannelOpen"][1];
try {
openResult = await callInternal(
"duplexChannelOpen",
{
hostRouteId,
driverKey: input.driverKey,
companyId: input.companyId,
environmentId: input.environmentId,
providerLeaseId: input.providerLeaseId,
command: input.command,
},
duplexChannelOpenTimeoutMs,
);
} catch (err) {
// A send failure, an RPC rejection, or an open timeout. Terminalize the
// route exactly once and fail closed.
await terminalizeDuplexChannelRoute(route);
throw err instanceof Error ? err : new Error(DUPLEX_CHANNEL_OPEN_FAILED);
}
const workerSessionId = readBindableWorkerSessionId(route, openResult);
if (!workerSessionId) {
// A malformed reply, or a route that already left `opening`. A late or a
// duplicate reply never binds, revives, or reopens a route.
await terminalizeDuplexChannelRoute(route);
throw new Error(DUPLEX_CHANNEL_OPEN_FAILED);
}
// Bind the worker session identifier one time and move the route to `open`.
route.workerSessionId = workerSessionId;
route.state = "open";
// Start the route lifetime timer now the route is open. The route ends when
// the timer expires. Every terminal path and the worker-exit path clears the
// timer. Unreference the timer so it never blocks the host process shutdown.
route.lifetimeTimer = setTimeout(() => {
void terminalizeDuplexChannelRoute(route);
}, maxDuplexChannelDurationMs);
route.lifetimeTimer.unref?.();
// Send one host→worker request under the pending-request bound. End the route
// when too many requests are in-flight, so a worker that never replies cannot
// make the host hold an unbounded number of pending requests.
const sendBoundedRequest = <
M extends "duplexChannelWrite" | "duplexChannelStop",
>(
method: M,
params: HostToWorkerMethods[M][0],
): void => {
if (route.state !== "open") return;
if (route.pendingRequests >= maxDuplexChannelPendingRequests) {
void terminalizeDuplexChannelRoute(route);
return;
}
route.pendingRequests += 1;
void callInternal(method, params, duplexChannelOpenTimeoutMs)
.catch(() => {})
.finally(() => {
route.pendingRequests -= 1;
});
};
return {
onData(listener: (chunk: string) => void): void {
route.listener = listener;
if (route.buffered.length > 0) {
const pending = route.buffered;
route.buffered = [];
route.bufferedChars = 0;
for (const chunk of pending) deliverDuplexChannelChunk(listener, chunk);
}
},
write(data: string): void {
const sid = route.workerSessionId;
if (route.state !== "open" || !sid) return;
if (data.length > maxDuplexChannelWriteChars) {
// The write is larger than the size bound. End the route before the
// write reaches the worker.
void terminalizeDuplexChannelRoute(route);
return;
}
sendBoundedRequest("duplexChannelWrite", { workerSessionId: sid, data });
},
wait(): Promise<{ exitCode: number | null }> {
return waitPromise;
},
kill(): void {
const sid = route.workerSessionId;
if (!sid) return;
sendBoundedRequest("duplexChannelStop", { workerSessionId: sid });
},
async close(): Promise<void> {
await terminalizeDuplexChannelRoute(route);
},
};
}
/**
* Extract the single company a worker→host call references, mirroring the SDK
* governed-access gate's own derivation (host-client-factory.ts
@@ -1440,6 +1960,18 @@ export function createPluginWorkerHandle(
return;
}
// Duplex channel notifications: deliver data and the exit to the one
// host-owned duplex route, bound by the worker session identifier while the
// route is open.
if (notification.method === DUPLEX_CHANNEL_DATA_NOTIFICATION) {
routeDuplexChannelData(notification);
return;
}
if (notification.method === DUPLEX_CHANNEL_EXIT_NOTIFICATION) {
routeDuplexChannelExit(notification);
return;
}
// Stream notifications: forward to the stream bus via callback
if (
notification.method === "streams.open" ||
@@ -1590,6 +2122,10 @@ export function createPluginWorkerHandle(
// already rejected through `rejectAllPending`.
closeSetupTokenPtyRouteOnWorkerExit();
// Close the one duplex channel route the same way. The pending channel calls
// already rejected through `rejectAllPending`.
closeDuplexChannelRouteOnWorkerExit();
// Emit synthetic close for any orphaned stream channels so SSE clients
// are notified instead of hanging indefinitely.
if (openStreamChannels.size > 0 && options.onStreamNotification) {
@@ -2063,6 +2599,17 @@ export function createPluginWorkerHandle(
return openSetupTokenPtySession(input);
},
openDuplexChannel(input: DuplexChannelOpenInput) {
if (status !== "running" && status !== "starting") {
return Promise.reject(
new Error(
`Cannot open a duplex channel — worker for "${pluginId}" is ${status}`,
),
);
}
return openDuplexChannel(input);
},
notify(method: string, params: unknown) {
if (status !== "running") return;
const invocationScope = deriveInvocationScope(method, params);
+12 -12
View File
@@ -49,8 +49,8 @@ import {
} from "../issue-execution-policy.js";
import {
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
buildIssueBlockersResolvedWakeIdempotencyKey,
findExistingIssueBlockersResolvedWakeForAnyKey,
buildIssueBlockersResolvedWakeStateKey,
findExistingIssueBlockersResolvedWakeForReadyState,
} from "../issue-dependency-wakeups.js";
import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js";
import { getRunLogStore } from "../run-log-store.js";
@@ -5240,19 +5240,19 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
continue;
}
const idempotencyKeys = readiness.blockerIssueIds.map((blockerIssueId) =>
buildIssueBlockersResolvedWakeIdempotencyKey({
dependentIssueId: candidate.id,
resolvedBlockerIssueId: blockerIssueId,
})
);
const idempotencyKey = buildIssueBlockersResolvedWakeIdempotencyKey({
// Level-triggered dedup: key on the full blocker set (the current ready
// state), not on any single resolved edge. An older completed per-edge
// wake for an earlier partial resolution has a different key, so it does
// not suppress this wake. The shared helper still suppresses a duplicate
// wake for the SAME ready state, which bounds reconciliation.
const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({
dependentIssueId: candidate.id,
resolvedBlockerIssueId,
blockerIssueIds: readiness.blockerIssueIds,
});
const existingWake = await findExistingIssueBlockersResolvedWakeForAnyKey(db, {
const existingWake = await findExistingIssueBlockersResolvedWakeForReadyState(db, {
companyId,
idempotencyKeys,
dependentIssueId: candidate.id,
blockerIssueIds: readiness.blockerIssueIds,
});
if (existingWake) {
result.existingWakeSkipped += 1;
+90 -1
View File
@@ -83,6 +83,8 @@ const LIVE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]);
const MAX_CATCH_UP_RUNS = 25;
const MAX_ROUTINE_REVISIONS = 100;
const EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE = "execution_issue_status";
const EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES = ["blocked", "cancelled"] as const;
const ACTIVITY_GATE_IGNORED_ACTIONS = [
"issue.read_marked",
"issue.read_unmarked",
@@ -100,6 +102,37 @@ const WEEKDAY_INDEX: Record<string, number> = {
Sat: 6,
};
type ExecutionIssueTransientFailureStatus = (typeof EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES)[number];
function executionIssueTransientFailureReason(status: ExecutionIssueTransientFailureStatus) {
return `Execution issue moved to ${status}`;
}
function executionIssueTransientFailureStatusFromPayload(payload: unknown): ExecutionIssueTransientFailureStatus | null {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
const transientFailure = (payload as Record<string, unknown>).transientFailure;
if (!transientFailure || typeof transientFailure !== "object" || Array.isArray(transientFailure)) return null;
const record = transientFailure as Record<string, unknown>;
if (record.code !== EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE) return null;
return EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES.find((status) => record.status === status) ?? null;
}
function executionIssueTransientFailureClearedAtFromPayload(payload: unknown): string | null {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
const transientFailure = (payload as Record<string, unknown>).transientFailure;
if (!transientFailure || typeof transientFailure !== "object" || Array.isArray(transientFailure)) return null;
const clearedAt = (transientFailure as Record<string, unknown>).clearedAt;
return typeof clearedAt === "string" ? clearedAt : null;
}
function legacyExecutionIssueTransientFailureStatus(
failureReason: string | null,
): ExecutionIssueTransientFailureStatus | null {
return EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES.find(
(status) => failureReason === executionIssueTransientFailureReason(status),
) ?? null;
}
async function resolveCompanyDefaultResponsibleUserId(db: Db, companyId: string) {
const company = await db
.select({ defaultResponsibleUserId: companies.defaultResponsibleUserId })
@@ -3144,17 +3177,73 @@ export function routineService(
.where(eq(issues.id, issueId))
.then((rows) => rows[0] ?? null);
if (!issue || issue.originKind !== "routine_execution" || !issue.originRunId) return null;
const run = await db
.select({
id: routineRuns.id,
status: routineRuns.status,
failureReason: routineRuns.failureReason,
triggerPayload: routineRuns.triggerPayload,
})
.from(routineRuns)
.where(eq(routineRuns.id, issue.originRunId))
.then((rows) => rows[0] ?? null);
if (!run) return null;
if (issue.status === "done") {
const transientFailureStatus = executionIssueTransientFailureStatusFromPayload(run.triggerPayload)
?? legacyExecutionIssueTransientFailureStatus(run.failureReason);
const transientFailureClearedAt = executionIssueTransientFailureClearedAtFromPayload(run.triggerPayload);
return finalizeRun(issue.originRunId, {
status: "completed",
failureReason: null,
completedAt: new Date(),
...(transientFailureStatus
? {
triggerPayload: {
...(run.triggerPayload ?? {}),
transientFailure: {
code: EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE,
status: transientFailureStatus,
reason: executionIssueTransientFailureReason(transientFailureStatus),
clearedAt: transientFailureClearedAt ?? new Date().toISOString(),
},
},
}
: {}),
});
}
if (issue.status === "blocked" || issue.status === "cancelled") {
const failureReason = executionIssueTransientFailureReason(issue.status);
return finalizeRun(issue.originRunId, {
status: "failed",
failureReason: `Execution issue moved to ${issue.status}`,
failureReason,
completedAt: new Date(),
triggerPayload: {
...(run.triggerPayload ?? {}),
transientFailure: {
code: EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE,
status: issue.status,
reason: failureReason,
recordedAt: new Date().toISOString(),
},
},
});
}
const transientFailureStatus = executionIssueTransientFailureStatusFromPayload(run.triggerPayload)
?? legacyExecutionIssueTransientFailureStatus(run.failureReason);
if (run.status === "failed" && transientFailureStatus) {
return finalizeRun(issue.originRunId, {
status: "issue_created",
failureReason: null,
completedAt: null,
triggerPayload: {
...(run.triggerPayload ?? {}),
transientFailure: {
code: EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE,
status: transientFailureStatus,
reason: executionIssueTransientFailureReason(transientFailureStatus),
clearedAt: new Date().toISOString(),
},
},
});
}
return null;
@@ -162,7 +162,7 @@ describe("diagnoseRuntimeListenerBinds against live listeners", () => {
expect(diagnosis).toContain(`port ${appPort}`);
// Node's hostless listen is dual-stack, so /proc shows :: and/or 0.0.0.0.
expect(diagnosis).toMatch(/0\.0\.0\.0|::/);
expect(diagnosis).toContain("--bind custom --bind-host 127.0.0.1");
expect(diagnosis).toContain("--bind loopback");
});
});
@@ -160,6 +160,6 @@ export async function diagnoseRuntimeListenerBinds(ports: number[]): Promise<str
`${violations.join("; ")} instead of loopback only. The broker will not expose a listener `
+ "reachable off-loopback. This means the workspace checkout's dev server ignored the managed "
+ "loopback bind — a checkout that predates managed HTTPS exposure overwrites PAPERCLIP_BIND "
+ "from its own --bind argv, so the start command must pass --bind custom --bind-host 127.0.0.1."
+ "from its own --bind argv, so the start command must pass --bind loopback."
);
}
@@ -0,0 +1,145 @@
import { describe, expect, it } from "vitest";
import { createSetupTokenReaper } from "./setup-token-reaper.js";
import {
isTerminalSessionState,
type SetupTokenCleanupIdentity,
type SetupTokenCleanupRecord,
} from "./setup-token-session.js";
const NOW = 5_000;
// An in-memory reaper store. The test seeds records directly, so no database
// runs. The `listReapable` predicate mirrors the production store: a record is
// reapable when its session is terminal, its deadline is past, or its claim is
// already consumed.
function createMemoryStore() {
const rows = new Map<string, SetupTokenCleanupRecord>();
const matches = (row: SetupTokenCleanupRecord, identity: SetupTokenCleanupIdentity) =>
row.companyId === identity.companyId &&
row.ownerUserId === identity.ownerUserId &&
row.adapterType === identity.adapterType;
return {
rows,
seed(record: Partial<SetupTokenCleanupRecord> & Pick<SetupTokenCleanupRecord, "sessionId" | "leaseId">) {
const row: SetupTokenCleanupRecord = {
sessionId: record.sessionId,
companyId: record.companyId ?? "company-1",
ownerUserId: record.ownerUserId ?? "user-1",
adapterType: record.adapterType ?? "claude_local",
environmentId: record.environmentId ?? "env-1",
leaseId: record.leaseId,
deadline: record.deadline ?? 1_000,
state: record.state ?? "failed",
boundAt: record.boundAt ?? null,
};
rows.set(row.sessionId, row);
return row;
},
async listReapable(now: number): Promise<SetupTokenCleanupRecord[]> {
return [...rows.values()].filter(
(row) => isTerminalSessionState(row.state) || row.deadline <= now || row.boundAt !== null,
);
},
async remove(identity: SetupTokenCleanupIdentity): Promise<void> {
const row = rows.get(identity.sessionId);
if (row && matches(row, identity)) rows.delete(identity.sessionId);
},
};
}
interface FakeLeasesOptions {
releaseImpl?: (leaseId: string) => Promise<void>;
}
function createFakeLeases(opts: FakeLeasesOptions = {}) {
const releaseByIdCalls: string[] = [];
return {
releaseByIdCalls,
async releaseById(leaseId: string): Promise<void> {
releaseByIdCalls.push(leaseId);
if (opts.releaseImpl) await opts.releaseImpl(leaseId);
},
};
}
describe("setup-token reaper", () => {
it("releases the lease and removes the record for a reapable session", async () => {
// A prior process crashed with a live record and a lease.
const store = createMemoryStore();
store.seed({ sessionId: "orphan-1", leaseId: "lease-orphan", state: "awaiting_code", deadline: 1_000 });
const leases = createFakeLeases();
const reaper = createSetupTokenReaper({ store, leases, now: () => NOW });
const result = await reaper.sweep();
expect(result.released).toBe(1);
expect(result.failed).toBe(0);
expect(leases.releaseByIdCalls).toEqual(["lease-orphan"]);
expect(store.rows.size).toBe(0);
});
it("holds a live, unexpired, unconsumed record and never releases its lease", async () => {
const store = createMemoryStore();
store.seed({
sessionId: "live-1",
leaseId: "lease-live",
state: "awaiting_code",
// The deadline is in the future and the claim is not consumed.
deadline: NOW + 60_000,
boundAt: null,
});
const leases = createFakeLeases();
const reaper = createSetupTokenReaper({ store, leases, now: () => NOW });
const result = await reaper.sweep();
expect(result.released).toBe(0);
expect(leases.releaseByIdCalls).toHaveLength(0);
expect(store.rows.has("live-1")).toBe(true);
});
it("keeps the record when the release fails, then clears it on the next sweep", async () => {
const store = createMemoryStore();
store.seed({ sessionId: "orphan-2", leaseId: "lease-orphan-2", state: "failed" });
let attempt = 0;
const leases = createFakeLeases({
releaseImpl: async () => {
attempt += 1;
if (attempt === 1) throw new Error("provider down");
},
});
const reaper = createSetupTokenReaper({ store, leases, now: () => NOW });
// First sweep: the release rejects, so the record stays for a retry.
const first = await reaper.sweep();
expect(first.released).toBe(0);
expect(first.failed).toBe(1);
expect(store.rows.has("orphan-2")).toBe(true);
// Second sweep: the release confirms, so the record clears.
const second = await reaper.sweep();
expect(second.released).toBe(1);
expect(second.failed).toBe(0);
expect(leases.releaseByIdCalls).toEqual(["lease-orphan-2", "lease-orphan-2"]);
expect(store.rows.size).toBe(0);
});
it("reaps every reapable record in one sweep and stays idempotent on the next", async () => {
const store = createMemoryStore();
store.seed({ sessionId: "orphan-a", leaseId: "lease-a", state: "failed" });
store.seed({ sessionId: "orphan-b", leaseId: "lease-b", state: "cancelled" });
// A consumed stored claim is reapable through the bound marker.
store.seed({ sessionId: "orphan-c", leaseId: "lease-c", state: "stored", deadline: NOW + 60_000, boundAt: 4_000 });
const leases = createFakeLeases();
const reaper = createSetupTokenReaper({ store, leases, now: () => NOW });
const first = await reaper.sweep();
expect(first.released).toBe(3);
expect(store.rows.size).toBe(0);
// The interval sweep runs over the now-empty store without an error.
const second = await reaper.sweep();
expect(second.released).toBe(0);
expect(second.failed).toBe(0);
});
});
+87
View File
@@ -0,0 +1,87 @@
import type { Db } from "@paperclipai/db";
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
import { environmentService } from "./environments.js";
import {
reapSetupTokenLeases,
type SetupTokenCleanupStore,
type SetupTokenLeaseManager,
type SetupTokenReapResult,
} from "./setup-token-session.js";
import {
createProductionSetupTokenCleanupStore,
createProductionSetupTokenSandboxProvider,
} from "./setup-token-transport-binding.js";
// The restart-safe cleanup backstop for the Claude setup-token login flow.
//
// The in-process five-minute timer in the login-session service stays the
// primary control. This reaper is the backstop. It runs on server startup and on
// a fixed interval, so a sandbox lease survives a server restart and a release
// failure. The reaper reads the durable cleanup store and releases any lease
// whose session is terminal, past its deadline, or already consumed. It uses the
// same convention as the Codex device-login reaper, so both flows use one reaper
// convention.
//
// The reaper never records secret data. The durable cleanup record holds only
// ids, the deadline, the claim marker, and the state. It never holds a URL, a
// code, a token, or a raw process chunk.
/** The dependencies the standalone reaper needs. The reaper only reads reapable
* records and releases a lease by id, so it takes a store subset and a lease
* releaser, not the full session service. */
export interface SetupTokenReaperDeps {
store: Pick<SetupTokenCleanupStore, "listReapable" | "remove">;
leases: Pick<SetupTokenLeaseManager, "releaseById">;
now?: () => number;
log?: (line: string) => void;
}
/**
* Builds the standalone setup-token reaper. One sweep reaps every reapable
* record: it releases the lease and clears the row. A failed release stays
* retryable, so the next sweep retries it.
*/
export function createSetupTokenReaper(deps: SetupTokenReaperDeps) {
const now = deps.now ?? Date.now;
async function sweep(): Promise<SetupTokenReapResult> {
return reapSetupTokenLeases({ store: deps.store, leases: deps.leases, log: deps.log }, now());
}
return { sweep };
}
export type SetupTokenReaper = ReturnType<typeof createSetupTokenReaper>;
// ---------------------------------------------------------------------------
// The production runtime binding.
// ---------------------------------------------------------------------------
export interface ProductionSetupTokenReaperDeps {
db: Db;
environmentRuntime: EnvironmentRuntimeService;
log?: (line: string) => void;
}
/**
* Builds the production setup-token reaper. It reads the durable cleanup store
* and releases a lease by id through the production sandbox provider. The
* provider release resolves the lease and its environment from the database and
* tears down the remote sandbox through the driver, so the release frees the
* remote sandbox and not only the database row. The reaper never acquires a
* lease, so it binds no live pseudo-terminal opener.
*/
export function createProductionSetupTokenReaper(
deps: ProductionSetupTokenReaperDeps,
): SetupTokenReaper {
const environments = environmentService(deps.db);
const sandbox = createProductionSetupTokenSandboxProvider({
environments,
environmentRuntime: deps.environmentRuntime,
log: deps.log,
});
const store = createProductionSetupTokenCleanupStore(deps.db);
return createSetupTokenReaper({
store,
leases: { releaseById: (leaseId) => sandbox.release(leaseId) },
log: deps.log,
});
}
+102 -51
View File
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import {
claudeSetupTokenSessions,
adapterAuthSessions,
companies,
createDb,
environments,
@@ -50,7 +50,6 @@ const SYNTH_TOKEN = "sk-ant-oat01-SYNTHETICSYNTHETICSYNTHETIC01";
const OWNER_SCOPE: SetupTokenSessionScope = {
companyId: "company-1",
ownerUserId: "user-1",
targetAgentId: "agent-1",
adapterType: "claude_local",
environmentId: "env-1",
};
@@ -171,7 +170,6 @@ function identityFromScope(scope: SetupTokenSessionScope, sessionId: string): Se
companyId: scope.companyId,
ownerUserId: scope.ownerUserId,
adapterType: scope.adapterType,
environmentId: scope.environmentId,
};
}
@@ -179,8 +177,7 @@ function identityMatchesRow(row: SetupTokenCleanupRecord, identity: SetupTokenCl
return (
row.companyId === identity.companyId &&
row.ownerUserId === identity.ownerUserId &&
row.adapterType === identity.adapterType &&
row.environmentId === identity.environmentId
row.adapterType === identity.adapterType
);
}
@@ -240,7 +237,7 @@ function buildService<L extends SetupTokenLeaseManager = FakeLeaseManager>(overr
leases?: L;
store?: FakeStore;
rateLimiter?: SetupTokenRateLimiter;
caps?: { perOwner: number; perAgent: number; perCompany: number };
caps?: { perOwner: number; perCompany: number };
ttlMs?: number;
tokenRetentionMs?: number;
now?: () => number;
@@ -276,7 +273,7 @@ function buildService<L extends SetupTokenLeaseManager = FakeLeaseManager>(overr
store,
completeCredential,
rateLimiter: overrides.rateLimiter ?? allowAllRateLimiter(),
caps: overrides.caps ?? { perOwner: 5, perAgent: 5, perCompany: 5 },
caps: overrides.caps ?? { perOwner: 5, perCompany: 5 },
ttlMs: overrides.ttlMs ?? 60_000,
tokenRetentionMs: overrides.tokenRetentionMs,
now: overrides.now,
@@ -321,10 +318,12 @@ describe("SetupTokenSessionService.start", () => {
});
});
it("rejects a caller over the per-owner cap with 429", async () => {
const { service } = buildService({ caps: { perOwner: 1, perAgent: 5, perCompany: 5 } });
it("rejects a second start on the same company, owner, and adapter slot with 429", async () => {
const { service } = buildService({ caps: { perOwner: 1, perCompany: 5 } });
await service.start(OWNER_SCOPE);
await expect(service.start({ ...OWNER_SCOPE, targetAgentId: "agent-2" })).rejects.toMatchObject({
// The second start holds the same company, owner, and adapter slot, so it
// fails closed with the fixed cap error.
await expect(service.start(OWNER_SCOPE)).rejects.toMatchObject({
status: 429,
message: SETUP_TOKEN_CAP_EXCEEDED,
});
@@ -332,7 +331,7 @@ describe("SetupTokenSessionService.start", () => {
});
describe("SetupTokenSessionService.start cap reservation (concurrency)", () => {
const capsPerOwnerOne = { perOwner: 1, perAgent: 5, perCompany: 5 };
const capsPerOwnerOne = { perOwner: 1, perCompany: 5 };
it("reserves the owner slot synchronously, so a concurrent start fails closed with 429", async () => {
const leases = new DeferredLeaseManager();
@@ -506,23 +505,19 @@ describe("SetupTokenSessionService.submitCode", () => {
describe("SetupTokenSessionService authorization boundary", () => {
const crossCompany: SetupTokenSessionScope = { ...OWNER_SCOPE, companyId: "company-2" };
const sameCompanyOtherUser: SetupTokenSessionScope = { ...OWNER_SCOPE, ownerUserId: "user-9" };
const otherAgent: SetupTokenSessionScope = { ...OWNER_SCOPE, targetAgentId: "agent-9" };
const otherAdapter: SetupTokenSessionScope = { ...OWNER_SCOPE, adapterType: "codex_local" };
const otherEnvironment: SetupTokenSessionScope = { ...OWNER_SCOPE, environmentId: "env-9" };
it("returns the same not-found for a cross-scope caller on every operation", async () => {
const { service, processes } = buildService();
const { sessionId } = await service.start(OWNER_SCOPE);
processes[0].surfacePrompt(FULL_LOGIN_URL);
// The full-scope match rejects a mismatch in any of the five scope fields:
// the company, the owner, the agent, the adapter, and the environment.
// The scope match rejects a mismatch in any of the three identity fields:
// the company, the owner, and the adapter.
for (const scope of [
crossCompany,
sameCompanyOtherUser,
otherAgent,
otherAdapter,
otherEnvironment,
]) {
expect(() => service.readPrompt(sessionId, scope)).toThrow(SETUP_TOKEN_SESSION_NOT_FOUND);
expect(() => service.submitCode(sessionId, scope, "code")).toThrow(SETUP_TOKEN_SESSION_NOT_FOUND);
@@ -539,21 +534,19 @@ describe("SetupTokenSessionService authorization boundary", () => {
});
describe("SetupTokenSessionService company-and-environment scope", () => {
// The agentless company scope. It carries a null agent id.
const COMPANY_SCOPE: SetupTokenSessionScope = { ...OWNER_SCOPE, targetAgentId: null };
const COMPANY_SCOPE: SetupTokenSessionScope = OWNER_SCOPE;
const companyKey = {
companyId: COMPANY_SCOPE.companyId,
ownerUserId: COMPANY_SCOPE.ownerUserId,
adapterType: COMPANY_SCOPE.adapterType,
};
it("resolves an agentless session by the company key and returns the intrinsic environment", async () => {
it("resolves a session by the company key and returns the intrinsic environment", async () => {
const { service, processes } = buildService();
const { sessionId } = await service.start(COMPANY_SCOPE);
processes[0].surfacePrompt(FULL_LOGIN_URL);
const scope = service.resolveCompanyScope(sessionId, companyKey);
expect(scope.targetAgentId).toBeNull();
expect(scope.environmentId).toBe(COMPANY_SCOPE.environmentId);
const descriptor = service.describeOwned(sessionId, scope);
@@ -562,16 +555,6 @@ describe("SetupTokenSessionService company-and-environment scope", () => {
expect(descriptor.loginUrl).toBe(FULL_LOGIN_URL);
});
it("rejects an agent-scoped session through the company key", async () => {
const { service } = buildService();
// The session carries a non-null agent id. The company key requires the
// agentless marker, so a company route cannot reach an agent session.
const { sessionId } = await service.start(OWNER_SCOPE);
expect(() => service.resolveCompanyScope(sessionId, companyKey)).toThrow(
SETUP_TOKEN_SESSION_NOT_FOUND,
);
});
it("returns the same not-found for a foreign company key", async () => {
const { service } = buildService();
const { sessionId } = await service.start(COMPANY_SCOPE);
@@ -1125,7 +1108,6 @@ describe("SetupTokenSessionService durable state scope", () => {
companyId: OWNER_SCOPE.companyId,
ownerUserId: "intruder",
adapterType: OWNER_SCOPE.adapterType,
environmentId: OWNER_SCOPE.environmentId,
},
"submitting",
);
@@ -1161,7 +1143,7 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
});
afterEach(async () => {
await db.delete(claudeSetupTokenSessions);
await db.delete(adapterAuthSessions);
await db.delete(environments);
await db.delete(companies);
});
@@ -1170,10 +1152,12 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
await stopDb?.();
});
// Seed a company and an environment, then return one fresh record identity.
async function seedScope(): Promise<SetupTokenCleanupIdentity> {
// The seeded scope carries the intrinsic environment. The store record needs
// the environment for the non-null column, but the identity match drops it.
type SeededScope = SetupTokenCleanupIdentity & { environmentId: string };
async function seedCompany(): Promise<string> {
const companyId = randomUUID();
const environmentId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Acme",
@@ -1183,6 +1167,11 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
createdAt: new Date(),
updatedAt: new Date(),
});
return companyId;
}
async function seedEnvironment(): Promise<string> {
const environmentId = randomUUID();
await db.insert(environments).values({
id: environmentId,
name: `sandbox-${environmentId.slice(0, 8)}`,
@@ -1192,6 +1181,13 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
createdAt: new Date(),
updatedAt: new Date(),
});
return environmentId;
}
// Seed a company and an environment, then return one fresh record scope.
async function seedScope(): Promise<SeededScope> {
const companyId = await seedCompany();
const environmentId = await seedEnvironment();
return {
sessionId: randomUUID(),
companyId,
@@ -1202,13 +1198,13 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
}
async function insertRecord(
identity: SetupTokenCleanupIdentity,
scope: SeededScope,
state: SetupTokenCleanupRecord["state"],
deadlineMs: number,
): Promise<void> {
const store = createDbSetupTokenCleanupStore(db);
await store.record({
...identity,
...scope,
leaseId: "lease-1",
deadline: deadlineMs,
state,
@@ -1219,23 +1215,53 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
async function readRow(sessionId: string) {
const rows = await db
.select()
.from(claudeSetupTokenSessions)
.where(eq(claudeSetupTokenSessions.sessionId, sessionId));
.from(adapterAuthSessions)
.where(eq(adapterAuthSessions.publicSessionId, sessionId));
return rows[0];
}
it("records the non-secret cleanup record and reads it back", async () => {
const identity = await seedScope();
await insertRecord(identity, "starting", Date.now() + 60_000);
const row = await readRow(identity.sessionId);
expect(row?.companyId).toBe(identity.companyId);
expect(row?.ownerUserId).toBe(identity.ownerUserId);
expect(row?.adapterType).toBe(identity.adapterType);
expect(row?.environmentId).toBe(identity.environmentId);
expect(row?.state).toBe("starting");
it("records the non-secret cleanup record on the unified table and reads it back", async () => {
const scope = await seedScope();
await insertRecord(scope, "starting", Date.now() + 60_000);
const row = await readRow(scope.sessionId);
// The store writes the unified `adapter_auth_sessions` columns. The public
// session id holds the record session id, and the owner maps to
// `started_by_user_id`, the state to `status`.
expect(row?.publicSessionId).toBe(scope.sessionId);
expect(row?.companyId).toBe(scope.companyId);
expect(row?.startedByUserId).toBe(scope.ownerUserId);
expect(row?.adapterType).toBe(scope.adapterType);
expect(row?.environmentId).toBe(scope.environmentId);
expect(row?.status).toBe("starting");
expect(row?.boundAt).toBeNull();
});
it("rejects a second active record on the same company, owner, and adapter slot", async () => {
const scope = await seedScope();
await insertRecord(scope, "awaiting_code", Date.now() + 60_000);
// A second active record for the same company, owner, and adapter conflicts
// on the active-slot unique index, even from a different environment. This
// proves the slot dropped the environment term.
const secondEnvironment = await seedEnvironment();
const sameSlot: SeededScope = {
...scope,
sessionId: randomUUID(),
environmentId: secondEnvironment,
};
await expect(insertRecord(sameSlot, "awaiting_code", Date.now() + 60_000)).rejects.toThrow();
// A different owner in the same company holds an independent slot, so the
// insert succeeds.
const otherOwner: SeededScope = {
...scope,
sessionId: randomUUID(),
ownerUserId: `user-${randomUUID().slice(0, 8)}`,
};
await insertRecord(otherOwner, "awaiting_code", Date.now() + 60_000);
expect((await readRow(otherOwner.sessionId))?.startedByUserId).toBe(otherOwner.ownerUserId);
});
it("consumes a stored claim once with one conditional write", async () => {
const identity = await seedScope();
await insertRecord(identity, "stored", Date.now() + 60_000);
@@ -1262,6 +1288,31 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
expect((await readRow(identity.sessionId))?.boundAt).toBeNull();
});
it("returns no row for a cross-company consume and leaves the claim unconsumed", async () => {
const identity = await seedScope();
await insertRecord(identity, "stored", Date.now() + 60_000);
const store = createDbSetupTokenCleanupStore(db);
// A claim from another company does not match the stored row. The predicate
// scopes on company_id, so the consume returns no row and leaves bound_at null.
const otherCompanyId = await seedCompany();
const foreign = await store.consumeStoredClaim({ ...identity, companyId: otherCompanyId });
expect(foreign).toBeNull();
expect((await readRow(identity.sessionId))?.boundAt).toBeNull();
});
it("returns no row for a cross-adapter consume and leaves the claim unconsumed", async () => {
const identity = await seedScope();
await insertRecord(identity, "stored", Date.now() + 60_000);
const store = createDbSetupTokenCleanupStore(db);
// A claim for a different adapter does not match the stored row. The predicate
// scopes on adapter_type, so the consume returns no row and leaves bound_at null.
const foreign = await store.consumeStoredClaim({ ...identity, adapterType: "codex_local" });
expect(foreign).toBeNull();
expect((await readRow(identity.sessionId))?.boundAt).toBeNull();
});
it("returns no row for an expired claim", async () => {
const identity = await seedScope();
await insertRecord(identity, "stored", Date.now() - 1_000);
@@ -1333,12 +1384,12 @@ describeEmbeddedPostgres("durable setup-token cleanup store (embedded postgres)"
});
const lockHeld = lockDb.transaction(async (tx) => {
await tx.execute(
sql`SELECT bound_at FROM claude_setup_token_sessions WHERE session_id = ${identity.sessionId} FOR UPDATE`,
sql`SELECT bound_at FROM adapter_auth_sessions WHERE public_session_id = ${identity.sessionId} FOR UPDATE`,
);
// Set a near deadline inside the locked transaction. The consume cannot see
// this value until the transaction commits.
await tx.execute(
sql`UPDATE claude_setup_token_sessions SET deadline_at = clock_timestamp() + interval '300 milliseconds' WHERE session_id = ${identity.sessionId}`,
sql`UPDATE adapter_auth_sessions SET expires_at = clock_timestamp() + interval '300 milliseconds' WHERE public_session_id = ${identity.sessionId}`,
);
signalLocked();
await gate;
+188 -149
View File
@@ -9,9 +9,8 @@
// The service gives the harness these operations against the one live session:
// start the session, read the login prompt, submit one browser code, cancel the
// session, expire the session on a timeout, and receive the token. Every
// operation verifies the company, the owner user, and the target agent. A
// missing session and a cross-scope session both return the same not-found
// error.
// operation verifies the company, the owner user, and the adapter. A missing
// session and a cross-scope session both return the same not-found error.
//
// Security controls folded here:
// * SR-1 (no secret in a log): the service keeps the full login URL, the
@@ -35,9 +34,15 @@
import { randomBytes } from "node:crypto";
import { and, eq, gt, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { claudeSetupTokenSessions } from "@paperclipai/db";
import { adapterAuthSessions } from "@paperclipai/db";
import type { AgentAdapterType } from "@paperclipai/shared";
// The setup-token login flow supports only the `claude_local` adapter. The
// unified `adapter_auth_sessions` table also holds the Codex device-login rows,
// so every store scan filters by this adapter to reach only the setup-token
// rows.
const SETUP_TOKEN_ADAPTER_TYPE: AgentAdapterType = "claude_local";
/**
* The session states. The four terminal states end the login. The `stored`
* state is not terminal: a session enters `stored` after a successful
@@ -68,24 +73,24 @@ export function isTerminalSessionState(state: SetupTokenSessionState): boolean {
}
/**
* The immutable owner scope of a session. Every operation verifies all five
* fields against the stored scope. The service builds the scope once at start
* and never changes it.
* The immutable owner scope of a session. The service builds the scope once at
* start and never changes it. The session identity is the company, the owner,
* and the adapter. Every operation verifies these three fields against the
* stored scope.
*
* The `targetAgentId` is null for a company-and-environment login. That login
* has no agent id: a hire flow starts one session before an agent exists. The
* agent-scoped login sets `targetAgentId` to the agent id. The full-scope match
* treats null as one immutable value, so a company login and an agent login
* never resolve one another.
* The active company credential slot is scoped to the company, the owner, and
* the adapter. Two owners in one company no longer conflict. The scope drops the
* environment term from the slot, so the same owner cannot hold one active login
* per environment.
*/
export interface SetupTokenSessionScope {
companyId: string;
ownerUserId: string;
targetAgentId: string | null;
// The adapter and the environment of the login. The durable cleanup record
// keys on the company, the owner, the adapter, and the environment, so a hire
// flow with no agent id still resolves one record.
// The adapter of the login. It is part of the session identity.
adapterType: string;
// The environment of the login. The store persists it on the row, but it is
// not part of the session identity: a caller never addresses a session by the
// environment, and the active slot no longer includes the environment.
environmentId: string;
// The optional confirmed-overwrite capture. The client sends it when a stored
// token fails the agent test. The secret writer reads it and rotates the
@@ -183,9 +188,9 @@ export interface SetupTokenLeaseManager {
/**
* The non-secret cleanup record. It holds only ids, the deadline, the claim
* marker, and the state. It never holds a URL, a code, a token, or a raw process
* chunk. The service persists it at start so a restart can reap the
* lease. The record keys on the company, the owner, the adapter, and the
* environment, not the agent id, so a hire flow with no agent still resolves it.
* chunk. The service persists it at start so a restart can reap the lease. The
* record carries the environment, so the store writes the non-null environment
* column on the row.
*/
export interface SetupTokenCleanupRecord {
sessionId: string;
@@ -202,16 +207,15 @@ export interface SetupTokenCleanupRecord {
}
/**
* The immutable identity of a cleanup record. It is the full owner scope plus
* the session id. Every durable write matches on all five fields, so a write
* never updates a row by session id alone.
* The immutable identity of a cleanup record. It is the company, the owner, the
* adapter, and the public session id. Every durable write matches on all four
* fields, so a write never updates a row by the session id alone.
*/
export interface SetupTokenCleanupIdentity {
sessionId: string;
companyId: string;
ownerUserId: string;
adapterType: string;
environmentId: string;
}
/**
@@ -249,20 +253,66 @@ export interface SetupTokenCleanupStore {
consumeStoredClaim(identity: SetupTokenCleanupIdentity): Promise<SetupTokenCleanupRecord | null>;
}
/** The counts one reaper sweep produced over the durable cleanup store. */
export interface SetupTokenReapResult {
/** The reapable records whose lease released and whose row cleared. */
released: number;
/** The reapable records whose lease release failed. The row stays for a retry. */
failed: number;
}
/**
* The shared setup-token reap logic. It reads the durable store and releases any
* lease whose session is terminal, past its deadline, or already consumed. It
* runs after a restart and on the scheduler interval, so it frees a lease that a
* crash left behind (SR-4). A release failure stays retryable: the sweep leaves
* the record for a later run. The standalone reaper and the in-flight session
* service both call this function, so both flows use one reap convention.
*/
export async function reapSetupTokenLeases(
deps: {
store: Pick<SetupTokenCleanupStore, "listReapable" | "remove">;
leases: Pick<SetupTokenLeaseManager, "releaseById">;
log?: (line: string) => void;
},
now: number,
): Promise<SetupTokenReapResult> {
const log = deps.log ?? (() => {});
const records = await deps.store.listReapable(now);
let released = 0;
let failed = 0;
for (const record of records) {
try {
await deps.leases.releaseById(record.leaseId);
await deps.store.remove({
sessionId: record.sessionId,
companyId: record.companyId,
ownerUserId: record.ownerUserId,
adapterType: record.adapterType,
});
released += 1;
} catch {
failed += 1;
log("[paperclip] Setup-token reaper: a lease release failed; it stays retryable.");
}
}
return { released, failed };
}
/** A per-key start rate limiter. It matches the invite-rate-limit shape. */
export interface SetupTokenRateLimiter {
consume(key: string): { allowed: boolean; retryAfterSeconds: number };
}
export interface SetupTokenSessionCaps {
// The active-session cap per company, owner, and adapter slot. The slot cap
// mirrors the database active-slot unique index as defense in depth.
perOwner: number;
perAgent: number;
perCompany: number;
}
export const DEFAULT_SETUP_TOKEN_SESSION_CAPS: SetupTokenSessionCaps = {
perOwner: 1,
perAgent: 1,
perCompany: 3,
};
@@ -523,8 +573,8 @@ export function assessConfidentialStartup(config: ConfidentialTransportConfig):
/**
* A synchronous capacity hold for the enforced caps. The service reserves the
* capacity for the owner, the agent, and the company before the first `await` in
* {@link SetupTokenSessionService.start}, so two concurrent starts for one owner
* capacity for the slot and the company before the first `await` in
* {@link SetupTokenSessionService.start}, so two concurrent starts for one slot
* never both pass the cap. The service holds the reservation for the whole
* session lifetime and releases it exactly one time: on an early start failure or
* on the terminal cleanup. The `released` flag makes the release idempotent.
@@ -532,8 +582,9 @@ export function assessConfidentialStartup(config: ConfidentialTransportConfig):
interface CapReservation {
released: boolean;
companyId: string;
ownerUserId: string;
targetAgentId: string | null;
// The company, owner, and adapter slot key. The reservation counts one active
// session per slot, so it mirrors the database active-slot unique index.
slotKey: string;
}
interface StoredSession {
@@ -635,11 +686,11 @@ function defaultSessionId(): string {
*/
export class SetupTokenSessionService {
private readonly sessions = new Map<string, StoredSession>();
// The live reservation counts per owner, per agent, and per company. The start
// path reads and increments these synchronously before the first `await`, so
// the cap decision and the capacity hold are one atomic step on the event loop.
private readonly reservedByOwner = new Map<string, number>();
private readonly reservedByAgent = new Map<string, number>();
// The live reservation counts per slot and per company. The start path reads
// and increments these synchronously before the first `await`, so the cap
// decision and the capacity hold are one atomic step on the event loop. The
// slot key is the company, the owner, and the adapter.
private readonly reservedBySlot = new Map<string, number>();
private readonly reservedByCompany = new Map<string, number>();
private readonly factory: SetupTokenLoginProcessFactory;
private readonly leases: SetupTokenLeaseManager;
@@ -669,8 +720,8 @@ export class SetupTokenSessionService {
/**
* Builds the durable-record identity for a session. The store matches every
* write on the full owner scope and the session id, so no write updates a row
* by the session id alone.
* write on the company, the owner, the adapter, and the session id, so no write
* updates a row by the session id alone.
*/
private identityOf(session: StoredSession): SetupTokenCleanupIdentity {
return {
@@ -678,7 +729,6 @@ export class SetupTokenSessionService {
companyId: session.scope.companyId,
ownerUserId: session.scope.ownerUserId,
adapterType: session.scope.adapterType,
environmentId: session.scope.environmentId,
};
}
@@ -725,40 +775,38 @@ export class SetupTokenSessionService {
}
}
/** Builds the company, owner, and adapter slot key. The reservation and the
* database active-slot unique index share this identity. */
private static slotKey(
scope: Pick<SetupTokenSessionScope, "companyId" | "ownerUserId" | "adapterType">,
): string {
return [scope.companyId, scope.ownerUserId, scope.adapterType].join("\u0000");
}
/**
* Reserves the capacity for one start under every enforced cap. The method is
* synchronous, so it runs to completion before the first `await` in
* {@link start}. Two concurrent starts for one owner cannot interleave inside
* it: the first reserves the owner slot, and the second reads the incremented
* count and fails closed with the fixed 429 cap error. The method holds the
* per-owner, per-agent, and per-company semantics: it counts the agent slot
* only for an agent-scoped login, because a null agent id is not a shared key
* across owners. It increments no counter on a rejection, so a rejected start
* {@link start}. Two concurrent starts for one slot cannot interleave inside
* it: the first reserves the slot, and the second reads the incremented count
* and fails closed with the fixed 429 cap error. The method holds the per-slot
* and per-company semantics; the slot is the company, the owner, and the
* adapter. It increments no counter on a rejection, so a rejected start
* reserves nothing.
*/
private reserveCapacity(scope: SetupTokenSessionScope): CapReservation {
if ((this.reservedByOwner.get(scope.ownerUserId) ?? 0) >= this.caps.perOwner) {
throw new SetupTokenSessionError(429, SETUP_TOKEN_CAP_EXCEEDED);
}
if (
scope.targetAgentId !== null &&
(this.reservedByAgent.get(scope.targetAgentId) ?? 0) >= this.caps.perAgent
) {
const slotKey = SetupTokenSessionService.slotKey(scope);
if ((this.reservedBySlot.get(slotKey) ?? 0) >= this.caps.perOwner) {
throw new SetupTokenSessionError(429, SETUP_TOKEN_CAP_EXCEEDED);
}
if ((this.reservedByCompany.get(scope.companyId) ?? 0) >= this.caps.perCompany) {
throw new SetupTokenSessionError(429, SETUP_TOKEN_CAP_EXCEEDED);
}
SetupTokenSessionService.incrementCount(this.reservedByOwner, scope.ownerUserId);
if (scope.targetAgentId !== null) {
SetupTokenSessionService.incrementCount(this.reservedByAgent, scope.targetAgentId);
}
SetupTokenSessionService.incrementCount(this.reservedBySlot, slotKey);
SetupTokenSessionService.incrementCount(this.reservedByCompany, scope.companyId);
return {
released: false,
companyId: scope.companyId,
ownerUserId: scope.ownerUserId,
targetAgentId: scope.targetAgentId,
slotKey,
};
}
@@ -766,16 +814,12 @@ export class SetupTokenSessionService {
* Releases a capacity reservation exactly one time. The `released` flag makes
* the release idempotent, so an early start failure and the terminal cleanup
* never double-release one reservation. The method decrements the same counters
* that {@link reserveCapacity} incremented, and it drops the agent counter only
* for an agent-scoped login.
* that {@link reserveCapacity} incremented.
*/
private releaseReservation(reservation: CapReservation): void {
if (reservation.released) return;
reservation.released = true;
SetupTokenSessionService.decrementCount(this.reservedByOwner, reservation.ownerUserId);
if (reservation.targetAgentId !== null) {
SetupTokenSessionService.decrementCount(this.reservedByAgent, reservation.targetAgentId);
}
SetupTokenSessionService.decrementCount(this.reservedBySlot, reservation.slotKey);
SetupTokenSessionService.decrementCount(this.reservedByCompany, reservation.companyId);
}
@@ -869,7 +913,6 @@ export class SetupTokenSessionService {
companyId: scope.companyId,
ownerUserId: scope.ownerUserId,
adapterType: scope.adapterType,
environmentId: scope.environmentId,
})
.catch(() => {});
this.releaseReservation(reservation);
@@ -962,12 +1005,11 @@ export class SetupTokenSessionService {
/**
* Resolves a session for an operation. It returns the session only when the
* id exists and the stored scope equals the caller scope in all five fields:
* the company, the owner, the agent, the adapter, and the environment. A
* missing session and a cross-scope session both throw the same not-found
* error, so a caller cannot tell them apart. The full-scope match makes
* a cross-adapter and a cross-environment session return the same not-found
* error as a missing session.
* id exists and the stored scope equals the caller scope in all three identity
* fields: the company, the owner, and the adapter. A missing session and a
* cross-scope session both throw the same not-found error, so a caller cannot
* tell them apart. The scope match makes a cross-company, a cross-owner, and a
* cross-adapter session return the same not-found error as a missing session.
*/
private resolveOwned(sessionId: string, scope: SetupTokenSessionScope): StoredSession {
const session = this.sessions.get(sessionId);
@@ -975,9 +1017,7 @@ export class SetupTokenSessionService {
!session ||
session.scope.companyId !== scope.companyId ||
session.scope.ownerUserId !== scope.ownerUserId ||
session.scope.targetAgentId !== scope.targetAgentId ||
session.scope.adapterType !== scope.adapterType ||
session.scope.environmentId !== scope.environmentId
session.scope.adapterType !== scope.adapterType
) {
throw new SetupTokenSessionError(404, SETUP_TOKEN_SESSION_NOT_FOUND);
}
@@ -988,16 +1028,14 @@ export class SetupTokenSessionService {
* Resolves the immutable scope of a company-and-environment session. The
* caller provides the company, the owner, and the adapter it derived from the
* request; the route path gives the company, the actor gives the owner, and
* the route fixes the adapter. The lookup matches these three fields and the
* agentless marker, then returns the full scope, including the intrinsic
* environment.
* the route fixes the adapter. The lookup matches these three fields, then
* returns the full scope, including the intrinsic environment.
*
* A caller never supplies the environment on a read, a submit, a cancel, or a
* completion. The environment is intrinsic to the session, so a foreign
* environment cannot address the session. A missing session and a
* cross-company, cross-owner, or cross-adapter session all throw the same
* not-found error. The agentless marker rejects an
* agent-scoped session, so a company route never resolves an agent session.
* not-found error.
*/
resolveCompanyScope(
sessionId: string,
@@ -1006,7 +1044,6 @@ export class SetupTokenSessionService {
const session = this.sessions.get(sessionId);
if (
!session ||
session.scope.targetAgentId !== null ||
session.scope.companyId !== key.companyId ||
session.scope.ownerUserId !== key.ownerUserId ||
session.scope.adapterType !== key.adapterType
@@ -1251,29 +1288,11 @@ export class SetupTokenSessionService {
* The startup reaper. It reads the durable store and releases any lease whose
* session is terminal or past its deadline. It runs after a restart, so it
* frees a lease that a crash left behind (SR-4). A release failure stays
* retryable: the reaper leaves the record for a later run.
* retryable: the reaper leaves the record for a later run. The standalone
* scheduled reaper calls the same {@link reapSetupTokenLeases} logic.
*/
async reap(now: number = this.now()): Promise<{ released: number; failed: number }> {
const records = await this.store.listReapable(now);
let released = 0;
let failed = 0;
for (const record of records) {
try {
await this.leases.releaseById(record.leaseId);
await this.store.remove({
sessionId: record.sessionId,
companyId: record.companyId,
ownerUserId: record.ownerUserId,
adapterType: record.adapterType,
environmentId: record.environmentId,
});
released += 1;
} catch {
failed += 1;
this.log("[paperclip] Setup-token reaper: a lease release failed; it stays retryable.");
}
}
return { released, failed };
async reap(now: number = this.now()): Promise<SetupTokenReapResult> {
return reapSetupTokenLeases({ store: this.store, leases: this.leases, log: this.log }, now);
}
/**
@@ -1294,35 +1313,48 @@ export class SetupTokenSessionService {
}
// --- The durable, database-backed cleanup store ------------------------------
//
// The store reads and writes the unified `adapter_auth_sessions` table. It maps
// the setup-token record fields to the unified columns:
// - `sessionId` -> `public_session_id`
// - `ownerUserId` -> `started_by_user_id`
// - `leaseId` -> `provider_lease_id`
// - `state` -> `status`
// - `deadline` -> `expires_at`
// The table also holds the Codex device-login rows, so every scan filters by the
// setup-token adapter to reach only the setup-token rows.
type ClaudeSetupTokenSessionRow = typeof claudeSetupTokenSessions.$inferSelect;
type AdapterAuthSessionRow = typeof adapterAuthSessions.$inferSelect;
/** Maps a database row to the non-secret cleanup record. */
function toCleanupRecord(row: ClaudeSetupTokenSessionRow): SetupTokenCleanupRecord {
function toCleanupRecord(row: AdapterAuthSessionRow): SetupTokenCleanupRecord {
return {
sessionId: row.sessionId,
sessionId: row.publicSessionId,
companyId: row.companyId,
ownerUserId: row.ownerUserId,
ownerUserId: row.startedByUserId,
adapterType: row.adapterType,
environmentId: row.environmentId,
leaseId: row.leaseId ?? "",
deadline: row.deadlineAt.getTime(),
state: row.state as SetupTokenSessionState,
leaseId: row.providerLeaseId ?? "",
// The setup-token flow always writes an expiry, so a null value means a
// corrupt row. Treat it as already expired, so the reaper reclaims it.
deadline: row.expiresAt ? row.expiresAt.getTime() : 0,
state: row.status as SetupTokenSessionState,
boundAt: row.boundAt ? row.boundAt.getTime() : null,
};
}
/**
* The database scope predicate. It matches the full owner scope and the session
* id, so no write ever addresses a row by the session id alone.
* The database scope predicate. It matches the company, the owner, the adapter,
* and the public session id, so no write ever addresses a row by the session id
* alone. The `company_id` and `started_by_user_id` terms hold the server-scoped
* lookup: a query never keys on the public session id alone.
*/
function setupTokenScopeMatch(identity: SetupTokenCleanupIdentity) {
return and(
eq(claudeSetupTokenSessions.sessionId, identity.sessionId),
eq(claudeSetupTokenSessions.companyId, identity.companyId),
eq(claudeSetupTokenSessions.ownerUserId, identity.ownerUserId),
eq(claudeSetupTokenSessions.adapterType, identity.adapterType as AgentAdapterType),
eq(claudeSetupTokenSessions.environmentId, identity.environmentId),
eq(adapterAuthSessions.publicSessionId, identity.sessionId),
eq(adapterAuthSessions.companyId, identity.companyId),
eq(adapterAuthSessions.startedByUserId, identity.ownerUserId),
eq(adapterAuthSessions.adapterType, identity.adapterType as AgentAdapterType),
);
}
@@ -1355,14 +1387,14 @@ export async function transitionSetupTokenSessionToStored(
identity: SetupTokenCleanupIdentity,
): Promise<boolean> {
const changed = await executor
.update(claudeSetupTokenSessions)
.set({ state: "stored", updatedAt: sql`clock_timestamp()` })
.update(adapterAuthSessions)
.set({ status: "stored", updatedAt: sql`clock_timestamp()` })
.where(
and(
setupTokenScopeMatch(identity),
inArray(claudeSetupTokenSessions.state, [...SETUP_TOKEN_STORED_PREDECESSOR_STATES]),
gt(claudeSetupTokenSessions.deadlineAt, sql`clock_timestamp()`),
isNull(claudeSetupTokenSessions.boundAt),
inArray(adapterAuthSessions.status, [...SETUP_TOKEN_STORED_PREDECESSOR_STATES]),
gt(adapterAuthSessions.expiresAt, sql`clock_timestamp()`),
isNull(adapterAuthSessions.boundAt),
),
)
.returning();
@@ -1382,66 +1414,73 @@ export function createDbSetupTokenCleanupStore(db: Db): SetupTokenCleanupStore {
return {
async record(record): Promise<void> {
await db.insert(claudeSetupTokenSessions).values({
sessionId: record.sessionId,
// The database generates `id`. The store fills `public_session_id` with the
// service session id, which the service builds from a CSPRNG at start.
await db.insert(adapterAuthSessions).values({
companyId: record.companyId,
ownerUserId: record.ownerUserId,
adapterType: record.adapterType as AgentAdapterType,
environmentId: record.environmentId,
leaseId: record.leaseId,
state: record.state,
deadlineAt: new Date(record.deadline),
adapterType: record.adapterType as AgentAdapterType,
startedByUserId: record.ownerUserId,
publicSessionId: record.sessionId,
providerLeaseId: record.leaseId,
status: record.state,
expiresAt: new Date(record.deadline),
boundAt: record.boundAt === null ? null : new Date(record.boundAt),
});
},
async markState(identity, state): Promise<void> {
// The compare-and-set predicate matches the full owner scope, so a write
// never updates a row by the session id alone.
// The compare-and-set predicate matches the company, the owner, and the
// adapter, so a write never updates a row by the session id alone.
await db
.update(claudeSetupTokenSessions)
.set({ state, updatedAt: sql`clock_timestamp()` })
.update(adapterAuthSessions)
.set({ status: state, updatedAt: sql`clock_timestamp()` })
.where(scopeMatch(identity));
},
async remove(identity): Promise<void> {
// The delete matches the full owner scope, so it never removes a row by the
// session id alone.
await db.delete(claudeSetupTokenSessions).where(scopeMatch(identity));
// The delete matches the company, the owner, and the adapter, so it never
// removes a row by the session id alone.
await db.delete(adapterAuthSessions).where(scopeMatch(identity));
},
async listReapable(now): Promise<SetupTokenCleanupRecord[]> {
// A record is reapable when its session is terminal, its deadline is past,
// or its claim is already consumed. The deadline index supports the scan.
// or its claim is already consumed. The scan filters by the setup-token
// adapter, so it never reaps a Codex device-login row on the shared table.
// The deadline index supports the scan.
const rows = await db
.select()
.from(claudeSetupTokenSessions)
.from(adapterAuthSessions)
.where(
or(
inArray(claudeSetupTokenSessions.state, [...SETUP_TOKEN_TERMINAL_STATES]),
lte(claudeSetupTokenSessions.deadlineAt, new Date(now)),
isNotNull(claudeSetupTokenSessions.boundAt),
and(
eq(adapterAuthSessions.adapterType, SETUP_TOKEN_ADAPTER_TYPE),
or(
inArray(adapterAuthSessions.status, [...SETUP_TOKEN_TERMINAL_STATES]),
lte(adapterAuthSessions.expiresAt, new Date(now)),
isNotNull(adapterAuthSessions.boundAt),
),
),
);
return rows.map(toCleanupRecord);
},
async consumeStoredClaim(identity): Promise<SetupTokenCleanupRecord | null> {
// One conditional write. The predicate carries the full owner scope, the
// session id, `state = stored`, an unexpired deadline, and an unconsumed
// marker. It checks the deadline with `clock_timestamp()`, so the database
// evaluates the current time after any row-lock wait. An expired claim
// cannot pass the deadline condition. The write sets `bound_at` one time
// and returns the row only on a valid consume.
// One conditional write. The predicate carries the company, the owner, the
// adapter, the session id, `status = stored`, an unexpired deadline, and an
// unconsumed marker. It checks the deadline with `clock_timestamp()`, so the
// database evaluates the current time after any row-lock wait. An expired
// claim cannot pass the deadline condition. The write sets `bound_at` one
// time and returns the row only on a valid consume.
const changed = await db
.update(claudeSetupTokenSessions)
.update(adapterAuthSessions)
.set({ boundAt: sql`clock_timestamp()`, updatedAt: sql`clock_timestamp()` })
.where(
and(
scopeMatch(identity),
eq(claudeSetupTokenSessions.state, "stored"),
gt(claudeSetupTokenSessions.deadlineAt, sql`clock_timestamp()`),
isNull(claudeSetupTokenSessions.boundAt),
eq(adapterAuthSessions.status, "stored"),
gt(adapterAuthSessions.expiresAt, sql`clock_timestamp()`),
isNull(adapterAuthSessions.boundAt),
),
)
.returning();
@@ -30,7 +30,6 @@ import {
const SCOPE: SetupTokenSessionScope = {
companyId: "company-1",
ownerUserId: "user-1",
targetAgentId: null,
adapterType: "claude_local",
environmentId: "env-1",
};
@@ -36,6 +36,7 @@ import {
import { secretService } from "./secrets.js";
import type { environmentService } from "./environments.js";
import type { environmentRuntimeService } from "./environment-runtime.js";
import { buildLoginLeaseAcquireArgs } from "./adapter-login-lease.js";
import type { Db } from "@paperclipai/db";
import {
createSetupTokenPtyTransport,
@@ -200,7 +201,6 @@ export function createSetupTokenSecretWriter(deps: {
companyId: scope.companyId,
ownerUserId: scope.ownerUserId,
adapterType: scope.adapterType,
environmentId: scope.environmentId,
});
if (!transitioned) {
// Zero-row result: roll back the whole transaction. The session then marks
@@ -219,9 +219,11 @@ export function createSetupTokenSecretWriter(deps: {
};
}
/** The immutable scope key that correlates one acquire with one factory call. */
/** The immutable scope key that correlates one acquire with one factory call. It
* is the company, the owner, and the adapter, so it matches the active-slot
* scope. The per-slot session cap is one, so one scope holds one live acquire. */
function scopeKey(scope: SetupTokenSessionScope): string {
return [scope.companyId, scope.ownerUserId, scope.adapterType, scope.environmentId].join("\u0000");
return [scope.companyId, scope.ownerUserId, scope.adapterType].join("\u0000");
}
/**
@@ -407,29 +409,29 @@ export function createProductionSetupTokenSandboxProvider(
return failClosed();
}
const leaseRecord = await deps.environmentRuntime.acquireRunLease({
companyId: scope.companyId,
environment,
issueId: null,
agentId: scope.targetAgentId ?? null,
// A null heartbeat run and a null execution workspace disable lease
// reuse, so the login session always runs in a fresh sandbox.
heartbeatRunId: null,
persistedExecutionWorkspace: null,
adapterType: scope.adapterType,
// Apply the active custom-image template, so the sandbox binds to the
// trusted image and runtime identity.
applyCustomImageTemplate: true,
// Re-check the environment company binding inside the lease insert
// transaction. The route guard ran earlier, so a managed reconciliation
// can bind this sandbox to another company between the guard and this
// acquire. The lease insert then rejects a foreign-company environment
// with the 403 `environment_company_mismatch` and holds no lease.
assertCompanyBinding: true,
// Bound the lease expiry to the session deadline. The runtime records
// the earlier of this deadline and the provider expiry on the lease row.
requestedExpiresAt: new Date(deadline),
});
const leaseRecord = await deps.environmentRuntime.acquireRunLease(
buildLoginLeaseAcquireArgs({
metadata: {
companyId: scope.companyId,
environment,
adapterType: scope.adapterType,
},
// The setup-token login is company-and-environment scoped. It carries
// no target agent, so the lease binds no agent.
targetAgentId: null,
// Re-check the environment company binding inside the lease insert
// transaction. The route guard ran earlier, so a managed
// reconciliation can bind this sandbox to another company between the
// guard and this acquire. The lease insert then rejects a
// foreign-company environment with the 403
// `environment_company_mismatch` and holds no lease.
assertCompanyBinding: true,
// Bound the lease expiry to the session deadline. The runtime records
// the earlier of this deadline and the provider expiry on the lease
// row.
requestedExpiresAt: new Date(deadline),
}),
);
const leaseId = leaseRecord.lease.id;
leaseRecords.set(leaseId, leaseRecord);
+55 -3
View File
@@ -1,6 +1,6 @@
import { and, desc, eq } from "drizzle-orm";
import { and, desc, eq, inArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { issueWorkProducts } from "@paperclipai/db";
import { issueWorkProducts, workspaceRuntimeServices } from "@paperclipai/db";
import type { IssueWorkProduct } from "@paperclipai/shared";
import { insertRowsInChunks } from "./batch-insert.js";
import type { ImportIssueWorkProductRow } from "./import-write-types.js";
@@ -33,6 +33,44 @@ function toIssueWorkProduct(row: IssueWorkProductRow): IssueWorkProduct {
};
}
/**
* Refresh runtime-service work products from the live runtime rows they point at
* (PAP-17572).
*
* A runtime URL is only valid for as long as the process holds that port. A
* managed restart can relocate it, which used to leave a user-facing preview link
* that answers with somebody else's service or nothing at all. The runtime row is
* the authoritative publication record, so it wins over the stored copy.
*
* Read-path only and deliberately non-destructive: a work product whose runtime
* row is gone keeps its recorded URL and is reported unhealthy rather than
* silently blanked, so the history of what was published survives.
*/
export function reconcileRuntimeServiceWorkProducts(
products: IssueWorkProduct[],
liveRuntimeServices: Array<{
id: string;
url: string | null;
status: string;
healthStatus: string;
}>,
): IssueWorkProduct[] {
if (products.length === 0) return products;
const liveById = new Map(liveRuntimeServices.map((service) => [service.id, service]));
return products.map((product) => {
if (product.type !== "runtime_service" || !product.runtimeServiceId) return product;
const live = liveById.get(product.runtimeServiceId);
if (!live) {
return product.healthStatus === "unhealthy" ? product : { ...product, healthStatus: "unhealthy" };
}
const isServing = live.status === "running" && live.healthStatus === "healthy";
const url = live.url ?? product.url;
const healthStatus: IssueWorkProduct["healthStatus"] = isServing ? "healthy" : "unhealthy";
if (product.url === url && product.healthStatus === healthStatus) return product;
return { ...product, url, healthStatus };
});
}
export function workProductService(db: Db) {
return {
listForIssue: async (issueId: string) => {
@@ -41,7 +79,21 @@ export function workProductService(db: Db) {
.from(issueWorkProducts)
.where(eq(issueWorkProducts.issueId, issueId))
.orderBy(desc(issueWorkProducts.isPrimary), desc(issueWorkProducts.updatedAt));
return rows.map(toIssueWorkProduct);
const products = rows.map(toIssueWorkProduct);
const runtimeServiceIds = products
.map((product) => (product.type === "runtime_service" ? product.runtimeServiceId : null))
.filter((value): value is string => Boolean(value));
if (runtimeServiceIds.length === 0) return products;
const liveRuntimeServices = await db
.select({
id: workspaceRuntimeServices.id,
url: workspaceRuntimeServices.url,
status: workspaceRuntimeServices.status,
healthStatus: workspaceRuntimeServices.healthStatus,
})
.from(workspaceRuntimeServices)
.where(inArray(workspaceRuntimeServices.id, [...new Set(runtimeServiceIds)]));
return reconcileRuntimeServiceWorkProducts(products, liveRuntimeServices);
},
getById: async (id: string) => {
@@ -165,14 +165,21 @@ describe("WorkspaceGitOperationScheduler", () => {
const scheduler = createWorkspaceGitOperationScheduler({ runner, defaultCacheTtlMs: 0 });
const first = scheduler.run(scanInput(workspace, "same"));
// Wait for the first request to register as the single-flight leader before
// the alias request starts. Both requests call fs.realpath concurrently, so
// without this barrier the symlink alias can resolve first and become the
// leader. That race makes the leader and joiner order non-deterministic.
await vi.waitFor(() => expect(scheduler.snapshot().inFlightCount).toBe(1));
const joined = scheduler.run(scanInput(alias, "same"));
await vi.waitFor(() => expect(scheduler.snapshot().totals.singleFlightJoins).toBe(1));
expect(calls).toBe(1);
gate.resolve();
await expect(Promise.all([first, joined])).resolves.toEqual([
expect.objectContaining({ stdout: "shared", singleFlightJoined: false }),
expect.objectContaining({ stdout: "shared", singleFlightJoined: true }),
const results = await Promise.all([first, joined]);
expect(results).toEqual([
expect.objectContaining({ stdout: "shared" }),
expect.objectContaining({ stdout: "shared" }),
]);
expect(results.map((result) => result.singleFlightJoined).sort()).toEqual([false, true]);
shouldFail = true;
await expect(scheduler.run(scanInput(workspace, "failure"))).rejects.toMatchObject({
@@ -1,5 +1,6 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
@@ -27,6 +28,27 @@ export function deriveWorktreeInstanceId(workspacePath: string): string {
return `${prefix}-${pathHash}`;
}
/**
* The instance id a seeded worktree actually runs as, read from the pointer the
* guest process itself loads.
*
* Synchronous and null-on-anything-unexpected by design: callers use it to
* decide whether they can prove workspace identity at all, and an unsafe or
* unreadable pointer must fail closed rather than fall back to a guess.
*/
export function readWorktreeInstanceId(workspacePath: string): string | null {
const envPath = path.join(path.resolve(workspacePath), ".paperclip", ".env");
let contents: string;
try {
contents = readFileSync(envPath, "utf8");
} catch {
return null;
}
const instanceId = parseEnvContents(contents).PAPERCLIP_INSTANCE_ID?.trim();
if (!instanceId || !INSTANCE_ID_RE.test(instanceId)) return null;
return instanceId;
}
export type WorktreeInstancePointer = {
envPath: string;
envContents: string;
@@ -289,9 +311,12 @@ export async function cleanupWorktreeInstanceArtifacts(input: {
return { status: "refused", instanceRoot: configuredInstanceRoot, warning };
}
const expectedInstanceRoot = input.expectedInstanceRoot
? path.resolve(input.expectedInstanceRoot)
: null;
const expectedInstanceRoot = input.expectedInstanceRoot ? path.resolve(input.expectedInstanceRoot) : null;
if (!expectedInstanceRoot) {
warning = `Refusing to remove instance directory "${configuredInstanceRoot}" because execution workspace ${input.workspaceId} has no persisted instance root.`;
await recordRefusal({ refusalReason: "persisted_instance_root_missing" });
return { status: "refused", instanceRoot: configuredInstanceRoot, warning };
}
if (expectedInstanceRoot && configuredInstanceRoot !== expectedInstanceRoot) {
warning = `Refusing to remove instance directory "${configuredInstanceRoot}" because it does not match execution workspace ${input.workspaceId}'s persisted instance root "${expectedInstanceRoot}".`;
await recordRefusal({
@@ -0,0 +1,215 @@
/**
* Issues signed workspace login handoff tickets (PAP-17572).
*
* Runs on the authenticated main control plane. Everything a ticket is bound to
* is resolved here from server-side state — the caller's authenticated identity,
* the live runtime row's published URL, and the worktree's recorded instance
* pointer — so a client cannot influence the audience, workspace, or instance a
* ticket is valid for.
*
* A ticket is only minted for a workspace that currently passes the protected
* readiness probe. Handing out a ticket for a half-restored clone would turn a
* diagnosable "clone is incomplete" state back into the opaque
* "invalid email or password" that started this incident.
*/
import { and, desc, eq } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { authUsers, instanceUserRoles, workspaceRuntimeServices } from "@paperclipai/db";
import type { WorkspaceReadiness } from "@paperclipai/shared";
import {
buildWorkspaceHandoffExchangeUrl,
issueWorkspaceHandoffTicket,
normalizeWorkspaceHandoffOrigin,
sanitizeWorkspaceHandoffRedirectPath,
WORKSPACE_HANDOFF_TICKET_TTL_SECONDS,
} from "../auth/workspace-login-handoff.js";
import { resolvePaperclipInstanceId } from "../home-paths.js";
import {
probeManagedWorkspaceReadiness,
resolveManagedWorkspaceIdentity,
} from "./managed-workspace-identity.js";
export type WorkspaceLoginHandoffActor = {
userId: string | null | undefined;
userEmail: string | null | undefined;
source: string | null | undefined;
};
export type WorkspaceLoginHandoffFailure =
| { reason: "no_board_identity" }
| { reason: "handoff_not_configured" }
| { reason: "runtime_not_running" }
| { reason: "runtime_url_unusable"; detail: string }
| { reason: "workspace_not_ready"; detail: string | null; readiness: WorkspaceReadiness | null };
export type WorkspaceLoginHandoffIssuance = {
url: string;
expiresAt: string;
/** The live runtime origin the ticket was bound to. */
origin: string;
nonce: string;
userId: string;
instanceId: string;
};
export type WorkspaceLoginHandoffResult =
| { ok: true; issuance: WorkspaceLoginHandoffIssuance }
| { ok: false; failure: WorkspaceLoginHandoffFailure };
/**
* Resolve the board user a ticket will be minted for.
*
* In `authenticated` mode this is the caller's own session identity. The
* `local_implicit` actor of a `local_trusted` instance has no auth row, so the
* instance's oldest instance admin stands in: that mode already grants
* unrestricted board access to this instance and its clones, so this widens
* nothing, and it keeps local development password-independent too.
*/
export async function resolveWorkspaceHandoffBoardIdentity(
db: Db,
actor: WorkspaceLoginHandoffActor,
): Promise<{ userId: string; email: string } | null> {
const userId = actor.userId?.trim();
const email = actor.userEmail?.trim();
if (userId && email && actor.source !== "local_implicit") {
return { userId, email };
}
if (actor.source !== "local_implicit") return null;
const admin = await db
.select({ id: authUsers.id, email: authUsers.email, createdAt: authUsers.createdAt })
.from(authUsers)
.innerJoin(
instanceUserRoles,
and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")),
)
.orderBy(authUsers.createdAt, authUsers.id)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!admin?.email) return null;
return { userId: admin.id, email: admin.email };
}
/**
* The live runtime row a workspace is currently served from.
*
* Reading the row (rather than a stored work-product URL) is what keeps the
* handoff target correct after a port change: the row is the same source the
* runtime publication path writes.
*/
export async function resolveLiveWorkspaceRuntimeOrigin(
db: Db,
input: { companyId: string; executionWorkspaceId: string },
): Promise<{ url: string; runtimeServiceId: string } | null> {
const row = await db
.select({
id: workspaceRuntimeServices.id,
url: workspaceRuntimeServices.url,
updatedAt: workspaceRuntimeServices.updatedAt,
})
.from(workspaceRuntimeServices)
.where(
and(
eq(workspaceRuntimeServices.companyId, input.companyId),
eq(workspaceRuntimeServices.executionWorkspaceId, input.executionWorkspaceId),
eq(workspaceRuntimeServices.status, "running"),
eq(workspaceRuntimeServices.healthStatus, "healthy"),
),
)
.orderBy(desc(workspaceRuntimeServices.updatedAt))
.then((rows) => rows.find((candidate) => Boolean(candidate.url)) ?? null);
if (!row?.url) return null;
return { url: row.url, runtimeServiceId: row.id };
}
export async function issueWorkspaceLoginHandoff(input: {
db: Db;
companyId: string;
executionWorkspace: { id: string; cwd: string | null };
actor: WorkspaceLoginHandoffActor;
next?: string | null;
ttlSeconds?: number;
now?: Date;
probe?: typeof probeManagedWorkspaceReadiness;
}): Promise<WorkspaceLoginHandoffResult> {
const identity = resolveManagedWorkspaceIdentity({
workspaceCwd: input.executionWorkspace.cwd,
executionWorkspaceId: input.executionWorkspace.id,
companyId: input.companyId,
});
if (!identity) return { ok: false, failure: { reason: "handoff_not_configured" } };
const boardIdentity = await resolveWorkspaceHandoffBoardIdentity(input.db, input.actor);
if (!boardIdentity) return { ok: false, failure: { reason: "no_board_identity" } };
const live = await resolveLiveWorkspaceRuntimeOrigin(input.db, {
companyId: input.companyId,
executionWorkspaceId: input.executionWorkspace.id,
});
if (!live) return { ok: false, failure: { reason: "runtime_not_running" } };
const origin = normalizeWorkspaceHandoffOrigin(live.url);
if (!origin) {
return { ok: false, failure: { reason: "runtime_url_unusable", detail: live.url } };
}
const probe = input.probe ?? probeManagedWorkspaceReadiness;
const readinessResult = await probe({
// Probe the origin the user will actually be sent to, so a stale row or a
// reassigned port is caught before a ticket exists rather than after.
healthUrl: new URL("/api/health", origin).toString(),
identity,
handoffSubject: boardIdentity,
});
if (!readinessResult.ok) {
return {
ok: false,
failure: {
reason: "workspace_not_ready",
detail: readinessResult.detail ?? readinessResult.reason,
readiness: readinessResult.readiness,
},
};
}
const { ticket, payload } = issueWorkspaceHandoffTicket({
key: identity.handoffKey,
userId: boardIdentity.userId,
email: boardIdentity.email,
executionWorkspaceId: identity.executionWorkspaceId,
companyId: identity.companyId,
instanceId: identity.instanceId,
origin,
issuerInstanceId: resolvePaperclipInstanceId(),
next: sanitizeWorkspaceHandoffRedirectPath(input.next),
ttlSeconds: input.ttlSeconds ?? WORKSPACE_HANDOFF_TICKET_TTL_SECONDS,
now: input.now,
});
return {
ok: true,
issuance: {
url: buildWorkspaceHandoffExchangeUrl({ origin, ticket }),
expiresAt: new Date(payload.exp * 1000).toISOString(),
origin,
nonce: payload.jti,
userId: boardIdentity.userId,
instanceId: identity.instanceId,
},
};
}
/** HTTP status for a failed issuance. */
export function workspaceLoginHandoffFailureStatus(failure: WorkspaceLoginHandoffFailure): number {
switch (failure.reason) {
case "no_board_identity":
return 403;
case "handoff_not_configured":
return 501;
case "runtime_not_running":
case "runtime_url_unusable":
case "workspace_not_ready":
return 409;
}
}
+43 -16
View File
@@ -14,7 +14,7 @@ type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect;
* Managed runtime control actions. Every one of these mutates the runtime rows and
* local listeners of a single execution workspace, so only one may be live at a time.
*/
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "run"]);
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "repair", "run"]);
/**
* Identity of this server process. A `running` runtime-control operation stamped with a
@@ -234,7 +234,10 @@ export interface WorkspaceOperationRecorder {
* timeout error is rethrown, so a hung provider can never leave an active operation.
*/
timeoutMs?: number | null;
run: () => Promise<{
run: (reportProgress: (input: {
metadata?: Record<string, unknown> | null;
system?: string | null;
}) => Promise<void>) => Promise<{
status?: WorkspaceOperationStatus;
exitCode?: number | null;
stdout?: string | null;
@@ -484,7 +487,7 @@ export function workspaceOperationService(db: Db) {
// Managed runtime controls get an ownership stamp so bounded recovery can tell a
// slow-but-live operation from one abandoned by a dead request or server process.
const runtimeControlAction = readRuntimeControlAction(recordInput.metadata);
const insertedMetadata = runtimeControlAction
let currentMetadata = runtimeControlAction
? {
...(recordInput.metadata ?? {}),
runtimeControlOwner: {
@@ -515,7 +518,7 @@ export function workspaceOperationService(db: Db) {
logStore: handle.store,
logRef: handle.logRef,
metadata: redactCurrentUserValue(
insertedMetadata,
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
startedAt,
@@ -532,17 +535,21 @@ export function workspaceOperationService(db: Db) {
if (runtimeControlAction) {
heartbeatTimer = setInterval(() => {
const heartbeatAt = new Date();
currentMetadata = combineMetadata(currentMetadata, {
runtimeControlOwner: {
ownerId: RUNTIME_CONTROL_OWNER_ID,
pid: process.pid,
action: runtimeControlAction,
heartbeatAt: heartbeatAt.toISOString(),
} satisfies RuntimeControlOwnerStamp,
});
void db
.update(workspaceOperations)
.set({
metadata: combineMetadata(insertedMetadata, {
runtimeControlOwner: {
ownerId: RUNTIME_CONTROL_OWNER_ID,
pid: process.pid,
action: runtimeControlAction,
heartbeatAt: heartbeatAt.toISOString(),
} satisfies RuntimeControlOwnerStamp,
}),
metadata: redactCurrentUserValue(
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
updatedAt: heartbeatAt,
})
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")))
@@ -551,9 +558,29 @@ export function workspaceOperationService(db: Db) {
heartbeatTimer.unref?.();
}
const reportProgress = async (progress: {
metadata?: Record<string, unknown> | null;
system?: string | null;
}) => {
await append("system", progress.system ?? null);
currentMetadata = combineMetadata(currentMetadata, progress.metadata);
await db
.update(workspaceOperations)
.set({
metadata: redactCurrentUserValue(
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
stdoutExcerpt: stdoutExcerpt || null,
stderrExcerpt: stderrExcerpt || null,
updatedAt: new Date(),
})
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")));
};
const timeoutMs = recordInput.timeoutMs ?? defaultRuntimeControlTimeoutMs(runtimeControlAction);
const settle = async () => {
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run();
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run(reportProgress);
const timeout = new Promise<never>((_resolve, reject) => {
timeoutTimer = setTimeout(
() => reject(new WorkspaceOperationTimeoutError(timeoutMs, runtimeControlAction)),
@@ -561,7 +588,7 @@ export function workspaceOperationService(db: Db) {
);
timeoutTimer.unref?.();
});
return await Promise.race([recordInput.run(), timeout]);
return await Promise.race([recordInput.run(reportProgress), timeout]);
};
try {
@@ -583,7 +610,7 @@ export function workspaceOperationService(db: Db) {
logSha256: finalized.sha256,
logCompressed: finalized.compressed,
metadata: redactCurrentUserValue(
combineMetadata(insertedMetadata, result.metadata),
combineMetadata(currentMetadata, result.metadata),
currentUserRedactionOptions,
) as Record<string, unknown> | null,
finishedAt,
@@ -613,7 +640,7 @@ export function workspaceOperationService(db: Db) {
...(runtimeControlAction
? {
metadata: redactCurrentUserValue(
combineMetadata(insertedMetadata, {
combineMetadata(currentMetadata, {
failureReason: error instanceof WorkspaceOperationTimeoutError
? "runtime_control_timeout"
: "runtime_control_error",
+296
View File
@@ -0,0 +1,296 @@
/**
* Protected workspace readiness (PAP-17572).
*
* `/api/health` answering 200 only proves a listener exists. This module answers
* the question the control plane and the UI actually need — "can a board user
* open this clone and see their data?" — from four independent signals: the
* isolated database, the versioned seed manifest, representative cloned rows,
* and the login-handoff configuration.
*
* Everything here is read-only and deliberately small: structural validation of
* a clone happens once, during provisioning or repair, not on every probe.
*
* `repairing` is intentionally not derivable here. A guest process cannot know
* that the control plane is running a managed repair against it; the UI layers
* that state on from the live workspace-operation row.
*/
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import { and, eq, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { authUsers, companies, companyMemberships, issues } from "@paperclipai/db";
import type {
WorkspaceReadiness,
WorkspaceReadinessState,
WorkspaceSeedReadinessState,
} from "@paperclipai/shared";
import {
resolveWorkspaceHandoffLocalCompanyId,
resolveWorkspaceHandoffLocalKey,
resolveWorkspaceHandoffLocalWorkspaceId,
} from "../auth/workspace-login-handoff.js";
import { resolvePaperclipInstanceId } from "../home-paths.js";
import { logger } from "../middleware/logger.js";
const WORKSPACE_SEED_MANIFEST_BASENAME = "seed-manifest.json";
const LEGACY_SEED_PENDING_BASENAME = "seed-pending";
const LEGACY_SEED_COMPLETE_BASENAME = "seed-complete";
type SeedManifestSummary = {
state: WorkspaceSeedReadinessState;
phase: string | null;
mode: "minimal" | "full" | null;
failurePhase: string | null;
};
/**
* Directory holding this instance's seed markers.
*
* `PAPERCLIP_CONFIG` is the authoritative pointer a seeded worktree is started
* with; the cwd fallback covers a guest launched without it.
*/
export function resolveWorkspaceSeedMarkerDir(env: NodeJS.ProcessEnv = process.env): string {
const configPath = env.PAPERCLIP_CONFIG?.trim();
if (configPath) return path.dirname(path.resolve(configPath));
return path.resolve(process.cwd(), ".paperclip");
}
function readSeedManifestSummary(markerDir: string): SeedManifestSummary {
const manifestPath = path.join(markerDir, WORKSPACE_SEED_MANIFEST_BASENAME);
if (existsSync(manifestPath)) {
let manifest: Record<string, unknown>;
try {
manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
} catch {
// A manifest we cannot parse is evidence of an interrupted write, which is
// a failure — never a silent "assume seeded".
return { state: "failed", phase: null, mode: null, failurePhase: "seed_manifest_unreadable" };
}
const rawState = typeof manifest.state === "string" ? manifest.state : null;
const state: WorkspaceSeedReadinessState =
rawState === "pending" || rawState === "running" || rawState === "verified" || rawState === "failed"
? rawState
: "unknown";
const phase = typeof manifest.phase === "string" ? manifest.phase : null;
const mode = manifest.seedMode === "minimal" || manifest.seedMode === "full" ? manifest.seedMode : null;
return {
state,
phase,
mode,
failurePhase: state === "failed" ? phase ?? "unknown" : null,
};
}
// Legacy markers predate the versioned manifest. `seed-complete`, or the
// absence of both markers on an already-running instance, means an adopted
// legacy clone whose completeness this probe cannot prove.
if (existsSync(path.join(markerDir, LEGACY_SEED_PENDING_BASENAME))) {
return { state: "pending", phase: "pending", mode: null, failurePhase: null };
}
if (existsSync(path.join(markerDir, LEGACY_SEED_COMPLETE_BASENAME))) {
return { state: "unknown", phase: "legacy_complete_marker", mode: null, failurePhase: null };
}
return { state: "absent", phase: null, mode: null, failurePhase: null };
}
/**
* Collapse the individual signals into one state.
*
* `degraded` is reserved for "the clone exists and was verified, but a signal
* regressed", which is the case an operator can fix with one bounded repair.
* A clone that never finished reports `provisioning`/`validating`/`failed`
* instead, so the UI never offers a repair for work that is still in progress.
*/
export function resolveWorkspaceReadinessState(input: {
databaseReady: boolean;
cloneDataReady: boolean;
authHandoffReady: boolean;
seedState: WorkspaceSeedReadinessState;
}): WorkspaceReadinessState {
if (input.seedState === "failed") return "failed";
if (input.seedState === "pending") return "provisioning";
if (input.seedState === "running") return "provisioning";
if (!input.databaseReady) return input.seedState === "verified" ? "degraded" : "provisioning";
if (!input.cloneDataReady || !input.authHandoffReady) {
return input.seedState === "verified" ? "degraded" : "validating";
}
// Cloned data and handoff both check out. A legacy or absent manifest is not
// proof of a verified seed, so surface it as validating rather than ready.
if (input.seedState !== "verified") return "validating";
return "ready";
}
/**
* Whether this process is a cloned workspace instance at all.
*
* The primary control plane has no seed manifest and no injected workspace
* identity, and reporting a hollow readiness block for it would invite consumers
* to treat "no clone" as "broken clone". Detection is by evidence on disk or
* injected identity, never by a branch-name or path heuristic.
*/
let managedWorkspaceInstanceCache: { markerDir: string; value: boolean; checkedAtMs: number } | null = null;
/**
* How long the marker-file answer is reused. `/api/health` is polled by the dev
* runner, the control plane's readiness probe and the UI, so three `existsSync`
* calls per request would be pure overhead on an instance whose answer is
* effectively static. Short enough that a manifest appearing mid-provision is
* picked up promptly.
*/
const MANAGED_WORKSPACE_DETECTION_TTL_MS = 5_000;
export function isManagedWorkspaceInstance(
env: NodeJS.ProcessEnv = process.env,
now: () => number = Date.now,
): boolean {
// Injected identity is authoritative and free to read, so it short-circuits
// ahead of any filesystem work.
if (resolveWorkspaceHandoffLocalKey(env)) return true;
if (resolveWorkspaceHandoffLocalWorkspaceId(env)) return true;
const markerDir = resolveWorkspaceSeedMarkerDir(env);
const cached = managedWorkspaceInstanceCache;
if (
cached
&& cached.markerDir === markerDir
&& now() - cached.checkedAtMs < MANAGED_WORKSPACE_DETECTION_TTL_MS
) {
return cached.value;
}
const value = existsSync(path.join(markerDir, WORKSPACE_SEED_MANIFEST_BASENAME))
|| existsSync(path.join(markerDir, LEGACY_SEED_PENDING_BASENAME))
|| existsSync(path.join(markerDir, LEGACY_SEED_COMPLETE_BASENAME));
managedWorkspaceInstanceCache = { markerDir, value, checkedAtMs: now() };
return value;
}
/** Test-only seam so a suite can change marker files without waiting out the TTL. */
export function resetManagedWorkspaceInstanceCacheForTests(): void {
managedWorkspaceInstanceCache = null;
}
export type WorkspaceReadinessDeps = {
db: Db | null | undefined;
env?: NodeJS.ProcessEnv;
handoffSubject?: { userId: string; email: string } | null;
};
/**
* Assemble this instance's readiness. Never throws: a probe that cannot read the
* database must report `databaseReady: false`, not fail the health endpoint.
*/
export async function resolveWorkspaceReadiness(deps: WorkspaceReadinessDeps): Promise<WorkspaceReadiness> {
const env = deps.env ?? process.env;
const seed = readSeedManifestSummary(resolveWorkspaceSeedMarkerDir(env));
const instanceId = resolvePaperclipInstanceId();
const executionWorkspaceId = resolveWorkspaceHandoffLocalWorkspaceId(env);
// The company this workspace's board represents. Both product probes below are
// scoped to it: "some company in the clone has issues" and "some user has some
// membership" can both be true while the company the operator is opening is
// missing or has no members, which is a workspace that must not report ready.
const companyId = resolveWorkspaceHandoffLocalCompanyId(env);
const handoffKeyPresent = Boolean(resolveWorkspaceHandoffLocalKey(env));
const handoffUserId = deps.handoffSubject?.userId.trim() || null;
const handoffUserEmail = deps.handoffSubject?.email.trim().toLowerCase() || null;
let databaseReady = false;
let cloneDataReady = false;
let clonedAdminPresent = false;
let probeFailurePhase: string | null = null;
if (deps.db) {
try {
await deps.db.execute(sql`SELECT 1`);
databaseReady = true;
} catch (error) {
logger.warn({ err: error }, "workspace readiness database probe failed");
probeFailurePhase = "database_unreachable";
}
if (databaseReady && companyId) {
try {
// One representative cloned company/issue pair proves the restore carried
// product rows, not just an empty migrated schema.
const clonedRows = await deps.db
.select({ companyId: companies.id })
.from(companies)
.innerJoin(issues, eq(issues.companyId, companies.id))
.where(eq(companies.id, companyId))
.limit(1)
.then((rows) => rows.length);
cloneDataReady = clonedRows > 0;
if (!cloneDataReady) probeFailurePhase ??= "clone_data_missing";
} catch (error) {
logger.warn({ err: error }, "workspace readiness clone-data probe failed");
probeFailurePhase ??= "clone_data_unreadable";
}
try {
// The handoff can only sign in a user who survived the clone with an
// active membership *in this workspace's company*, so readiness asserts
// that identity exists here rather than discovering it at click time.
// Existence, not a count: this runs on every protected health request and
// counting the whole join would scale with instance size for an answer
// that needs one row.
const eligibleUsers = await deps.db
.select({ userId: authUsers.id })
.from(authUsers)
.innerJoin(
companyMemberships,
and(
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, authUsers.id),
eq(companyMemberships.status, "active"),
eq(companyMemberships.companyId, companyId),
...(handoffUserId ? [eq(authUsers.id, handoffUserId)] : []),
...(handoffUserEmail ? [sql`lower(${authUsers.email}) = ${handoffUserEmail}`] : []),
),
)
.limit(1)
.then((rows) => rows.length);
clonedAdminPresent = eligibleUsers > 0;
if (!clonedAdminPresent) probeFailurePhase ??= "cloned_membership_missing";
} catch (error) {
logger.warn({ err: error }, "workspace readiness identity probe failed");
probeFailurePhase ??= "cloned_identity_unreadable";
}
}
} else {
probeFailurePhase = "database_not_configured";
}
const authHandoffReady = Boolean(
handoffKeyPresent
&& executionWorkspaceId
&& companyId
&& clonedAdminPresent,
);
if (!companyId) probeFailurePhase ??= "workspace_company_not_configured";
if (!executionWorkspaceId) probeFailurePhase ??= "workspace_identity_not_configured";
if (!handoffKeyPresent) probeFailurePhase ??= "auth_handoff_not_configured";
const state = resolveWorkspaceReadinessState({
databaseReady,
cloneDataReady,
authHandoffReady,
seedState: seed.state,
});
return {
state,
databaseReady,
cloneDataReady,
authHandoffReady,
authHandoffUserId: handoffUserId,
seedState: seed.state,
seedPhase: seed.phase,
seedMode: seed.mode,
instanceId,
executionWorkspaceId,
companyId,
// A seed-recorded failure phase is more specific than anything this probe
// can infer, so it wins.
failurePhase: seed.failurePhase ?? (state === "ready" ? null : probeFailurePhase),
};
}
@@ -6,7 +6,10 @@ import path from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import type { BrokerClient, BrokerListenerRequest } from "./runtime-exposure/broker-client.js";
import { diagnoseRuntimeListenerBinds } from "./runtime-exposure/loopback-listener.js";
import {
diagnoseRuntimeListenerBinds,
readListenerBindFacts,
} from "./runtime-exposure/loopback-listener.js";
import {
resetRuntimeServicesForTests,
setWorkspaceRuntimeExposureDepsForTests,
@@ -35,7 +38,9 @@ afterEach(async () => {
});
function serviceCommand() {
return `node -e 'const http=require("http");const p=Number(process.env.PORT);for(const q of [p,p+10000])http.createServer((_,r)=>{r.statusCode=200;r.end("ok")}).listen(q,"127.0.0.1");setInterval(()=>{},1000)'`;
// Answers `/api/health` the way a real Paperclip dev runtime does: managed
// publication requires semantic health, not just a 200 (PAP-17572).
return `node -e 'const http=require("http");const p=Number(process.env.PORT);for(const q of [p,p+10000])http.createServer((rq,r)=>{if(rq.url==="/api/health"){r.setHeader("content-type","application/json");r.end(JSON.stringify({status:"ok"}));return}r.statusCode=200;r.end("ok")}).listen(q,"127.0.0.1");setInterval(()=>{},1000)'`;
}
/**
@@ -70,8 +75,11 @@ const host = mode === "custom" ? (valueOf("--bind-host") ?? "127.0.0.1")
: mode === "lan" ? "0.0.0.0"
: "127.0.0.1";
const p = Number(process.env.PORT);
// Even a pre-exposure checkout answered /api/health semantically; these guests
// model bind behaviour, not health behaviour.
const health = (rq, r) => { if (rq.url === "/api/health") { r.setHeader("content-type", "application/json"); r.end(JSON.stringify({ status: "ok" })); return true; } return false; };
for (const q of [p, p + 10000]) {
http.createServer((_, r) => { r.statusCode = 200; r.end("ok"); }).listen(q, host);
http.createServer((rq, r) => { if (health(rq, r)) return; r.statusCode = 200; r.end("ok"); }).listen(q, host);
}
setInterval(() => {}, 1000);
`;
@@ -89,7 +97,8 @@ const argv = process.argv.slice(2);
const at = argv.indexOf("--bind-host");
const host = at >= 0 ? argv[at + 1] : "127.0.0.1";
const p = Number(process.env.PORT);
http.createServer((_, r) => { r.statusCode = 200; r.end("ok"); }).listen(p, host);
const health = (rq, r) => { if (rq.url === "/api/health") { r.setHeader("content-type", "application/json"); r.end(JSON.stringify({ status: "ok" })); return true; } return false; };
http.createServer((rq, r) => { if (health(rq, r)) return; r.statusCode = 200; r.end("ok"); }).listen(p, host);
// No host argument: Vite's own HMR listener lands on the wildcard.
http.createServer((_, r) => { r.statusCode = 426; r.end(); }).listen(p + 10000);
setInterval(() => {}, 1000);
@@ -99,8 +108,9 @@ setInterval(() => {}, 1000);
const ALWAYS_WILDCARD_GUEST = `
import http from "node:http";
const p = Number(process.env.PORT);
const health = (rq, r) => { if (rq.url === "/api/health") { r.setHeader("content-type", "application/json"); r.end(JSON.stringify({ status: "ok" })); return true; } return false; };
for (const q of [p, p + 10000]) {
http.createServer((_, r) => { r.statusCode = 200; r.end("ok"); }).listen(q, "0.0.0.0");
http.createServer((rq, r) => { if (health(rq, r)) return; r.statusCode = 200; r.end("ok"); }).listen(q, "0.0.0.0");
}
setInterval(() => {}, 1000);
`;
@@ -110,6 +120,10 @@ beforeAll(async () => {
await fs.writeFile(path.join(guestDir, "dev-runner.mjs"), PRE_MANAGED_EXPOSURE_GUEST);
await fs.writeFile(path.join(guestDir, "dev-runner-legacy.mjs"), ALWAYS_WILDCARD_GUEST);
await fs.writeFile(path.join(guestDir, "dev-runner-wildcard-hmr.mjs"), WILDCARD_HMR_GUEST);
await fs.writeFile(
path.join(guestDir, "dev-runner-bind-conflict.mjs"),
'process.stderr.write("local_trusted requires server.bind=loopback\\n"); process.exit(1);\n',
);
});
afterAll(async () => {
@@ -155,6 +169,9 @@ function createBroker() {
* "make exposure fixture honor occupied ports").
*/
async function isLoopbackPortFree(port: number): Promise<boolean> {
const facts = await readListenerBindFacts(port);
if (facts?.present) return false;
return await new Promise<boolean>((resolve) => {
const probe = net.createServer();
probe.unref();
@@ -461,7 +478,7 @@ describe("loopback bind is forced on the guest, not merely requested (PAP-17256)
}));
// The launched command carries the loopback bind, replacing `--bind lan`.
expect(runtime.command).toContain("--bind custom --bind-host 127.0.0.1");
expect(runtime.command).toContain("--bind loopback");
expect(runtime.command).not.toContain("--bind lan");
// And because the guest honoured it, both listeners are loopback-only, so
@@ -517,7 +534,7 @@ describe("loopback bind is forced on the guest, not merely requested (PAP-17256)
// The bare code alone is what cost PAP-17254 three diagnostic cycles.
expect(error!.message).toContain("listener_ownership_mismatch");
expect(error!.message).toContain("loopback");
expect(error!.message).toContain("--bind custom --bind-host 127.0.0.1");
expect(error!.message).toContain("--bind loopback");
}, 20_000);
it("leaves a non-Paperclip service's --bind argument alone", async () => {
@@ -534,7 +551,7 @@ describe("loopback bind is forced on the guest, not merely requested (PAP-17256)
}));
expect(runtime.command).toBe(declared);
expect(runtime.command).not.toContain("--bind custom");
expect(runtime.command).not.toContain("--bind loopback");
expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
expect(runtime.exposure?.state).toBe("ready");
}, 20_000);
@@ -554,6 +571,20 @@ describe("loopback bind is forced on the guest, not merely requested (PAP-17256)
expect(calls).toEqual([]);
expect(runtime.command).toBe(declared);
}, 20_000);
it("surfaces a deployment/bind conflict from a guest that exits during startup", async () => {
const { broker } = createBroker();
installDeps({ broker });
await expect(startRuntimeServicesForWorkspaceControl(startInput({
serviceName: "paperclip-dev",
command: guestCommand("dev-runner-bind-conflict.mjs"),
expose: LEGACY_HTTP_EXPOSE,
port: { type: "auto", envKey: "PORT" },
}))).rejects.toThrow(
/deployment\/bind conflict: local_trusted requires server\.bind=loopback.*output: local_trusted requires server\.bind=loopback/s,
);
}, 20_000);
});
describe("readiness probes loopback for an exposed runtime (PAP-17256)", () => {
@@ -28,7 +28,7 @@ const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([
]);
/** Runtime control actions that mutate the workspace and therefore need the lease. */
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart"];
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart", "repair"];
/**
* Upper bound on how long a lease survives without the owner touching it. Recovery
File diff suppressed because it is too large. Load diff
Loaded 100 of 227 files, more files were not shown because too many files have changed in this diff. Show more