test: verify sleep producer readiness and CI contracts (#18795)

Reviewed and verified at 9f87b490d1.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-11-03 17:41:15 -06:00
1 parent ce332ecec0
commit ab2d2098a2
39 files changed
+2823 -47

No files matched your search

@@ -0,0 +1,36 @@
import { mock } from 'node:test';
import { DEFAULT_ANNOUNCEMENT_FEED_URL, announcementManifestSchema } from '@paperclipai/shared';
/** Test-child transport seam: actual route/feed/cache, no DNS or HTTPS claim. */
export async function installMaintenanceAnnouncementBoundary() {
const actual = await import('../../../app.js');
const requests: string[] = [], unmatched: string[] = [];
let intercepted = false;
const manifest = announcementManifestSchema.parse({ schemaVersion: 1, announcement: {
id: 'new-projects', eyebrow: 'New', title: 'Projects', description: 'Organize your work.',
primaryAction: { kind: 'route', label: 'Open', path: '/projects' },
} });
const wrapped: typeof actual.createApp = (db, options) => {
const announcements = options.announcements;
if (!announcements?.enabled || announcements.feedUrl !== DEFAULT_ANNOUNCEMENT_FEED_URL) {
throw new Error('Configured announcement boundary requires the exact enabled default feed');
}
intercepted = true;
const fetch = async (url: URL, init: RequestInit) => {
if (url.href !== DEFAULT_ANNOUNCEMENT_FEED_URL || init.method !== 'GET') {
unmatched.push(url.href);
throw new Error('Unmatched announcement transport is forbidden');
}
requests.push(url.href);
return Response.json(manifest, { headers: { ETag: '"qualification-v1"' } });
};
const withTransport = { ...options, announcements: { ...announcements, fetch } };
return actual.createApp(db, withTransport);
};
mock.module(new URL('../../../app.ts', import.meta.url).href, { namedExports: { ...actual, createApp: wrapped } });
// Fail before server startup if the TS loader/module cache bypasses the mock.
if ((await import('../../../app.js')).createApp !== wrapped) throw new Error('Announcement module boundary did not intercept the startup import');
return { assertIntercepted() { if (!intercepted) throw new Error('Actual server bypassed the announcement boundary'); },
snapshot: () => ({ intercepted, requests: [...requests], unmatched: [...unmatched],
boundary: 'Feed transport injected through existing fetch seam; actual route/feed validation/cache/UI. DNS and HTTPS guard are not exercised in this combined case.' }) };
}
@@ -0,0 +1,54 @@
import { createHash } from 'node:crypto';
import { constants } from 'node:fs';
import { lstat, open, readdir } from 'node:fs/promises';
import path from 'node:path';
import { Transform, Writable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { createGunzip } from 'node:zlib';
/** Inspect only archives created by the normal idle owner after the quiet counters freeze.
* Stream through EOF to validate the gzip trailer; retain no SQL/credential contents. */
export async function observeIdleBackupCheckpoint(directory: string, beforeNames: string[], startedAt: string, fixtureId: string) {
if (!/^[a-f0-9-]{36}$/.test(fixtureId)) throw new Error('Invalid owned backup fixture identifier');
const checkpointMs = Date.parse(startedAt);
if (!Number.isFinite(checkpointMs) || checkpointMs <= 0) throw new Error('Invalid owned checkpoint timestamp');
const backupDir = path.join(directory, 'backups');
const marker = await lstat(path.join(backupDir, '.idle-backup-wake-required'));
if (!marker.isFile() || marker.isSymbolicLink() || marker.size === 0) throw new Error('Missing normal idle backup wake marker');
for (const failure of [path.join(directory, 'health/db-backup-to-s3.failure'), path.join(directory, 'db-backup-to-s3.failure'), path.join(backupDir, 'db-backup-to-s3.failure')]) {
try { await lstat(failure); throw new Error('Observed database backup failure marker'); }
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
}
const names = (await readdir(backupDir)).filter(name => name.endsWith('.sql.gz') && !beforeNames.includes(name)).sort();
if (!names.length) throw new Error('Normal idle owner produced no new archive');
const archives = [];
for (const name of names) {
const handle = await open(path.join(backupDir, name), constants.O_RDONLY | constants.O_NOFOLLOW);
try {
const stat = await handle.stat();
// Some filesystems report birthtime=0. The new-name filter and opened
// non-symlink file still apply; use its content mtime only in that case.
// An old positive birthtime must not be rescued by a recent modification.
const createdAtSource = stat.birthtimeMs === 0 ? 'mtime' : 'birthtime';
const createdMs = stat.birthtimeMs === 0 ? stat.mtimeMs : stat.birthtimeMs;
if (!stat.isFile() || stat.size <= 0 || !Number.isFinite(createdMs) || createdMs <= 0
|| createdMs < checkpointMs || Math.trunc(createdMs) > Date.now()) throw new Error('Archive predates owned checkpoint or is invalid');
const hash = createHash('sha256'); let sqlBytes = 0, tail = '', fixtureIdentifierFound = false;
await pipeline(handle.createReadStream({ autoClose: false }),
new Transform({ transform(chunk, _encoding, done) { hash.update(chunk); done(null, chunk); } }), createGunzip(),
new Writable({ write(chunk, _encoding, done) {
sqlBytes += chunk.length;
if (sqlBytes > 512 * 1024 * 1024) { done(new Error('Owned fixture archive exceeded bounded observation size')); return; }
const text = tail + chunk.toString('utf8'); fixtureIdentifierFound ||= text.includes(fixtureId);
tail = text.slice(-fixtureId.length); done();
} }));
if (!sqlBytes || !fixtureIdentifierFound) throw new Error('Archive does not contain the owned fixture');
archives.push({ name, bytes: stat.size, sqlBytes, sha256: hash.digest('hex'), createdAt: new Date(createdMs).toISOString(), createdAtSource, gzipVerified: true, fixtureIdentifierFound });
} finally { await handle.close(); }
}
return { owner: 'normal-idle-checkpoint', startedAt, observedAt: new Date().toISOString(), archives, wakeMarkerObserved: true, failureObserved: false };
}
export async function idleBackupArchiveNames(directory: string) {
try { return (await readdir(path.join(directory, 'backups'))).filter(name => name.endsWith('.sql.gz')); }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error; }
}
@@ -0,0 +1,82 @@
import { setTimeout as delay } from 'node:timers/promises';
import { withNativeMaintenanceBrowser } from './native-browser.js';
/** Actual Chromium visibility; never substitutes a fake document or frozen lifecycle. */
export async function observeMaintenanceBrowser(origin: string, announcementDescription?: string, tenantHeaders?: Record<string, string>) {
return withNativeMaintenanceBrowser(async (browser, context) => {
const unmatchedOrigins: string[] = [];
if (tenantHeaders) {
// Both HTTP and native WS need the simulated Cloud front door identity.
// Gate every possible foreign transport BEFORE networking; no fixture
// header can reach another origin. about:blank does not use networking.
await context.route('**/*', async route => {
if (new URL(route.request().url()).origin !== origin) { unmatchedOrigins.push(route.request().url()); await route.abort('blockedbyclient'); return; }
await route.continue();
});
await context.routeWebSocket(url => url.origin.replace(/^ws/, 'http') !== origin, route => {
unmatchedOrigins.push(route.url()); route.close({ code: 1008, reason: 'Outside owned app origin' });
});
await context.setExtraHTTPHeaders(tenantHeaders);
}
const page = await context.newPage();
const requests: Array<{ at: number; path: string }> = [];
page.on('request', request => { const url = new URL(request.url()); if (url.origin === origin) requests.push({ at: performance.now(), path: url.pathname }); });
await page.goto(origin, { waitUntil: 'networkidle' });
if (announcementDescription) await page.getByText(announcementDescription, { exact: true }).waitFor({ state: 'visible' });
await page.bringToFront();
if (await page.evaluate(() => document.visibilityState) !== 'visible') throw new Error('Browser active-use observation requires a visible page');
const beforeActive = requests.length;
await page.reload({ waitUntil: 'networkidle' });
if (!requests.slice(beforeActive).some(request => request.path.startsWith('/api/'))) throw new Error('Active browser did not make ordinary app requests');
const visibilityEvents: Array<{ state: string; rendererAt: number }> = [];
const prefix = 'paperclip-sleep-visibility:';
page.on('console', message => { if (message.text().startsWith(prefix)) visibilityEvents.push(JSON.parse(message.text().slice(prefix.length))); });
// Passive native events only; never replace document visibility properties.
await page.evaluate(prefix => document.addEventListener('visibilitychange', () => console.log(prefix + JSON.stringify({ state: document.visibilityState, rendererAt: performance.now() }))), prefix);
const visibility = () => page.evaluate(() => ({ state: document.visibilityState, focused: document.hasFocus(), rendererAt: performance.now(),
ownVisibilityOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'visibilityState')), ownHiddenOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hidden')),
ownFocusOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hasFocus')) }));
const initialVisibility = await visibility();
if (initialVisibility.state !== 'visible') throw new Error('Native browser is not visible before the actual tab switch');
const foreground = await context.newPage();
await foreground.goto('about:blank'); await foreground.bringToFront();
const native = await browser.newBrowserCDPSession();
const windowIdentity = async (target: typeof page) => {
const session = await context.newCDPSession(target);
try {
const { targetInfo } = await session.send('Target.getTargetInfo');
const { windowId } = await native.send('Browser.getWindowForTarget', { targetId: targetInfo.targetId });
return { targetId: targetInfo.targetId, windowId };
} finally { await session.detach(); }
};
const appWindow = await windowIdentity(page), foregroundWindow = await windowIdentity(foreground);
if (appWindow.windowId !== foregroundWindow.windowId) throw new Error('Background browser qualification requires an actual same-window tab switch');
const hiddenDeadline = performance.now() + 5_000;
while (await page.evaluate(() => document.visibilityState) !== 'hidden') {
if (performance.now() > hiddenDeadline) throw new Error('Browser could not establish a real background tab; visibility simulation is not qualification');
await delay(50);
}
// Let requests already admitted by the transition settle before the untouched window.
await delay(2_000);
const initialHidden = await visibility();
if (initialHidden.state !== 'hidden') throw new Error('Browser lost native hidden state during settling');
if (!visibilityEvents.some(event => event.state === 'hidden' && event.rendererAt <= initialHidden.rendererAt)) throw new Error('Native hidden event was not observed');
const hiddenRequests = requests.length, started = performance.now();
await delay(90_000);
const finalHidden = await visibility();
const hiddenRealElapsedMs = performance.now() - started;
if (hiddenRealElapsedMs < 90_000) throw new Error('Native hidden observation was shorter than 90 real seconds');
const intervalVisibilityEvents = visibilityEvents.filter(event => event.rendererAt >= initialHidden.rendererAt && event.rendererAt <= finalHidden.rendererAt);
if (finalHidden.state !== 'hidden' || intervalVisibilityEvents.some(event => event.state !== 'hidden')) throw new Error(`Browser left native hidden state: ${JSON.stringify({ initialHidden, finalHidden, intervalVisibilityEvents })}`);
if ([initialVisibility, initialHidden, finalHidden].some(state => state.ownVisibilityOverride || state.ownHiddenOverride || state.ownFocusOverride)) throw new Error('Browser visibility or focus properties were overridden');
const hidden = requests.slice(hiddenRequests);
if (hidden.some(request => request.path.startsWith('/api/'))) throw new Error(`Background tab produced tenant traffic: ${JSON.stringify(hidden)}`);
if (unmatchedOrigins.length) throw new Error(`Browser attempted non-owned transport: ${JSON.stringify(unmatchedOrigins)}`);
await page.bringToFront(); const restoredVisibility = await visibility();
if (restoredVisibility.state !== 'visible') throw new Error('Native app tab did not become visible after activation');
if (!visibilityEvents.some(event => event.state === 'visible' && event.rendererAt >= finalHidden.rendererAt)) throw new Error('Native visible restoration event was not observed');
return { activeRequests: requests.slice(beforeActive, hiddenRequests).length, hiddenRealElapsedMs,
hiddenRequests: hidden, unmatchedOrigins, visibility: 'Native Chromium default context with no Playwright focus override', initialVisibility, initialHidden, finalHidden,
intervalVisibilityEvents, visibilityEvents, restoredVisibility, appWindow, foregroundWindow, announcementRendered: Boolean(announcementDescription), authBoundary: tenantHeaders ? 'Owned Cloud front-door headers; actual tenant resolver/membership' : 'Legacy local board', closed: true };
});
}
@@ -0,0 +1,21 @@
import assert from 'node:assert/strict';
import type { Db } from '@paperclipai/db';
import { githubIngressFixture } from '../github-webhook-ingress.js';
import { chatChannelService } from '../../../services/chat-channels.js';
/** The fixture supplies the named external signed-webhook boundary; domain owners write receipts. */
export async function produceChatHistory(db: Db) {
const fixture = await githubIngressFixture(db);
const action = await fixture.insertAction();
await fixture.service.processGitHubWebhookIngress(action.id);
const finished = await fixture.readAction(action.id);
assert.equal(fixture.calls.length, 1);
assert.equal(finished.status, 'processed');
const service = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error('Unexpected chat turn'); } },
fetch: async () => { throw new Error('Unexpected provider HTTP outside signed-webhook fixture'); } });
try {
await service.configure(fixture.endpoint.id, { action: 'remove' }, 'local-board');
} finally { await service.shutdown(); }
return { companyId: fixture.companyId, actionId: action.id, endpointId: fixture.endpoint.id,
boundary: 'Existing signed webhook fixture input; actual chat ingress completion and ordinary endpoint removal preserve history' };
}
@@ -0,0 +1,48 @@
import type { createDb } from '@paperclipai/db';
import { expect } from 'vitest';
import { randomUUID } from 'node:crypto';
import { readFile, writeFile } from 'node:fs/promises';
import path from 'node:path';
import type { startMaintenanceServer } from './server-harness.js';
/** Real routes own request creation, human rejection, run cancellation and outbox retirement. */
export async function produceCompletedToolReview(server: Awaited<ReturnType<typeof startMaintenanceServer>>, db: ReturnType<typeof createDb>, companyId: string) {
await server.api('POST', `/companies/${companyId}/tools/policies`, { name: 'Review harmless qualification writes', policyType: 'require_approval', selectors: { toolName: 'mcp-remote-fixture:update_note' } });
const readyPath = path.join(server.directory, `tool-review-ready-${randomUUID()}`);
const readyProgram = `const fs=require('node:fs');const marker=${JSON.stringify(readyPath)};fs.writeFileSync(marker+'.pending',process.env.PAPERCLIP_RUN_ID,{flag:'wx'});fs.renameSync(marker+'.pending',marker);setTimeout(()=>process.exit(0),60000)`;
const agent = await server.api<{ id: string }>('POST', `/companies/${companyId}/agents`, { name: 'Owned tool review process', role: 'general', adapterType: 'process',
adapterConfig: { command: process.execPath, args: ['-e', readyProgram], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } });
const issue = await server.api<{ id: string }>('POST', `/companies/${companyId}/issues`, { title: 'Harmless tool review history', status: 'todo', assigneeAgentId: agent.id });
let runId: string | undefined;
await expect.poll(async () => {
const [run] = await db.$client`select id,status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`;
runId = run?.id;
if (run?.status !== 'running') return false;
// The process runs only after its awaited adapter.invoke write can commit
// behind the final dispatch gate's run lock. Status alone precedes setup.
try { return await readFile(readyPath, 'utf8') === run.id; }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; }
}, { timeout: 30_000 }).toBe(true);
const readiness = { runId, marker: path.basename(readyPath), observedAt: new Date().toISOString(), source: 'actual-process-run-id-marker' };
await writeFile(path.join(server.directory, 'sleep-tool-review-readiness.json'), JSON.stringify(readiness, null, 2));
const session = await server.api<{ sessionId: string; token: string }>('POST', '/tool-gateway/sessions', { companyId, agentId: agent.id, runId, issueId: issue.id });
const call = await fetch(`http://127.0.0.1:${server.port}/api/tool-gateway/tools/call`, { method: 'POST', signal: AbortSignal.timeout(15_000),
headers: { 'Content-Type': 'application/json', 'x-paperclip-tool-gateway-token': session.token },
body: JSON.stringify({ tool: 'mcp-remote-fixture:update_note', parameters: { noteId: 'qualification', body: 'Harmless denied operation' } }) });
expect(call.ok).toBe(false);
expect(JSON.stringify(await call.json())).toContain('approval_required');
const [request] = await db.$client`select id,status from tool_action_requests where issue_id=${issue.id}`;
expect(request.status).toBe('pending');
await server.api('POST', `/tool-gateway/action-requests/${request.id}/decline`, { companyId, reason: 'Qualification declines the harmless write' });
await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' });
await expect.poll(async () => {
const [run] = await db.$client`select status from heartbeat_runs where id=${runId!}`;
return ['succeeded', 'failed', 'cancelled', 'timed_out'].includes(run?.status);
}, { timeout: 75_000 }).toBe(true);
await expect.poll(async () => {
const [delivery] = await db.$client`select delivered_at is not null as delivered from tool_action_deliveries where action_request_id=${request.id}`;
return delivery?.delivered;
}, { timeout: 30_000 }).toBe(true);
await server.api('POST', `/tool-gateway/sessions/${session.sessionId}/revoke`, { companyId });
return { requestId: request.id, issueId: issue.id, runId, readiness, boundary: 'Actual request/rejection/session revocation and closed-task retirement; no provider write executed' };
}
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import { assertEarlySleepDiagnostics } from './early-sleep-diagnostics.js';
import type { startMaintenanceServer } from './server-harness.js';
type Report = { ownerId: string; draining: boolean; quiescent: boolean; expiresAt: string;
idleSleepSafety: { diagnostics?: { complete?: boolean; checks?: Array<{ reason: string; state: string }> } } };
const transientStates = new Map([['inspection.generation', 'unknown'], ['local.tracked_work', 'present'], ['plugins.runtime_drain', 'not_checked']]);
export function classifyEarlyDiagnostic(report: Report, elapsedSinceApiMs: number): 'strict' | 'transient' {
const checks = report.idleSleepSafety.diagnostics?.checks;
if (!Array.isArray(checks)) throw new Error('Missing early diagnostic checks');
// A real durable blocker fails now, even if unrelated inspection work is in flight.
const expectedStates = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]);
for (const check of checks) {
if (check.state === (expectedStates.get(check.reason) ?? 'none')) continue;
if (transientStates.get(check.reason) !== check.state) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`);
}
if (checks.some(check => transientStates.get(check.reason) === check.state)) return 'transient';
assertEarlySleepDiagnostics(report, elapsedSinceApiMs);
return 'strict';
}
/** Bounded setup-only observation, never a quiet-window poll or an uncertainty exemption. */
export async function stabilizeEarlyDiagnostics(server: Awaited<ReturnType<typeof startMaintenanceServer>>, options: {
ownerId: string; apiSucceededAt: number; read: () => Promise<Report>; persist: (observations: unknown[]) => Promise<void>;
}) {
const observations: unknown[] = [];
for (let round = 1; round <= 3; round++) {
const before = await server.settled();
observations.push({ round, kind: 'settled-before', at: new Date().toISOString(), snapshot: before }); await options.persist(observations);
assert(before.managed?.apiActivityActive, 'Actual successful managed API key activity is not held');
const reports: Report[] = []; let transient = false;
for (const position of ['initial', 'final']) {
const report = await options.read(); reports.push(report);
observations.push({ round, kind: position, at: new Date().toISOString(), report }); await options.persist(observations);
assert(report.ownerId === options.ownerId && report.draining && report.quiescent && Date.parse(report.expiresAt) > Date.now(), 'Early diagnostic lost exact live hold');
const classification = classifyEarlyDiagnostic(report, performance.now() - options.apiSucceededAt);
transient = classification === 'transient' || transient;
}
const after = await server.snapshot();
observations.push({ round, kind: 'snapshot-after', at: new Date().toISOString(), snapshot: after }); await options.persist(observations);
if (transient) continue;
assert(before.idle.active === 0 && after.idle.active === 0, 'Accepted early diagnostic has active owned work');
// Each real HTTP authentication is tracked work. Its generation changes are
// observer stimuli; each report separately proves its own scan stayed stable.
assert.deepEqual(reports[0]!.idleSleepSafety, reports[1]!.idleSleepSafety, 'Accepted early diagnostic pair differs');
assert(after.managed?.apiActivityActive, 'Real managed API activity expired during setup');
return { rounds: round, initial: reports[0]!, final: reports[1]!, observations,
boundary: 'Strict complete stable setup inventory with real API cooldown active; not sleep eligibility' };
}
throw new Error('Early inspection did not stabilize in three bounded owner-settled rounds');
}
@@ -0,0 +1,18 @@
type Check = { reason: string; state: string };
/** Early diagnostics are only a fail-fast blocker inventory, never sleep proof.
* The registered-probe aggregate includes the real 10-minute API activity hold;
* it cannot distinguish other probes, so the later full cooldown/proof is mandatory. */
export function assertEarlySleepDiagnostics(report: { idleSleepSafety?: { diagnostics?: { complete?: boolean; checks?: Check[] } } }, elapsedSinceApiMs: number) {
const diagnostics = report.idleSleepSafety?.diagnostics;
if (!Number.isFinite(elapsedSinceApiMs) || elapsedSinceApiMs < 0 || elapsedSinceApiMs >= 600_000) throw new Error('Early diagnostic requires the actual API cooldown still active');
if (diagnostics?.complete !== true || !Array.isArray(diagnostics.checks)) throw new Error('Early diagnostic coverage incomplete');
const expected = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]);
const seen = new Set<string>();
for (const check of diagnostics.checks) {
if (seen.has(check.reason) || check.state !== (expected.get(check.reason) ?? 'none')) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`);
seen.add(check.reason);
}
for (const required of [...expected.keys(), 'inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog']) {
if (!seen.has(required)) throw new Error(`Missing early sleep diagnostic: ${required}`);
}
}
@@ -0,0 +1,47 @@
import { eq } from 'drizzle-orm';
import { randomUUID } from 'node:crypto';
import { toolApplications, toolConnections, connectionGrants, type Db } from '@paperclipai/db';
import { writeConnectionCredential } from '../../../services/connection-credentials.js';
import { secretService } from '../../../services/secrets.js';
import { createOAuthCredentialStore } from '../../../services/tool-access/oauth/credential-store.js';
import { createOAuthRefreshLeases } from '../../../services/tool-access/oauth/refresh-leases.js';
import { createOAuthGrantRefresh } from '../../../services/tool-access/oauth/grant-refresh.js';
import { createGitHubGrantContinuity } from '../../../services/tool-access/github-grant-continuity.js';
import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from '../../../services/paperclip-cloud-connector.js';
/** Bootstrap the named OAuth consent boundary, then use real vault, binding and continuity owners. */
export async function prepareGitHubConfiguration(db: Db, companyId: string) {
const owner = 'local-board', now = Date.now();
const [application] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: 'GitHub qualification', type: 'mcp_http', status: 'active' }).returning();
const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: application.id, uid: randomUUID(), name: 'Retained GitHub', status: 'active', enabled: false,
transport: 'mcp_remote', authKind: 'oauth', credentialPolicy: 'per_user', credentialSource: 'paperclip_vault', transportConfig: {},
config: { url: 'https://api.github.com/mcp', sourceTemplateKey: 'github', oauth: { strategy: 'paperclip_cloud_connector', connectorProfile: 'github.code' } } }).returning();
const refs = [];
for (const [configPath, value] of [['oauth.access_token', 'qualification-access'], ['oauth.refresh_token', 'qualification-refresh']] as const) {
const saved = await writeConnectionCredential(db, { companyId, connectionName: 'GitHub', configPath, label: configPath, value, ownerUserId: owner });
refs.push({ configPath, secretId: saved.secret.id, versionSelector: 'latest' as const, required: true });
}
const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection.id, kind: 'user', subjectUserId: owner, status: 'active', credentialSecretRefs: refs,
providerTenant: { oauth: { strategy: 'paperclip_cloud_connector', accessTokenExpiresAt: new Date(now + 8 * 3_600_000).toISOString(), refreshedAt: new Date(now).toISOString() } } }).returning();
await createOAuthCredentialStore(db, secretService(db)).syncCredentialBindings(connection, refs);
return { connectionId: connection.id, grantId: grant.id };
}
export async function settleGitHubConfiguration(db: Db, input: { connectionId: string; grantId: string }) {
const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, input.connectionId)))[0];
const connection = await getConnection();
const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, input.grantId));
const config = paperclipCloudConnectorConfigFromEnv();
if (!config) throw new Error('Owned qualification connector is not configured');
const connector = createPaperclipCloudConnector({ config });
const credentials = createOAuthCredentialStore(db, secretService(db));
const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, credentials.getOAuthGrantRow);
const refresh = createOAuthGrantRefresh(db, { now: () => new Date(), store: credentials, leases, getConnectionRow: getConnection,
currentCloudConnector: () => connector, exchangeOAuthToken: async () => { throw new Error('Unexpected direct OAuth outside managed GitHub fixture'); } });
const continuity = createGitHubGrantContinuity(db, { now: () => new Date(), currentCloudConnector: () => connector,
refreshOAuthGrantCredentials: refresh, resolveOAuthGrantSecret: credentials.resolveOAuthGrantSecret });
const settled = await continuity.refreshManagedGitHubGrantAccess(connection, grant, { actorType: 'system', actorId: 'qualification' });
if (settled.providerTenant?.github?.bindingSync || settled.providerTenant?.github?.continuityRetry) throw new Error('Real GitHub binding debt did not settle');
return { connectionId: connection.id, grantId: grant.id, credentialRefs: settled.credentialSecretRefs.map(ref => ref.configPath),
boundary: 'Actual encrypted user credential/version/binding owners and GitHub authority refresh; real signed broker binding; named external GitHub HTTP only' };
}
@@ -0,0 +1,50 @@
import assert from 'node:assert/strict';
import { installMaintenanceIdleWorkCapture } from './idle-work-capture.js';
// Isolated Node/TS-loader identity probe. Never imports app/index or opens a DB.
const original = await import('../../../services/task-admission.js');
const before = process.argv[2] === 'unhandled' ? undefined : original.beginIdleTrackedWork();
const capture = await installMaintenanceIdleWorkCapture();
try {
const observed = await import('../../../services/task-admission.js');
assert.equal(observed.idleWorkSnapshot, original.idleWorkSnapshot);
if (process.argv[2] === 'unhandled') {
const reason = new Error('original unhandled tracked rejection');
let listener!: (reason: unknown, promise: Promise<unknown>) => void;
let timer: ReturnType<typeof setTimeout> | undefined;
const event = new Promise<{ reason: unknown; promise: Promise<unknown> }>((resolve, reject) => {
listener = (reason, promise) => resolve({ reason, promise });
process.once('unhandledRejection', listener);
timer = setTimeout(() => reject(new Error('Original tracked rejection did not reach unhandledRejection')), 1500);
});
try {
// Deliberately never await or attach a handler to this original promise.
const tracked = observed.trackIdleWork(Promise.reject(reason));
const unhandled = await event;
assert.equal(unhandled.promise, tracked);
assert.equal(unhandled.reason, reason);
assert.equal(original.idleWorkSnapshot().active, 0);
assert.equal(capture.snapshot().observedActive, 0);
console.log(JSON.stringify({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 }));
} finally {
if (timer) clearTimeout(timer);
process.removeListener('unhandledRejection', listener);
}
} else {
assert.equal(observed.idleWorkSnapshot().active, 1); // Existing state was not reset.
assert.equal(capture.snapshot().observedActive, 0); // The pre-install path is explicitly missing.
before!();
const finish = observed.beginIdleTrackedWork();
assert.equal(original.idleWorkSnapshot().active, 1);
finish(); finish();
assert.equal(original.idleWorkSnapshot().active, 0);
assert.equal(await observed.trackIdleWork(Promise.resolve('original-result')), 'original-result');
assert.equal(original.idleWorkSnapshot().active, 0);
const result = capture.summary();
assert(result.snapshot.missingPaths > 0);
console.log(JSON.stringify({ identity: true, originalStatePreserved: true, finalActive: original.idleWorkSnapshot().active, missingPathsExplicit: result.snapshot.missingPaths }));
}
} finally {
before?.();
capture.dispose();
}
@@ -0,0 +1,137 @@
import { mock } from 'node:test';
import { promiseHooks } from 'node:v8';
type Admission = typeof import('../../../services/task-admission.js');
type Entry = { id: number; kind: 'begin' | 'track'; sourceId: number; startedAt: number; generation: number };
type Completion = Entry & { finishedAt: number; finishedGeneration: number };
export type IdleWorkCaptureSnapshot = {
actual: { active: number; generation: number }; observedActive: number; active: Entry[]; completionTail: Completion[];
newSources: Array<{ id: number; callers: string[] }>; overflow: number; sourceOverflow: number;
missingCallsites: number; missingPaths: number; observerErrors: number; disposed: boolean;
};
/** Test-only metadata. No original work, promise, drain or counter is changed. */
export function createMaintenanceIdleWorkCapture(actual: Pick<Admission, 'beginIdleTrackedWork' | 'trackIdleWork' | 'idleWorkSnapshot'>,
controls: { now?: () => number; stack?: () => string | undefined } = {}) {
const active = new Map<number, Entry>(), sources = new Map<string, { id: number; callers: string[] }>();
const tail: Completion[] = [];
let trackedPromises = new WeakMap<Promise<unknown>, number>();
let disposed = false;
let serial = 0, emittedSourceId = 0, overflow = 0, sourceOverflow = 0, missingCallsites = 0, missingPaths = 0, observerErrors = 0;
const now = controls.now ?? Date.now;
function start(kind: Entry['kind']): number | undefined {
try {
if (disposed) return;
if (active.size >= 512) { overflow++; return; }
const callers: string[] = [];
for (const line of (controls.stack?.() ?? new Error().stack ?? '').split('\n')) {
const match = line.match(/[/\\]server[/\\](?:src|dist)[/\\]((?:(?:services|middleware|routes|modules)[/\\][A-Za-z0-9_./\\-]+|(?:app|index))\.[cm]?[jt]s):(\d+):(\d+)/);
if (!match || match[1]!.split(/[/\\]/).includes('..')) continue;
const caller = `${match[1]!.replaceAll('\\', '/')}:${match[2]}:${match[3]}`;
if (!callers.includes(caller)) callers.push(caller);
if (callers.length === 4) break;
}
if (!callers.length) missingCallsites++;
const key = JSON.stringify(callers); let source = sources.get(key);
if (!source) {
if (sources.size >= 512) { sourceOverflow++; return; }
source = { id: sources.size + 1, callers }; sources.set(key, source);
}
const id = ++serial;
active.set(id, { id, kind, sourceId: source.id, startedAt: now(), generation: actual.idleWorkSnapshot().generation });
return id;
} catch { observerErrors++; return; }
}
// V8 observes settlement without attaching a continuation or marking a rejected
// original promise handled. Unrelated promises only perform a WeakMap lookup.
const stopSettlementHook = promiseHooks.onSettled(promise => {
try {
const id = trackedPromises.get(promise);
if (id === undefined) return;
trackedPromises.delete(promise);
finish(id);
} catch { observerErrors++; }
});
function finish(id: number | undefined) {
try {
if (id === undefined) return;
const entry = active.get(id); if (!entry) return;
active.delete(id);
tail.push({ ...entry, finishedAt: now(), finishedGeneration: actual.idleWorkSnapshot().generation });
if (tail.length > 64) tail.shift();
} catch { observerErrors++; }
}
const beginIdleTrackedWork: Admission['beginIdleTrackedWork'] = () => {
const originalFinish = actual.beginIdleTrackedWork(); const id = start('begin');
return () => { try { return originalFinish(); } finally { finish(id); } };
};
const trackIdleWork: Admission['trackIdleWork'] = <T>(work: Promise<T>) => {
const original = actual.trackIdleWork(work); const id = start('track');
try { if (id !== undefined) trackedPromises.set(original, id); } catch { observerErrors++; }
return original;
};
return { beginIdleTrackedWork, trackIdleWork,
dispose() {
if (disposed) return;
disposed = true;
try { stopSettlementHook(); } finally { trackedPromises = new WeakMap(); }
},
snapshot(): IdleWorkCaptureSnapshot {
const state = actual.idleWorkSnapshot();
if (state.active !== active.size) missingPaths++;
const newSources = [...sources.values()].filter(source => source.id > emittedSourceId).map(source => ({ ...source, callers: [...source.callers] }));
emittedSourceId = sources.size;
return { actual: state, observedActive: active.size, active: [...active.values()].map(entry => ({ ...entry })), completionTail: tail.map(entry => ({ ...entry })),
newSources, overflow, sourceOverflow, missingCallsites, missingPaths, observerErrors, disposed };
},
summary() { return { sourceCatalog: [...sources.values()], snapshot: this.snapshot() }; },
};
}
export async function installMaintenanceIdleWorkCapture() {
const actual = await import('../../../services/task-admission.js');
const capture = createMaintenanceIdleWorkCapture(actual);
let moduleCapture: ReturnType<typeof mock.module> | undefined;
const disposeCapture = capture.dispose;
capture.dispose = () => { try { disposeCapture(); } finally { const installed = moduleCapture; moduleCapture = undefined; installed?.restore(); } };
try {
moduleCapture = mock.module(new URL('../../../services/task-admission.ts', import.meta.url).href, {
namedExports: { ...actual, beginIdleTrackedWork: capture.beginIdleTrackedWork, trackIdleWork: capture.trackIdleWork },
});
const observed = await import('../../../services/task-admission.js');
if (observed.beginIdleTrackedWork !== capture.beginIdleTrackedWork || observed.trackIdleWork !== capture.trackIdleWork
|| Object.keys(observed).sort().join(',') !== Object.keys(actual).sort().join(',')
|| Object.keys(actual).some(key => !['beginIdleTrackedWork', 'trackIdleWork'].includes(key)
&& observed[key as keyof Admission] !== actual[key as keyof Admission])) throw new Error('Idle work module capture did not preserve original exports/identity');
return capture;
} catch (error) {
try { capture.dispose(); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Idle work capture installation and cleanup failed'); }
throw error;
}
}
/** Reuse existing settle-loop observations. No extra poll, timer, SQL or raw stack per sample. */
export function createSettledWorkSamples(sourceCatalog: () => Array<{ id: number; callers: string[] }>) {
// Finite longest setup observation at the existing 25ms cadence, plus its
// terminal sample. Overflow still makes diagnostic completion fail closed.
const sampleLimit = Math.ceil(61_000 / 25) + 1;
const samples: Array<{ id: number; at: number; queryTotal: number; queryDelta: number; pending: number;
idle: { active: number; generation: number }; activeIds: number[] }> = [];
let previous: number | undefined, overflowSamples = 0, lastCapture: IdleWorkCaptureSnapshot | undefined;
const startedAt = Date.now();
return {
sample(snapshot: { now: number; idle: { active: number; generation: number }; workCapture?: IdleWorkCaptureSnapshot }, queryTotal: number, pending: number) {
lastCapture = snapshot.workCapture;
if (samples.length < sampleLimit) samples.push({ id: samples.length + 1, at: snapshot.now, queryTotal,
queryDelta: previous === undefined ? 0 : queryTotal - previous, pending, idle: { ...snapshot.idle }, activeIds: lastCapture?.active.map(entry => entry.id) ?? [] });
else overflowSamples++;
previous = queryTotal;
},
report() { return { startedAt, finishedAt: Date.now(), settleDeadlineMs: 30_000, sampleLimit, overflowSamples,
samples, sourceCatalog: sourceCatalog(), lastCapture }; },
assertComplete() {
if (!lastCapture || overflowSamples || lastCapture.overflow || lastCapture.sourceOverflow || lastCapture.missingCallsites
|| lastCapture.missingPaths || lastCapture.observerErrors || lastCapture.disposed) throw new Error('Passive idle work diagnostic capture incomplete');
},
};
}
@@ -0,0 +1,40 @@
import { randomUUID } from "node:crypto";
import type { Server } from "node:http";
import type { IncomingHttpSnapshot } from "../../../../../scripts/sleep-qualification/evidence.mjs";
/** Passive test-child observation of the actual app listener, including direct clients. */
export function observeMaintenanceIncomingHttp(server: Server) {
// A sole upgrade listener suppresses Node's default unsupported-upgrade close.
// The real app installs its WebSocket handlers before startServer returns.
if (server.listenerCount("request") === 0 || server.listenerCount("upgrade") === 0) {
throw new Error("Maintenance HTTP observation requires existing app request and upgrade handlers");
}
const captureId = randomUUID();
let requests = 0, installed = true, valid = true;
const count = () => {
if (requests === Number.MAX_SAFE_INTEGER) valid = false;
else requests++;
};
const detached = (event: string | symbol, listener: unknown) => {
if ((event === "request" || event === "upgrade") && listener === count) valid = false;
};
// Count before application handlers without inspecting or retaining request data.
server.prependListener("request", count);
server.prependListener("upgrade", count);
server.on("removeListener", detached);
return {
snapshot(): IncomingHttpSnapshot {
return { source: "node-http-server-events", captureId, installed,
valid: valid && installed && server.listening
&& server.listeners("request").includes(count) && server.listeners("upgrade").includes(count),
requests, observedAt: new Date().toISOString() };
},
dispose() {
if (!installed) return;
installed = false; valid = false;
server.off("removeListener", detached);
server.off("request", count);
server.off("upgrade", count);
},
};
}
@@ -0,0 +1,34 @@
import assert from 'node:assert/strict';
import { authUsers, companyMemberships, type Db } from '@paperclipai/db';
import { and, eq } from 'drizzle-orm';
import { cloudTenantPrimaryCompanyId } from '../../../services/cloud-instance.js';
import type { MaintenanceConfiguration, MaintenanceServerOptions, startMaintenanceServer } from './server-harness.js';
export const managedTenantActor = { userId: 'local-board', email: 'sleep-owner@example.test', name: 'Owned sleep operator' };
export function managedSleepConfiguration(collectorOrigin: string, serveUi: boolean): MaintenanceConfiguration {
return { automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true,
telemetryOrigin: `${collectorOrigin}/telemetry`, otelOrigin: collectorOrigin, serveUi,
managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' };
}
export function managedSleepServerOptions(configuration: MaintenanceConfiguration, jwks: string, s3Origin: string,
broker: { origin: string; variables: Record<string, string> }): MaintenanceServerOptions {
return { configuration, controlJwks: jwks, cloud: { jwks, s3Origin, standby: false }, tenantActor: managedTenantActor,
githubOrigin: broker.origin, connector: { origin: broker.origin,
instanceId: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID,
signPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY,
sealPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY } };
}
export async function provisionManagedSleepCompany(server: Awaited<ReturnType<typeof startMaintenanceServer>>, db: Db) {
// This real request invokes resolveCloudTenantActor, never a table seed or a bypassed company-create route.
const companies = await server.api<Array<{ id: string }>>('GET', '/companies');
const id = cloudTenantPrimaryCompanyId('maintenance-stack');
assert(companies.some(company => company.id === id), 'Actual managed primary company was not provisioned');
const [authority] = await db.select({ userId: authUsers.id, status: companyMemberships.status, membershipRole: companyMemberships.membershipRole })
.from(authUsers).innerJoin(companyMemberships, and(eq(companyMemberships.principalId, authUsers.id), eq(companyMemberships.principalType, 'user')))
.where(and(eq(authUsers.id, managedTenantActor.userId), eq(companyMemberships.companyId, id)));
assert.deepEqual(authority, { userId: managedTenantActor.userId, status: 'active', membershipRole: 'owner' });
await server.api('PATCH', `/companies/${id}`, { defaultResponsibleUserId: managedTenantActor.userId });
const observation = (await server.snapshot()).managed;
assert(observation?.isCloudManaged && observation.deploymentMode === 'authenticated' && observation.storageProvider === 's3', 'Actual supported managed profile not active');
return { id, authority, observation, boundary: 'Fixture Cloud front-door headers; actual resolver creates canonical company/user/member/default grants' };
}
@@ -0,0 +1,65 @@
import { spawn, type ChildProcess } from 'node:child_process';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import type { Browser, BrowserContext } from '@playwright/test';
/** A fresh owned browser, not a user's profile. Never terminate by process name. */
export async function withNativeMaintenanceBrowser<T>(run: (browser: Browser, context: BrowserContext) => Promise<T>): Promise<T> {
const { chromium } = await import('@playwright/test');
const profile = await mkdtemp(path.join(tmpdir(), 'paperclip-sleep-browser-'));
let child: ChildProcess | undefined, browser: Browser | undefined, exited: Promise<void> | undefined;
let exitObserved = false, launchError: Error | undefined, stderr = '', failure: { error: unknown } | undefined;
try {
child = spawn(chromium.executablePath(), [`--user-data-dir=${profile}`, '--remote-debugging-address=127.0.0.1', '--remote-debugging-port=0',
'--no-first-run', '--no-default-browser-check', '--disable-background-networking', '--disable-component-update', '--disable-sync', 'about:blank'], { stdio: ['ignore', 'ignore', 'pipe'] });
const ownedChild = child;
exited = new Promise<void>(resolve => {
ownedChild.once('exit', () => { exitObserved = true; resolve(); });
ownedChild.once('error', error => { launchError = error; if (!ownedChild.pid) resolve(); });
});
child.stderr?.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-16_384); });
const deadline = performance.now() + 10_000;
let port: number | undefined;
while (!port) {
if (launchError) throw launchError;
if (child.exitCode !== null || child.signalCode !== null || performance.now() > deadline) throw new Error(`Owned Chromium startup failed: ${stderr}`);
try {
const candidate = Number((await readFile(path.join(profile, 'DevToolsActivePort'), 'utf8')).split('\n')[0]);
if (Number.isInteger(candidate) && candidate > 0 && candidate < 65536) port = candidate;
} catch { /* The owned startup file is not ready yet. */ }
if (!port) await delay(50);
}
// Playwright1.62 installs focus emulation on normal pages. Measured native
// tab switches stayed visible there. The supported noDefaults/default-context
// path avoids installing that override (types.d.ts ConnectOverCDPOptions).
browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`, { noDefaults: true, timeout: 10_000 });
const context = browser.contexts()[0];
if (!context) throw new Error('Owned Chromium has no native default context');
return await run(browser, context);
} catch (error) { failure = { error }; throw error; }
finally {
const errors: unknown[] = [];
let disconnectTimer: ReturnType<typeof setTimeout> | undefined;
try {
await Promise.race([browser?.close(), new Promise<never>((_, reject) => { disconnectTimer = setTimeout(() => reject(new Error('Owned browser disconnect timed out')), 5_000); })]);
} catch (error) { errors.push(error); }
finally { if (disconnectTimer) clearTimeout(disconnectTimer); }
if (child && exited) {
const waitExit = async () => {
let timer: ReturnType<typeof setTimeout> | undefined;
try { return await Promise.race([exited!.then(() => true), new Promise<boolean>(resolve => { timer = setTimeout(() => resolve(false), 5_000); })]); }
finally { if (timer) clearTimeout(timer); }
};
try {
if (child.exitCode === null && child.signalCode === null && !launchError) child.kill('SIGTERM');
if (!(await waitExit())) { child.kill('SIGKILL'); if (!(await waitExit())) throw new Error('Owned Chromium did not exit after termination'); }
} catch (error) { errors.push(error); }
}
if (!child?.pid || exitObserved) {
try { await rm(profile, { recursive: true, force: true }); } catch (error) { errors.push(error); }
} else errors.push(new Error(`Owned Chromium exit was not confirmed; profile preserved at ${profile}`));
if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Native browser work or cleanup failed');
}
}
@@ -0,0 +1,10 @@
import { getStartupTracer, instrumentationReady, shutdownInstrumentation } from '../../../instrumentation.js';
await instrumentationReady;
const span = getStartupTracer('paperclip.sleep-qualification').startSpan('sleep-qualification-stimulus');
span.end();
// Keep the real SDK alive beyond its 60s default metric-reader interval. This
// observes unwanted exporter traffic without disabling any exporter through env.
if (process.argv.includes('--observe-exporters')) {
await new Promise(resolve => setTimeout(resolve, 90_000));
}
await shutdownInstrumentation();
@@ -0,0 +1,79 @@
import { createHash } from 'node:crypto';
import { createServer } from 'node:http';
/** Adapted from the real-SDK loopback fixture in managed-cloud-prerequisites.test.ts.
* Small owned objects only; no provider credentials/IAM/durability qualification. */
export function createMaintenanceS3Boundary() {
const objects = new Map<string, { bytes: Buffer; etag: string; contentType: string }>();
const requests: Array<{ method: string; path: string; status: number }> = [];
const errors: string[] = [];
const xml = (text: string) => text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
const server = createServer(async (req, res) => {
const url = new URL(req.url!, 'http://fixture.invalid');
const operation = { method: req.method!, path: url.pathname + url.search, status: 0 }; requests.push(operation);
const reply = (status: number, body?: string | Buffer) => { operation.status = status; res.statusCode = status; res.end(body); };
try {
if (!url.pathname.startsWith('/maintenance-test/') && url.pathname !== '/maintenance-test') throw new Error('Unknown S3 fixture bucket');
const key = decodeURIComponent(url.pathname.slice('/maintenance-test/'.length));
if (req.method === 'GET' && ['/maintenance-test', '/maintenance-test/'].includes(url.pathname) && url.searchParams.get('list-type') === '2'
&& [...url.searchParams.keys()].every(key => ['list-type', 'prefix', 'max-keys', 'x-id'].includes(key))) {
const keys = [...objects.keys()].filter(key => key.startsWith(url.searchParams.get('prefix') ?? '')).sort();
if (keys.length > Math.min(1000, Number(url.searchParams.get('max-keys') ?? 1000))) throw new Error('Listing exceeds bounded single-page fixture');
res.setHeader('content-type', 'application/xml');
reply(200, `<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IsTruncated>false</IsTruncated>${keys.map(key => `<Contents><Key>${xml(key)}</Key><Size>${objects.get(key)!.bytes.length}</Size></Contents>`).join('')}</ListBucketResult>`); return;
}
if (!key || [...url.searchParams.keys()].some(key => key !== 'x-id')) throw new Error('Unknown S3 fixture operation/query');
if (req.method === 'PUT') {
const chunks: Buffer[] = []; let length = 0;
for await (const chunk of req) { length += chunk.length; if (length > 17 * 1024 * 1024) throw new Error('S3 fixture object too large'); chunks.push(Buffer.from(chunk)); }
let bytes = Buffer.concat(chunks);
if (req.headers['content-encoding'] === 'aws-chunked') bytes = decodeMaintenanceAwsChunks(bytes);
else if (req.headers['content-encoding']) throw new Error('Unknown S3 fixture content framing');
const decodedLength = req.headers['x-amz-decoded-content-length'];
if (decodedLength !== undefined && Number(decodedLength) !== bytes.length) throw new Error('S3 decoded length mismatch');
const previous = objects.get(key);
if ((req.headers['if-none-match'] !== undefined && req.headers['if-none-match'] !== '*')
|| (req.headers['if-match'] !== undefined && typeof req.headers['if-match'] !== 'string')) throw new Error('Unknown conditional S3 request');
if ((req.headers['if-none-match'] === '*' && previous) || (req.headers['if-match'] && previous?.etag !== req.headers['if-match'])) {
res.setHeader('content-type', 'application/xml'); reply(412, '<Error><Code>PreconditionFailed</Code></Error>'); return;
}
const total = [...objects.values()].reduce((sum, object) => sum + object.bytes.length, 0) - (previous?.bytes.length ?? 0) + bytes.length;
if (bytes.length > 16 * 1024 * 1024 || total > 256 * 1024 * 1024 || objects.size >= 2048) throw new Error('S3 fixture storage bound exceeded');
const object = { bytes, etag: `"${createHash('md5').update(bytes).digest('hex')}"`, contentType: String(req.headers['content-type'] ?? 'application/octet-stream') };
objects.set(key, object); res.setHeader('ETag', object.etag); reply(200); return;
}
if (req.method === 'DELETE') { objects.delete(key); reply(204); return; }
if (req.method !== 'GET' && req.method !== 'HEAD') throw new Error('Unknown S3 fixture method');
const object = objects.get(key);
if (!object) { res.setHeader('content-type', 'application/xml'); reply(404, req.method === 'GET' ? '<Error><Code>NoSuchKey</Code></Error>' : undefined); return; }
if (req.headers.range) throw new Error('Range is outside this small-object fixture contract');
res.setHeader('ETag', object.etag); res.setHeader('content-type', object.contentType); res.setHeader('content-length', object.bytes.length);
reply(200, req.method === 'GET' ? object.bytes : undefined);
} catch (error) {
errors.push(`${req.method} ${url.pathname + url.search}: ${error instanceof Error ? error.message : String(error)}`);
if (!res.headersSent) { res.setHeader('content-type', 'application/xml'); reply(501, '<Error><Code>NotImplemented</Code></Error>'); } else res.destroy();
}
});
return { server, snapshot: () => ({ boundary: 'Actual AWS SDK HTTP; bounded in-memory S3 protocol fixture, no IAM/SigV4/provider durability proof', requests: requests.map(request => ({ ...request })), errors: [...errors],
objects: [...objects].map(([key, object]) => ({ key, bytes: object.bytes.length, sha256: createHash('sha256').update(object.bytes).digest('hex') })) }) };
}
/** Decode SDK checksum framing without pretending to validate provider signatures. */
export function decodeMaintenanceAwsChunks(wire: Buffer) {
const chunks: Buffer[] = []; let offset = 0;
while (offset < wire.length) {
const end = wire.indexOf('\r\n', offset); if (end < 0) throw new Error('Malformed AWS chunk header');
const header = wire.subarray(offset, end).toString('ascii');
if (!/^[0-9a-f]+$/i.test(header)) throw new Error('Unknown AWS chunk extension');
const length = Number.parseInt(header, 16); offset = end + 2;
if (!Number.isSafeInteger(length) || length < 0 || offset + length + 2 > wire.length) throw new Error('Truncated AWS chunk');
if (length === 0) {
const trailers = wire.subarray(offset).toString('ascii');
if (!/^(?:x-amz-checksum-[a-z0-9]+:[A-Za-z0-9+/=]+\r\n)*\r\n$/.test(trailers)) throw new Error('Unknown AWS checksum trailer');
return Buffer.concat(chunks);
}
chunks.push(wire.subarray(offset, offset + length)); offset += length;
if (wire.subarray(offset, offset + 2).toString('ascii') !== '\r\n') throw new Error('Malformed AWS chunk terminator'); offset += 2;
}
throw new Error('Missing AWS final chunk');
}
@@ -2,6 +2,7 @@ import { channel } from "node:diagnostics_channel";
import { mock } from "node:test";
import type { Db } from "@paperclipai/db";
import type { Socket } from "node:net";
import { observeMaintenanceIncomingHttp } from "./incoming-http-observation.js";
const realTimeout = globalThis.setTimeout, realInterval = globalThis.setInterval, clearRealInterval = globalThis.clearInterval, realNow = Date.now;
const fakeClock = process.env.PAPERCLIP_MAINTENANCE_FAKE_CLOCK === "true";
@@ -40,14 +41,42 @@ const observeRequest = () => { outgoingHttpRequests++; };
channel("undici:request:create").subscribe(observeRequest);
channel("http.client.request.start").subscribe(observeRequest);
let workCapture: Awaited<ReturnType<typeof import('./idle-work-capture.js')['installMaintenanceIdleWorkCapture']>> | undefined;
let databaseCapture: ReturnType<typeof import('@paperclipai/db/response-capture')['databaseResponseCaptureFromEnv']>;
let incomingHttp: ReturnType<typeof observeMaintenanceIncomingHttp> | undefined;
try {
// Must precede the announcement wrapper's first import of the real app graph.
if (process.env.PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE === '1') {
workCapture = await (await import('./idle-work-capture.js')).installMaintenanceIdleWorkCapture();
}
const announcement = process.env.PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE === '1'
? await (await import('./announcement-boundary.js')).installMaintenanceAnnouncementBoundary() : undefined;
const { startServer } = await import("../../../index.js");
const { idleWorkSnapshot, startTaskDrain, stopTaskDrain } = await import("../../../services/task-admission.js");
const { getStartupRecoveryState } = await import("../../../startup-recovery-state.js");
const server = await startServer();
if (workCapture) databaseCapture = (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv();
const managedObservation = process.env.PAPERCLIP_MAINTENANCE_MANAGED_TENANT === '1' ? await (async () => {
const { loadConfig } = await import('../../../config.js');
const { isCloudManagedInstance } = await import('../../../services/cloud-instance.js');
const { idleApiActivity } = await import('../../../services/idle-api-activity.js');
const config = loadConfig();
return () => ({ isCloudManaged: isCloudManagedInstance(), deploymentMode: config.deploymentMode,
storageProvider: config.storageProvider, apiActivityActive: idleApiActivity.active() });
})() : undefined;
const app = server.server.listeners("request")[0] as unknown as { locals: { paperclipDb: Db } };
try { incomingHttp = observeMaintenanceIncomingHttp(server.server); }
catch (error) {
try { await server.shutdown(); }
catch (cleanupError) { throw new AggregateError([error, cleanupError], "Incoming HTTP observation and owned startup cleanup failed"); }
throw error;
}
async function shutdownServer() {
try { await server.shutdown(); } finally { incomingHttp?.dispose(); }
}
try { announcement?.assertIntercepted(); } catch (error) { await shutdownServer(); throw error; }
process.send?.({ kind: "ready", port: server.listenPort });
process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } }) => {
process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } }) => {
try {
if (message.command === "snapshot" || message.command === "advance" || message.command === "drain" || message.command === "resume" || message.command === "freeze" || message.command === "realtime") {
if (message.command === "freeze" && bootClock) { clearRealInterval(bootClock); bootClock = realInterval(() => mock.timers.tick(0), 10); }
@@ -59,7 +88,23 @@ try {
if (message.command === "drain") startTaskDrain({ purpose: "idle" });
if (message.command === "resume") stopTaskDrain();
if (fakeClock) mock.timers.tick(0);
process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), now: Date.now() });
process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), incomingHttp: incomingHttp?.snapshot(), now: Date.now(), announcement: announcement?.snapshot(), managed: managedObservation?.(), workCapture: workCapture?.snapshot() });
} else if (message.command === "prepareGitHub" || message.command === "settleGitHub") {
const { prepareGitHubConfiguration, settleGitHubConfiguration } = await import("./github-configuration-producer.js");
const input = message.github;
if (!input) throw new Error("Missing GitHub fixture identity");
const result = message.command === "prepareGitHub"
? await prepareGitHubConfiguration(app.locals.paperclipDb, input.companyId!)
: await settleGitHubConfiguration(app.locals.paperclipDb, { connectionId: input.connectionId!, grantId: input.grantId! });
process.send?.({ id: message.id, kind: "githubConfiguration", result });
} else if (message.command === "produceChatHistory") {
const { produceChatHistory } = await import("./chat-history-producer.js");
process.send?.({ id: message.id, kind: "chatHistory", result: await produceChatHistory(app.locals.paperclipDb) });
} else if (message.command === "trace") {
const { getStartupTracer } = await import("../../../instrumentation.js");
const span = getStartupTracer("paperclip.sleep-qualification").startSpan("sleep-qualification-stimulus");
span.end();
process.send?.({ id: message.id, kind: "traceEnded" });
} else if (message.command === "createOrphanLease") {
if (!message.orphanLease) throw new Error("Missing orphan lease fixture");
const { environmentLeases } = await import("@paperclipai/db");
@@ -83,7 +128,20 @@ try {
process.send?.({ id: message.id, kind: "published" });
} else if (message.command === "shutdown") {
if (fakeClock) runClock();
await server.shutdown();
if (workCapture) {
const failures: unknown[] = [];
let before: ReturnType<typeof workCapture.summary> | undefined;
try { before = workCapture.summary(); } catch (error) { failures.push(error); }
try { await shutdownServer(); } catch (error) { failures.unshift(error); }
// A passive flush failure must not replace the original shutdown error.
try { databaseCapture?.flush(); } catch (error) { failures.push(error); }
try {
await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-cleanup.json'), JSON.stringify({ before, after: workCapture.summary() }, null, 2));
} catch (error) { failures.push(error); }
finally { try { workCapture.dispose(); } catch (error) { failures.push(error); } }
if (failures.length > 1) throw new AggregateError(failures, 'Owned shutdown and passive evidence failed');
if (failures.length) throw failures[0];
} else await shutdownServer();
if (bootClock) { clearRealInterval(bootClock); bootClock = undefined; }
process.send?.({ id: message.id, kind: "stopped", outgoingHttpRequests });
if (fakeClock) mock.timers.reset();
@@ -93,6 +151,16 @@ try {
} catch (error) { process.send?.({ id: message.id, kind: "error", error: error instanceof Error ? error.stack : String(error) }); }
});
} catch (error) {
incomingHttp?.dispose();
if (workCapture) {
try {
// Existing bounded DB capture performs no SQL while flushing metadata.
databaseCapture ??= (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv();
databaseCapture?.flush();
await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-startup-failure.json'), JSON.stringify(workCapture.summary(), null, 2));
} catch { console.warn('Passive startup evidence could not be persisted; original error preserved'); }
finally { try { workCapture.dispose(); } catch { console.warn('Passive capture cleanup failed; original startup error preserved'); } }
}
if (bootClock) clearRealInterval(bootClock);
process.send?.({ kind: "error", error: error instanceof Error ? error.stack : String(error) });
process.exitCode = 1;
@@ -1,3 +1,5 @@
import type { IdleWorkCaptureSnapshot } from './idle-work-capture.js';
import type { IncomingHttpSnapshot } from '../../../../../scripts/sleep-qualification/evidence.mjs';
import { fork } from "node:child_process";
import type { Server as HttpServer } from "node:http";
import { randomBytes, randomUUID } from "node:crypto";
@@ -42,9 +44,54 @@ export async function withMaintenanceProvider<T>(provider: HttpServer, run: (ori
}
}
type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number };
type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number;
incomingHttp: IncomingHttpSnapshot;
managed?: { isCloudManaged: boolean; deploymentMode: string; storageProvider: string; apiActivityActive: boolean };
workCapture?: IdleWorkCaptureSnapshot;
announcement?: { intercepted: boolean; requests: string[]; unmatched: string[]; boundary: string } };
export async function startMaintenanceServer(databaseUrl: string, options: { artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string }; cloud?: { jwks: string; s3Origin: string; standby: boolean } } = {}) {
export interface MaintenanceConfiguration {
automaticBackups?: { intervalMinutes: number; idleCheckpoint: boolean };
announcements?: boolean;
telemetryOrigin?: string;
otelOrigin?: string;
serveUi?: boolean;
managedInstance?: boolean;
deploymentMode?: "local_trusted" | "authenticated";
storageProvider?: "local_disk" | "s3";
}
export interface MaintenanceServerOptions {
artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string;
connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string; instanceId?: string };
cloud?: { jwks: string; s3Origin: string; standby: boolean };
configuration?: MaintenanceConfiguration;
controlJwks?: string;
captureIdleWork?: boolean;
tenantActor?: { userId: string; email: string; name: string };
}
export async function startMaintenanceServer(databaseUrl: string, options: MaintenanceServerOptions = {}) {
const configuration = options.configuration;
if (options.captureIdleWork && options.clock) throw new Error("Passive setup diagnostics require the real clock");
if (options.tenantActor && (!options.cloud || !configuration?.managedInstance || configuration.deploymentMode !== 'authenticated' || configuration.storageProvider !== 's3')) {
throw new Error('Managed tenant fixture requires the supported authenticated S3 profile');
}
const tenantHeaders: Record<string, string> = options.tenantActor ? {
'x-paperclip-cloud-tenant-token': 'maintenance-tenant-token', 'x-paperclip-cloud-stack-id': 'maintenance-stack',
'x-paperclip-cloud-stack-role': 'owner', 'x-paperclip-cloud-user-id': options.tenantActor.userId,
'x-paperclip-cloud-user-email': options.tenantActor.email, 'x-paperclip-cloud-user-name': options.tenantActor.name,
'x-paperclip-cloud-paperclip-company-name': 'Configured sleep acceptance',
} : {};
for (const origin of [configuration?.telemetryOrigin, configuration?.otelOrigin]) {
if (!origin) continue;
const endpoint = new URL(origin);
if (endpoint.protocol !== "http:" || endpoint.hostname !== "127.0.0.1" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) {
throw new Error("Maintenance collectors must be explicit loopback HTTP endpoints");
}
}
if (configuration?.automaticBackups && (!Number.isInteger(configuration.automaticBackups.intervalMinutes) || configuration.automaticBackups.intervalMinutes < 1)) {
throw new Error("Maintenance backup interval must be a positive integer");
}
if (configuration && options.clock) throw new Error("Configured acceptance requires the real clock");
const artifacts = await createMaintenanceArtifacts(options.artifactRoot);
const { directory, retainedDirectory } = artifacts;
const reservation = createServer();
@@ -72,43 +119,57 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art
const configPath = path.join(directory, "config.json");
await writeFile(configPath, JSON.stringify({
$meta: { version: 1, updatedAt: new Date().toISOString(), source: "onboard" },
database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: false, dir: path.join(directory, "backups") } },
database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: Boolean(configuration?.automaticBackups), intervalMinutes: configuration?.automaticBackups?.intervalMinutes ?? 60, dir: path.join(directory, "backups") } },
logging: { mode: "file", logDir: path.join(directory, "logs") },
server: { deploymentMode: "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: false },
server: { deploymentMode: options.tenantActor ? "authenticated" : "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: configuration?.serveUi ?? false },
storage: options.cloud ? { provider: "s3", s3: { bucket: "maintenance-test", region: "us-east-1", endpoint: options.cloud.s3Origin, forcePathStyle: true } }
: { provider: "local_disk", localDisk: { baseDir: path.join(directory, "storage") } },
secrets: { provider: "local_encrypted", strictMode: false, localEncrypted: { keyFilePath: path.join(directory, "master.key") } },
telemetry: { enabled: false }, updates: { checkEnabled: false },
...(options.tenantActor ? { auth: { baseUrlMode: "explicit", publicBaseUrl: `http://127.0.0.1:${address.port}`, disableSignUp: true } } : {}),
telemetry: { enabled: Boolean(configuration?.telemetryOrigin) }, updates: { checkEnabled: false },
}), { mode: 0o600 });
await new Promise<void>(resolve => reservation.close(() => resolve()));
const captureStartedAt = options.captureIdleWork ? Date.now() : undefined;
const captureSources = options.captureIdleWork ? new Map<number, string[]>() : undefined;
let settledDiagnosticId = 0;
const child = fork(fileURLToPath(new URL("./server-child.ts", import.meta.url)), [], {
cwd: fileURLToPath(new URL("../../../../..", import.meta.url)),
execArgv: ["--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))],
execArgv: [...((configuration?.announcements || options.captureIdleWork) ? ['--experimental-test-module-mocks'] : []), "--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))],
env: {
...(captureStartedAt === undefined ? {} : {
PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE: '1',
PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW: `${new Date(captureStartedAt).toISOString()}/${new Date(captureStartedAt + 60_000).toISOString()}`,
}),
PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: "production",
PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test",
PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"),
DATABASE_URL: proxy.url, DATABASE_POOL_MAX: "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test",
DATABASE_URL: proxy.url, DATABASE_POOL_MAX: configuration ? "4" : "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test",
PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(),
PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(),
CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false",
PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_UI_DEV_MIDDLEWARE: "false",
CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: String(configuration?.announcements ?? false),
...(configuration?.announcements ? { PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE: '1' } : {}),
PAPERCLIP_DB_BACKUP_ENABLED: String(Boolean(configuration?.automaticBackups)), PAPERCLIP_UI_DEV_MIDDLEWARE: "false",
...(configuration?.automaticBackups ? { PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED: configuration.automaticBackups.idleCheckpoint ? "1" : "0" } : {}),
...(configuration?.telemetryOrigin ? { PAPERCLIP_TELEMETRY_ENDPOINT: configuration.telemetryOrigin } : {}),
...(configuration?.otelOrigin ? { OTEL_EXPORTER_OTLP_ENDPOINT: configuration.otelOrigin, OTEL_EXPORTER_OTLP_PROTOCOL: "http/json" } : {}),
HEARTBEAT_SCHEDULER_ENABLED: String(options.enabled ?? true),
PAPERCLIP_MAINTENANCE_FAKE_CLOCK: String(options.clock ?? false),
...(options.suppressed ? { PAPERCLIP_IN_WORKTREE: "true" } : {}),
...(options.githubOrigin ? { PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN: options.githubOrigin } : {}),
...(options.connector ? {
PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test",
PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: options.connector.instanceId ?? "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test",
PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY: options.connector.signPrivateKey,
PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY: options.connector.sealPrivateKey,
PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: options.connector.origin,
} : {}),
...(options.controlJwks ? { PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.controlJwks } : {}),
...(options.cloud ? {
PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } }),
PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"),
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "maintenance-tenant-token", PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack",
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.cloud.jwks, PAPERCLIP_CLOUD_WARM_STANDBY: options.cloud.standby ? "1" : "0",
PAPERCLIP_PUBLIC_URL: "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: "pool-maintenance.staging.paperclip.app",
PAPERCLIP_PUBLIC_URL: options.tenantActor ? `http://127.0.0.1:${address.port}` : "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: options.tenantActor ? "127.0.0.1" : "pool-maintenance.staging.paperclip.app",
...(options.tenantActor ? { PAPERCLIP_MAINTENANCE_MANAGED_TENANT: "1" } : {}),
AWS_ACCESS_KEY_ID: "maintenance-fixture", AWS_SECRET_ACCESS_KEY: "maintenance-fixture", AWS_EC2_METADATA_DISABLED: "true",
} : {}),
PAPERCLIP_MIGRATION_PROMPT: "never", PAPERCLIP_MIGRATION_AUTO_APPLY: "true", PAPERCLIP_MANAGED_RUNTIME_HTTPS: "off",
@@ -139,12 +200,18 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art
for (const request of requests.values()) request.reject(error);
requests.clear();
});
async function request<T>(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } } = {}): Promise<T> {
async function request<T>(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } } = {}): Promise<T> {
const id = ++requestId;
const result = new Promise<T>((resolve, reject) => requests.set(id, { resolve, reject }));
child.send({ id, command, ...payload });
try { return await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); }
finally { requests.delete(id); }
try {
const value = await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`);
if (captureSources) {
const capture = (value as { workCapture?: IdleWorkCaptureSnapshot }).workCapture;
for (const source of capture?.newSources ?? []) captureSources.set(source.id, source.callers);
}
return value;
} finally { requests.delete(id); }
}
stopChild = async () => {
try {
@@ -165,7 +232,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art
if (options.clock) await request("realtime");
try {
const response = await fetch(`http://127.0.0.1:${port}/api${route}`, { method, signal: AbortSignal.timeout(30_000),
headers: { "Content-Type": "application/json", ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
headers: { "Content-Type": "application/json", ...(Object.keys(headers ?? {}).some(key => ["authorization", "x-paperclip-cloud-control"].includes(key.toLowerCase())) ? {} : tenantHeaders), ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
const value = await response.json();
if (!response.ok) throw new Error(`Maintenance API ${method} ${route}: ${response.status} ${JSON.stringify(value)}`);
return value as T;
@@ -182,6 +249,16 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art
finally { if (options.clock) await request("freeze"); }
},
publishBackup: (publication: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }) => request("publishBackup", { publication }),
async prepareGitHub(companyId: string) {
const result = await request<{ result: { connectionId: string; grantId: string } }>("prepareGitHub", { github: { companyId } });
return result.result;
},
async settleGitHub(input: { connectionId: string; grantId: string }) {
return request("settleGitHub", { github: input });
},
tenantHeaders,
produceChatHistory: () => request("produceChatHistory"),
trace: () => request("trace"),
snapshot: () => request<Snapshot>("snapshot"),
drain: () => request<Snapshot>("drain"), resume: () => request<Snapshot>("resume"),
realtime: () => request<Snapshot>("realtime"),
@@ -197,17 +274,38 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art
},
freeze: () => request<Snapshot>("freeze"),
get output() { return output; },
async settled(quietMs = 500, deferredStartupHolds = 0) {
async settled(quietMs = 500, deferredStartupHolds = 0, settleOptions: { managedSetupRecovery?: true } = {}) {
// A successful managed health check self-notifies, then reconciles its
// future deadline on the next existing 30s pass. Observe both passes +
// 500ms quiet + 500ms margin without changing worker timing/admission.
const settleTimeoutMs = settleOptions.managedSetupRecovery ? 61_000 : 30_000;
const diagnostic = options.captureIdleWork ? (await import('./idle-work-capture.js')).createSettledWorkSamples(
() => [...captureSources!].map(([id, callers]) => ({ id, callers }))) : undefined;
const diagnosticId = diagnostic ? ++settledDiagnosticId : undefined;
let failure: { error: unknown } | undefined;
let previous = -1, quietSince = Date.now();
for (const deadline = Date.now() + 30_000; Date.now() < deadline;) {
try {
for (const deadline = Date.now() + settleTimeoutMs; Date.now() < deadline;) {
const snapshot = await request<Snapshot>("snapshot");
diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending);
if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed");
if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now();
previous = proxy.queries.length;
if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) {
diagnostic?.assertComplete(); return snapshot;
}
await delay(25);
}
const snapshot = await request<Snapshot>("snapshot");
if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed");
if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now();
previous = proxy.queries.length;
if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) return snapshot;
await delay(25);
diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending);
throw new Error(`Maintenance did not settle: ${JSON.stringify({ timeoutMs: settleTimeoutMs, quietMs, queries: proxy.queries.length, pending: proxy.pending, snapshot })}\n${output.slice(-12_000)}`);
} catch (error) { failure = { error }; throw error; }
finally {
if (diagnostic) {
try { await writeFile(path.join(directory, `idle-work-settled-samples-${diagnosticId}.json`), JSON.stringify({ ...diagnostic.report(), settleDeadlineMs: settleTimeoutMs }, null, 2)); }
catch (error) { if (failure) throw new AggregateError([failure.error, error], 'Original settling and passive evidence failed'); throw error; }
}
}
throw new Error(`Maintenance did not settle: ${JSON.stringify({ queries: proxy.queries.length, pending: proxy.pending, snapshot: await request("snapshot") })}\n${output.slice(-12_000)}`);
},
};
} catch (error) {
@@ -0,0 +1,30 @@
import { afterEach, expect, it, vi } from 'vitest';
const sdk = vi.hoisted(() => ({ options: [] as Array<Record<string, unknown>>, start: vi.fn(), shutdown: vi.fn(async () => {}) }));
vi.mock('../peer-version-check.js', () => ({ checkExactPeerVersions: () => ({ ok: true }) }));
vi.mock('@opentelemetry/sdk-node', () => ({ NodeSDK: class {
constructor(options: Record<string, unknown>) { sdk.options.push(options); }
start = sdk.start;
shutdown = sdk.shutdown;
} }));
vi.mock('@opentelemetry/auto-instrumentations-node', () => ({ getNodeAutoInstrumentations: () => [] }));
vi.mock('@opentelemetry/resources', () => ({ resourceFromAttributes: (attributes: unknown) => attributes }));
vi.mock('@opentelemetry/semantic-conventions', () => ({ ATTR_SERVICE_NAME: 'service.name', ATTR_SERVICE_VERSION: 'service.version' }));
vi.mock('@opentelemetry/exporter-trace-otlp-http', () => ({ OTLPTraceExporter: class {} }));
afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); });
it('configures current plural SDK signal options so an opted-in trace exporter cannot enable default metrics or logs', async () => {
vi.stubEnv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector.invalid:4318');
vi.stubEnv('OTEL_EXPORTER_OTLP_PROTOCOL', 'http/json');
vi.spyOn(process, 'once').mockReturnValue(process);
vi.resetModules();
const instrumentation = await import('../instrumentation.js');
await instrumentation.instrumentationReady;
expect(sdk.options).toHaveLength(1);
expect(sdk.options[0]).toMatchObject({ metricReaders: [], logRecordProcessors: [], traceExporter: expect.any(Object) });
expect(sdk.options[0]).not.toHaveProperty('metricReader');
expect(sdk.options[0]).not.toHaveProperty('logRecordProcessor');
expect(sdk.start).toHaveBeenCalledOnce();
await instrumentation.shutdownInstrumentation();
expect(sdk.shutdown).toHaveBeenCalledOnce();
});
@@ -125,3 +125,11 @@ describe("owned maintenance providers", () => {
}
});
});
describe("configured acceptance boundaries", () => {
it("rejects remote collectors and fake-clock configuration before opening PostgreSQL", async () => {
await expect(startMaintenanceServer("not-a-database-url", { configuration: { telemetryOrigin: "https://telemetry.example.test" } })).rejects.toThrow("loopback");
await expect(startMaintenanceServer("not-a-database-url", { clock: true, configuration: { announcements: true } })).rejects.toThrow("real clock");
await expect(startMaintenanceServer("not-a-database-url", { configuration: { automaticBackups: { intervalMinutes: 0, idleCheckpoint: true } } })).rejects.toThrow("positive integer");
});
});
@@ -0,0 +1,61 @@
import { expect, it, vi } from 'vitest';
import { mkdtemp, mkdir, open, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { gzipSync } from 'node:zlib';
import { observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js';
it('observes complete owned gzip bytes without retaining SQL and refuses missing, truncated, foreign, historical or failed checkpoints', async () => {
const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-observer-'));
const id = '12345678-1234-1234-1234-123456789012';
const startedAt = new Date(Date.now() - 1000).toISOString();
const backupDir = path.join(directory, 'backups'); await mkdir(backupDir);
try {
const observe = () => observeIdleBackupCheckpoint(directory, [], startedAt, id);
await expect(observe()).rejects.toThrow();
await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker');
await expect(observe()).rejects.toThrow('no new archive');
const archive = path.join(backupDir, 'owned.sql.gz'), bytes = gzipSync(`COPY fixture;\n${id}\n`);
await writeFile(archive, bytes);
const receipt = await observe();
expect(receipt.archives[0]).toMatchObject({ bytes: bytes.length, gzipVerified: true, fixtureIdentifierFound: true });
expect(receipt.archives[0].sha256).toMatch(/^[a-f0-9]{64}$/);
expect(JSON.stringify(receipt)).not.toContain(id);
await expect(observeIdleBackupCheckpoint(directory, ['owned.sql.gz'], startedAt, id)).rejects.toThrow('no new archive');
await expect(observeIdleBackupCheckpoint(directory, [], new Date(Date.now() + 1000).toISOString(), id)).rejects.toThrow('predates');
await writeFile(archive, bytes.subarray(0, bytes.length - 4)); await expect(observe()).rejects.toThrow();
await writeFile(archive, gzipSync('foreign fixture')); await expect(observe()).rejects.toThrow('owned fixture');
await writeFile(archive, bytes); await writeFile(path.join(backupDir, 'db-backup-to-s3.failure'), 'failed');
await expect(observe()).rejects.toThrow('failure marker');
} finally { await rm(directory, { recursive: true, force: true }); }
});
it('uses opened-file mtime when birthtime is zero and still rejects old or malformed provenance', async () => {
const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-no-birthtime-'));
const id = '12345678-1234-1234-1234-123456789012', startedAt = new Date(Date.now() - 1000).toISOString();
const backupDir = path.join(directory, 'backups'); await mkdir(backupDir);
const archive = path.join(backupDir, 'owned.sql.gz');
await writeFile(archive, gzipSync(`COPY fixture;\n${id}\n`));
await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker');
const handle = await open(archive, 'r'), prototype = Object.getPrototypeOf(handle), originalStat = prototype.stat;
await handle.close();
let override: { birthtimeMs: number; mtimeMs?: number } = { birthtimeMs: 0 };
const spy = vi.spyOn(prototype, 'stat').mockImplementation(async function (this: unknown, ...args: unknown[]) {
return Object.assign(await originalStat.apply(this, args), override);
});
try {
const observed = await observeIdleBackupCheckpoint(directory, [], startedAt, id);
expect(observed.archives[0]).toMatchObject({ createdAtSource: 'mtime', gzipVerified: true, fixtureIdentifierFound: true });
expect(Date.parse(observed.archives[0]!.createdAt)).toBeGreaterThanOrEqual(Date.parse(startedAt));
for (const invalid of [
{ birthtimeMs: 0, mtimeMs: Date.parse(startedAt) - 1 }, { birthtimeMs: 0, mtimeMs: 0 },
{ birthtimeMs: 0, mtimeMs: NaN }, { birthtimeMs: NaN }, { birthtimeMs: -1 },
{ birthtimeMs: 0, mtimeMs: Date.now() + 60_000 },
{ birthtimeMs: Date.parse(startedAt) - 1, mtimeMs: Date.now() },
]) {
override = invalid;
await expect(observeIdleBackupCheckpoint(directory, [], startedAt, id)).rejects.toThrow('predates');
}
await expect(observeIdleBackupCheckpoint(directory, [], 'invalid', id)).rejects.toThrow('checkpoint timestamp');
} finally { spy.mockRestore(); await rm(directory, { recursive: true, force: true }); }
});
@@ -0,0 +1,312 @@
import { createHash, generateKeyPairSync, randomUUID, sign } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { pathToFileURL } from 'node:url';
import { createServer } from 'node:http';
import { writeFile } from 'node:fs/promises';
import path from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { beforeAll, describe, expect, it } from 'vitest';
import { createDb } from '@paperclipai/db';
import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js';
import { configuredSleepFeatures, finalizeLocalSleepReceipt, observeConfiguredSignals, observeIncomingHttpWindow, observedNoIncomingHttp, sleepConfigurationDigest } from '../../../scripts/sleep-qualification/evidence.mjs';
import { idleBackupArchiveNames, observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js';
import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js';
import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js';
import { managedSleepConfiguration, managedSleepServerOptions, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js';
import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js';
import { produceCompletedToolReview } from './helpers/maintenance/configuration-producers.js';
import { observeMaintenanceBrowser } from './helpers/maintenance/browser-observation.js';
import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js';
import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js';
const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL');
if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !databaseUrl) throw new Error('Configured acceptance requires an explicit dedicated PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL; embedded allocation is forbidden');
const cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA;
if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !/^[a-f0-9]{40}$/.test(cloudSha ?? '')) throw new Error('Configured acceptance requires the exact PAPERCLIP_SLEEP_CLOUD_SHA');
const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT;
if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !cloudCheckout) throw new Error('Configured acceptance requires a reviewed PAPERCLIP_SLEEP_CLOUD_CHECKOUT with its built qualification broker adapter');
const enabled = process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && Boolean(databaseUrl);
const quietMs = 90_000, apiCooldownMs = 600_000;
if (enabled && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Configured qualification requires retained raw evidence');
describe.skipIf(!enabled)('configured assembled sleep qualification (explicit owned external PostgreSQL only)', () => {
beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000);
it('retains normal credentials, completed work and a future routine through a real cooldown and quiet window', async () => {
const phase = (name: string) => console.info(JSON.stringify({ qualificationPhase: name, at: new Date().toISOString() }));
phase('starting-owned-fixtures');
const collectorRequests: Array<{ path: string; body: unknown }> = [];
const collector = createServer(async (req, res) => {
const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk));
collectorRequests.push({ path: req.url!, body: JSON.parse(Buffer.concat(chunks).toString('utf8')) });
res.setHeader('Content-Type', 'application/json'); res.end('{}');
});
if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Qualification Cloud checkout does not match the exact requested SHA');
const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href);
const broker = await startQualificationBroker();
const s3 = createMaintenanceS3Boundary();
try {
await withMaintenanceProvider(s3.server, async s3Origin => {
await withMaintenanceProvider(collector, async origin => {
const pair = generateKeyPairSync('ed25519');
const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'sleep-qualification', alg: 'EdDSA', use: 'sig' };
const configuration = managedSleepConfiguration(origin, process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1');
// Independent intended logical contract; collector ports are observed separately.
const expected = { target: 'owned-local-assembled', appSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), cloudSha: cloudSha!,
configDigest: createHash('sha256').update(JSON.stringify({ automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true,
telemetry: true, otel: { enabled: true, protocol: 'http/json' }, serveUi: process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1',
managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' })).digest('hex'),
collectorOrigins: { telemetry: `${origin}/telemetry`, otel: origin } };
expect(sleepConfigurationDigest(configuration)).toBe(expected.configDigest);
const startedAt = new Date().toISOString();
const server = await startMaintenanceServer(databaseUrl!, managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker));
phase('actual-server-ready');
let fixture = createDb(databaseUrl!, { maxConnections: 1 });
const control = (action: string, idleCapabilities?: unknown) => {
const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url');
const now = Math.floor(Date.now() / 1000);
const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid });
const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) });
return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` };
};
let failure: { error: unknown } | undefined;
try {
await writeFile(path.join(server.directory, 'sleep-managed-startup.json'), JSON.stringify(await server.snapshot(), null, 2));
const company = await provisionManagedSleepCompany(server, fixture);
expect(await server.api('GET', '/announcements/current')).toMatchObject({ id: 'new-projects' });
const announcement = (await server.snapshot()).announcement;
expect(announcement?.intercepted).toBe(true);
expect(announcement?.requests).toEqual(['https://pages.paperclip.ing/announcements/v1/current.json']);
expect(announcement?.unmatched).toEqual([]);
const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Owned process worker', role: 'general', adapterType: 'process',
adapterConfig: { command: process.execPath, args: ['-e', "console.log('sleep-acceptance-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } });
const chatHistory = await server.produceChatHistory();
const toolHistory = await produceCompletedToolReview(server, fixture, company.id);
const github = await server.prepareGitHub(company.id);
await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true });
const githubContinuity = await server.settleGitHub(github);
await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1);
await broker.enqueue();
await broker.deliver(`http://127.0.0.1:${server.port}`, () => control('connector-events:wake')[CLOUD_CONTROL_HEADER]);
// The normally enabled connection owner leases and acknowledges the real sealed event.
await expect.poll(async () => {
const [delivery] = await fixture.$client`select status from connection_event_deliveries where provider_delivery_id='qualification-delivery'`;
return delivery?.status;
}, { timeout: 30_000 }).toBe('processed');
expect(await broker.pending()).toEqual([]);
const brokerBinding = await broker.binding();
expect(brokerBinding?.bindings).toHaveLength(1);
const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Retained qualification credential' });
const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'Owned completed history', status: 'todo', assigneeAgentId: agent.id });
await expect.poll(async () => {
const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`;
return run?.status;
}, { timeout: 30_000 }).toBe('succeeded');
await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' });
const routine = await server.api<{ id: string }>('POST', `/companies/${company.id}/routines`, { title: 'Harmless future obligation', assigneeAgentId: agent.id });
await server.api('POST', `/routines/${routine.id}/triggers`, { kind: 'schedule', cronExpression: (() => { const next = new Date(Date.now() + 24 * 60 * 60_000); return `0 ${next.getUTCHours()} ${next.getUTCDate()} ${next.getUTCMonth() + 1} *`; })(), timezone: 'UTC' });
await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` });
const lastAuthenticatedApiAt = performance.now();
await writeFile(path.join(server.directory, 'sleep-setup-receipt.json'), JSON.stringify({ ...expected, configuration, company,
agentId: agent.id, keyId: key.id, announcement, chatHistory, toolHistory, githubContinuity, brokerBinding,
brokerBoundary: broker.boundary, s3: s3.snapshot(), snapshot: await server.snapshot(),
boundary: 'Actual managed startup/tenant provisioning/producers; Cloud front door, GitHub HTTP and S3 service are owned simulated transports' }, null, 2));
expect(s3.snapshot().errors).toEqual([]);
let browser;
try {
browser = process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1' ? await observeMaintenanceBrowser(`http://127.0.0.1:${server.port}`, 'Organize your work.', server.tenantHeaders) : undefined;
await writeFile(path.join(server.directory, 'sleep-browser-receipt.json'), JSON.stringify({ browser, observedAt: new Date().toISOString() }, null, 2));
} catch (error) {
await writeFile(path.join(server.directory, 'sleep-browser-error.json'), JSON.stringify({ observedAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) }, null, 2));
throw error;
}
// Setup-only owner settling never substitutes for the later full quiet window.
await server.settled(500, 0, { managedSetupRecovery: true });
const earlyDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start'));
try {
const earlyCapabilities = { incomingWake: await broker.capability(earlyDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' };
const earlyRead = () => server.api<any>('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:read', earlyCapabilities));
const early = await stabilizeEarlyDiagnostics(server, { ownerId: earlyDrain.ownerId, apiSucceededAt: lastAuthenticatedApiAt, read: earlyRead,
persist: observations => writeFile(path.join(server.directory, 'sleep-early-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) });
await writeFile(path.join(server.directory, 'sleep-early-diagnostics.json'), JSON.stringify(early, null, 2));
} finally { await server.api('DELETE', `/instance/task-drain?ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:stop')); }
// Anchor the full cooldown to a genuine successful opaque-key request.
await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` });
const lastApiActivity = performance.now();
await fixture.$client.end({ timeout: 1 });
await server.trace();
phase('first-real-ten-minute-api-cooldown');
// The real authenticated-use cooldown is part of ordinary behavior.
// No fake clock, production timing override, tenant health poll or SQL observer.
await delay(apiCooldownMs + 100);
await server.settled();
const activeSignals = observeConfiguredSignals(collectorRequests);
expect(activeSignals.qualificationSpans).toBeGreaterThan(0);
expect(activeSignals.telemetryRequests).toBeGreaterThan(0);
const before = await server.snapshot(), queries = server.proxy.queries.length, transport = server.proxy.transport;
const wallStart = performance.now(), quietStartedAt = new Date().toISOString();
phase('first-real-ninety-second-quiet-window');
await delay(quietMs);
const elapsedMs = performance.now() - wallStart, quietFinishedAt = new Date().toISOString();
const after = await server.snapshot(), finalTransport = server.proxy.transport;
const quiet = { startedAt: quietStartedAt, finishedAt: quietFinishedAt, elapsedMs, clock: 'real',
sqlQueries: server.proxy.queries.length - queries,
connectionsOpened: finalTransport.connectionsOpened - transport.connectionsOpened,
connectionsRemaining: finalTransport.connectionsOpened - finalTransport.connectionsClosed,
nonDatabaseRequests: after.outgoingHttpRequests - before.outgoingHttpRequests,
nonDatabaseBytes: after.network.nonDatabaseBytesWritten - before.network.nonDatabaseBytesWritten,
observerRequests: observeIncomingHttpWindow(before.incomingHttp, after.incomingHttp) };
await writeFile(path.join(server.directory, 'sleep-first-frozen-quiet.json'), JSON.stringify({ quiet, announcementBefore: before.announcement, announcementAfter: after.announcement }, null, 2));
const signalObservation = observeConfiguredSignals(collectorRequests);
await writeFile(path.join(server.directory, 'sleep-first-signal-observation.json'), JSON.stringify(signalObservation, null, 2));
expect(signalObservation.qualificationSpans).toBeGreaterThan(0);
expect(signalObservation.telemetryRequests).toBeGreaterThan(0);
expect(signalObservation.metricRequests).toBe(0); expect(signalObservation.logRequests).toBe(0);
expect(signalObservation.malformedTraceRequests).toBe(0); expect(signalObservation.unexpectedRequests).toBe(0);
expect(quiet.elapsedMs).toBeGreaterThanOrEqual(quietMs);
expect(quiet.sqlQueries).toBe(0);
expect(quiet.connectionsOpened).toBe(0);
expect(quiet.connectionsRemaining).toBe(0);
expect(quiet.nonDatabaseRequests).toBe(0);
expect(quiet.nonDatabaseBytes).toBe(0);
// The installed listener must have observed the actual setup HTTP traffic.
expect(quiet.observerRequests.before.requests).toBeGreaterThan(0);
expect(observedNoIncomingHttp(quiet.observerRequests, quiet)).toBe(true);
expect(after.announcement).toEqual(before.announcement);
const backupNamesBefore = await idleBackupArchiveNames(server.directory);
const checkpointStartedAt = new Date().toISOString();
const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start'));
const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' };
const read = () => server.api<any>('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities));
const initial = await read();
await writeFile(path.join(server.directory, 'sleep-initial-safety.json'), JSON.stringify(initial, null, 2));
const final = await read();
phase('first-owned-eligibility-reports-read');
// Persist raw refusal evidence before any assertion can terminate the case.
await writeFile(path.join(server.directory, 'sleep-initial-final-safety.json'), JSON.stringify({ initial, final }, null, 2));
if (initial.idleSleepSafety.backgroundWork !== 'none' || final.idleSleepSafety.backgroundWork !== 'none') {
const diagnostic = await server.api<any>('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities));
await writeFile(path.join(server.directory, 'sleep-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2));
}
const { parseApiWakeAdmission, authorizeApiWakeAdmission } = await import('../services/idle-api-wake-admission.js');
const assertIncomingProof = (report: any, incomingWake: any, ownerId: string) => {
expect(report.idleSleepSafety.incomingWake).toEqual(incomingWake);
const admission = parseApiWakeAdmission(report.idleSleepSafety.apiWakeAdmission);
expect(admission).toMatchObject({ stackId: 'maintenance-stack', ownerId,
capabilityGeneration: incomingWake.generation, challenge: incomingWake.apiKeyProof.challenge });
expect(authorizeApiWakeAdmission(admission, { stackId: 'maintenance-stack', token: key.token, now: Date.now() })).toBeDefined();
};
assertIncomingProof(initial, capabilities.incomingWake, drain.ownerId);
assertIncomingProof(final, capabilities.incomingWake, drain.ownerId);
expect(final.idleSleepSafety.apiWakeAdmission).toEqual(initial.idleSleepSafety.apiWakeAdmission);
const proof = final.idleSleepSafety;
const backupCheckpoint = await observeIdleBackupCheckpoint(server.directory, backupNamesBefore, checkpointStartedAt, company.id);
const observations = { version: 1, ...expected, startedAt,
configuration, browser, s3: s3.snapshot(), announcement: after.announcement, chatHistory, toolHistory, githubContinuity, brokerBinding, brokerBoundary: broker.boundary, backupCheckpoint,
signalObservation, features: configuredSleepFeatures({ announcement: after.announcement, signalObservation, backupCheckpoint }),
quiet,
eligibility: { observedAt: new Date().toISOString(), backgroundWork: proof.backgroundWork, ownedHold: final.ownerId === drain.ownerId, apiCooldownElapsedMs: performance.now() - lastApiActivity,
unchangedGeneration: /^[a-f0-9]{64}$/.test(proof.deadlineProof?.generation ?? '') && initial.idleSleepSafety.deadlineProof?.generation === proof.deadlineProof?.generation }, initial, final,
live: { databaseSuspension: 'INCONCLUSIVE', hostSuspension: 'INCONCLUSIVE', backupPersistence: 'INCONCLUSIVE', reason: 'Local actual server with simulated Cloud signer; no deployed provider target authorized' },
coverage: { announcements: 'Actual route, validated feed/cache and UI with one named injected provider transport fetch; DNS/HTTPS guard covered separately, not exercised here.', browser: browser ? 'Actual Chromium foreground/background measured, then closed before server quiet window' : 'Closed; set PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE=1 with built UI and headed Chromium for background/active evidence', broker: 'Retained enabled connection, actual encrypted OAuth vault, actual continuity/binding owners and signed/sealed production broker delivery; host capability from actual ConnectorStore', history: 'Normal task/credential/routine, actual tool review rejection/outbox retirement, actual signed chat ingress completion and endpoint archival' } };
await writeFile(path.join(server.directory, 'sleep-configuration-observations.json'), JSON.stringify(observations, null, 2));
const { receipt, report } = finalizeLocalSleepReceipt(observations, expected);
await writeFile(path.join(server.directory, 'sleep-configuration-receipt.json'), JSON.stringify(receipt, null, 2));
await writeFile(path.join(server.directory, 'sleep-configuration-evaluation.json'), JSON.stringify(report, null, 2));
expect(proof.version).toBe(3); expect(proof.backgroundWork).toBe('none');
expect(proof.deadlineProof).toMatchObject({ catalog: 'domain_deadlines_v1', generation: expect.stringMatching(/^[a-f0-9]{64}$/) });
fixture = createDb(databaseUrl!, { maxConnections: 1 });
const [retained] = await fixture.$client`select (select count(*)::int from agent_api_keys where id=${key.id}) as keys, (select count(*)::int from issues where id=${issue.id}) as issues, (select count(*)::int from routines where id=${routine.id}) as routines, (select enabled from tool_connections where id=${github.connectionId}) as github_enabled`;
expect(retained).toEqual({ keys: 1, issues: 1, routines: 1, github_enabled: true });
await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop'));
const nextIssue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'First ordinary producer after quiet', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` });
const reactivationApiAt = performance.now();
await expect.poll(async () => {
const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${nextIssue.id} order by created_at desc limit 1`;
return run?.status;
}, { timeout: 30_000 }).toBe('succeeded');
await server.api('PATCH', `/issues/${nextIssue.id}`, { status: 'done' });
const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${nextIssue.id}`;
expect(executions.count).toBe(1);
await fixture.$client.end({ timeout: 1 });
phase('reactivation-real-ten-minute-api-cooldown');
await delay(apiCooldownMs + 100);
await server.settled();
const resumed = await server.snapshot(), resumedQueries = server.proxy.queries.length, resumedTransport = server.proxy.transport;
const resumedStart = performance.now(), resumedStartedAt = new Date().toISOString();
phase('reactivation-real-ninety-second-quiet-window');
await delay(quietMs);
const resumedElapsedMs = performance.now() - resumedStart, resumedFinishedAt = new Date().toISOString();
const resumedEnd = await server.snapshot(), resumedFinalTransport = server.proxy.transport;
// Freeze every measurement before signed drain/proof requests do SQL.
const resumedQuiet = { startedAt: resumedStartedAt, finishedAt: resumedFinishedAt, elapsedMs: resumedElapsedMs, clock: 'real',
sqlQueries: server.proxy.queries.length - resumedQueries,
connectionsOpened: resumedFinalTransport.connectionsOpened - resumedTransport.connectionsOpened,
connectionsRemaining: resumedFinalTransport.connectionsOpened - resumedFinalTransport.connectionsClosed,
nonDatabaseRequests: resumedEnd.outgoingHttpRequests - resumed.outgoingHttpRequests,
nonDatabaseBytes: resumedEnd.network.nonDatabaseBytesWritten - resumed.network.nonDatabaseBytesWritten,
observerRequests: observeIncomingHttpWindow(resumed.incomingHttp, resumedEnd.incomingHttp) };
await writeFile(path.join(server.directory, 'sleep-resumed-frozen-quiet.json'), JSON.stringify({ quiet: resumedQuiet, announcementBefore: resumed.announcement, announcementAfter: resumedEnd.announcement }, null, 2));
const resumedSignals = observeConfiguredSignals(collectorRequests);
await writeFile(path.join(server.directory, 'sleep-resumed-signal-observation.json'), JSON.stringify(resumedSignals, null, 2));
expect(resumedSignals.qualificationSpans).toBeGreaterThan(0);
expect(resumedSignals.telemetryRequests).toBeGreaterThan(0);
expect(resumedSignals.metricRequests).toBe(0); expect(resumedSignals.logRequests).toBe(0);
expect(resumedSignals.malformedTraceRequests).toBe(0); expect(resumedSignals.unexpectedRequests).toBe(0);
expect(resumedQuiet.elapsedMs).toBeGreaterThanOrEqual(quietMs);
expect(resumedQuiet.sqlQueries).toBe(0);
expect(resumedQuiet.connectionsOpened).toBe(0);
expect(resumedQuiet.connectionsRemaining).toBe(0);
expect(resumedQuiet.nonDatabaseRequests).toBe(0);
expect(resumedQuiet.nonDatabaseBytes).toBe(0);
expect(resumedQuiet.observerRequests.before.requests).toBeGreaterThan(0);
expect(observedNoIncomingHttp(resumedQuiet.observerRequests, resumedQuiet)).toBe(true);
expect(resumedEnd.announcement).toEqual(resumed.announcement);
expect(resumedEnd.announcement?.unmatched).toEqual([]);
const resumedDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start'));
const resumedCapabilities = { incomingWake: await broker.capability(resumedDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' };
const readResumed = () => server.api<any>('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities));
const resumedInitial = await readResumed();
await writeFile(path.join(server.directory, 'sleep-resumed-initial-safety.json'), JSON.stringify(resumedInitial, null, 2));
const resumedProof = await readResumed();
await writeFile(path.join(server.directory, 'sleep-resumed-initial-final-safety.json'), JSON.stringify({ initial: resumedInitial, final: resumedProof }, null, 2));
if (resumedInitial.idleSleepSafety.backgroundWork !== 'none' || resumedProof.idleSleepSafety.backgroundWork !== 'none') {
const diagnostic = await server.api<any>('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities));
await writeFile(path.join(server.directory, 'sleep-resumed-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2));
}
assertIncomingProof(resumedInitial, resumedCapabilities.incomingWake, resumedDrain.ownerId);
assertIncomingProof(resumedProof, resumedCapabilities.incomingWake, resumedDrain.ownerId);
expect(resumedProof.idleSleepSafety.apiWakeAdmission).toEqual(resumedInitial.idleSleepSafety.apiWakeAdmission);
expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toMatch(/^[a-f0-9]{64}$/);
expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toBe(resumedInitial.idleSleepSafety.deadlineProof?.generation);
expect(resumedProof.idleSleepSafety.backgroundWork).toBe('none');
await writeFile(path.join(server.directory, 'sleep-local-reactivation-receipt.json'), JSON.stringify({
version: 1, actualProducer: 'POST company issues with process assignee using the retained opaque key after owned drain release', executions: executions.count,
realApiCooldownElapsedMs: performance.now() - reactivationApiAt,
quiet: resumedQuiet,
signalObservation: resumedSignals,
realElapsedMs: resumedQuiet.elapsedMs, sqlQueries: resumedQuiet.sqlQueries,
networkRequests: resumedQuiet.nonDatabaseRequests,
initial: resumedInitial,
eligibility: resumedProof.idleSleepSafety,
boundary: 'Actual local application reactivation, real cooldown and fresh eligibility; no provider suspension or deployed inbound-wake claim',
}, null, 2));
const s3Evidence = s3.snapshot();
await writeFile(path.join(server.directory, 'sleep-s3-boundary.json'), JSON.stringify(s3Evidence, null, 2));
expect(s3Evidence.errors).toEqual([]);
phase('configured-qualification-receipts-complete');
} catch (error) { failure = { error }; throw error; }
finally {
const errors: unknown[] = [];
try { await writeFile(path.join(server.directory, 'sleep-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); }
try { await server.stop(); } catch (error) { errors.push(error); }
try { await fixture.$client.end(); } catch (error) { errors.push(error); }
// Shutdown can flush storage; capture and validate only after child exit.
try { await writeFile(path.join(server.directory, 'sleep-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); }
if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`));
if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Configured work or owned cleanup failed');
}
});
});
} finally { await broker.stop(); }
}, 1_800_000);
});
@@ -0,0 +1,15 @@
import { expect, it } from 'vitest';
import { assertEarlySleepDiagnostics } from './helpers/maintenance/early-sleep-diagnostics.js';
const fixture = () => ({ idleSleepSafety: { diagnostics: { complete: true, checks: [
{ reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' },
{ reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog', 'issues'].map(reason => ({ reason, state: 'none' })),
] } } });
it('retains only explicit early cooldown/diagnostic-backup reasons and refuses every unexpected or unknown blocker', () => {
expect(() => assertEarlySleepDiagnostics(fixture(), 90_000)).not.toThrow();
for (const [reason, state] of [['issues', 'present'], ['deadlines.catalog', 'unknown'], ['inspection.hold', 'not_checked'], ['local.registered_probes', 'unknown']]) {
const f = fixture(); f.idleSleepSafety.diagnostics.checks.find(check => check.reason === reason)!.state = state;
expect(() => assertEarlySleepDiagnostics(f, 90_000)).toThrow('Unexpected');
}
const missing = fixture(); missing.idleSleepSafety.diagnostics.checks.shift(); expect(() => assertEarlySleepDiagnostics(missing, 90_000)).toThrow('Missing');
for (const duration of [NaN, Infinity, -1, 600_000]) expect(() => assertEarlySleepDiagnostics(fixture(), duration)).toThrow('cooldown');
});
@@ -0,0 +1,90 @@
import { afterEach, expect, it } from 'vitest';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { fileURLToPath } from 'node:url';
import { createMaintenanceIdleWorkCapture as createCapture, createSettledWorkSamples } from './helpers/maintenance/idle-work-capture.js';
const captures: ReturnType<typeof createCapture>[] = [];
function createMaintenanceIdleWorkCapture(...args: Parameters<typeof createCapture>) {
const capture = createCapture(...args); captures.push(capture); return capture;
}
afterEach(() => { for (const capture of captures.splice(0)) capture.dispose(); });
const stack = () => 'Error\n at begin (/checkout/server/src/services/owned-work.ts:12:3)\n at owner (/checkout/server/src/index.ts:100:5)';
function original() {
let active = 0, generation = 0, finishes = 0, tracked: Promise<unknown> | undefined;
const beginIdleTrackedWork = () => {
active++; generation++; let done = false;
return () => { finishes++; if (!done) { done = true; active--; generation++; } };
};
const trackIdleWork = <T>(work: Promise<T>) => {
const finish = beginIdleTrackedWork(); const result = work.finally(finish); tracked = result; return result;
};
return { beginIdleTrackedWork, trackIdleWork, idleWorkSnapshot: () => ({ active, generation }),
finishes: () => finishes, lastTracked: () => tracked };
}
it('delegates original finish calls/idempotence and returns the exact original tracked promise and rejection', async () => {
const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack });
const finish = capture.beginIdleTrackedWork(); expect(actual.idleWorkSnapshot().active).toBe(1);
finish(); finish(); expect(actual.finishes()).toBe(2); expect(actual.idleWorkSnapshot().active).toBe(0);
let resolve!: (value: string) => void;
const tracked = capture.trackIdleWork(new Promise<string>(done => { resolve = done; }));
expect(tracked).toBe(actual.lastTracked()); expect(capture.snapshot().observedActive).toBe(1);
resolve('same'); await expect(tracked).resolves.toBe('same'); expect(capture.snapshot().observedActive).toBe(0);
const failure = new Error('original rejection'), rejected = capture.trackIdleWork(Promise.reject(failure));
expect(rejected).toBe(actual.lastTracked()); await expect(rejected).rejects.toBe(failure);
expect(capture.snapshot().observedActive).toBe(0); expect(actual.idleWorkSnapshot().active).toBe(0);
capture.dispose(); capture.dispose(); expect(capture.snapshot().disposed).toBe(true);
await expect(capture.trackIdleWork(Promise.resolve('after-dispose'))).resolves.toBe('after-dispose');
expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().observedActive).toBe(0);
});
it('cannot throw from passive metadata into actual work and makes missing observation explicit', () => {
const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack, now: () => { throw new Error('private observer detail'); } });
let finish!: () => void; expect(() => { finish = capture.beginIdleTrackedWork(); }).not.toThrow();
expect(actual.idleWorkSnapshot().active).toBe(1); expect(capture.snapshot().missingPaths).toBeGreaterThan(0);
expect(() => finish()).not.toThrow(); expect(actual.idleWorkSnapshot().active).toBe(0);
expect(capture.snapshot().observerErrors).toBeGreaterThan(0); expect(JSON.stringify(capture.summary())).not.toContain('private observer detail');
});
it('bounds active entries, completion tail and filtered callsites without clearing original work', () => {
const actual = original();
const callers = () => 'private outside value\n' + Array.from({ length: 8 }, (_, i) => ` at work (/root/server/src/services/owned-${i}.ts:1:2)`).join('\n');
const capture = createMaintenanceIdleWorkCapture(actual, { stack: callers });
const finish = Array.from({ length: 513 }, () => capture.beginIdleTrackedWork());
const busy = capture.snapshot(); expect(busy.active).toHaveLength(512); expect(busy.overflow).toBe(1);
expect(busy.newSources[0]!.callers).toHaveLength(4); expect(JSON.stringify(busy)).not.toContain('private outside value');
expect(actual.idleWorkSnapshot().active).toBe(513); expect(busy.missingPaths).toBeGreaterThan(0);
finish.forEach(done => done()); expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().completionTail).toHaveLength(64);
});
it('bounds unique source metadata and preserves original finisher failures', () => {
const actual = original(); let source = 0;
const capture = createMaintenanceIdleWorkCapture(actual, { stack: () => `Error\n at owner (/checkout/server/src/services/source-${source++}.ts:1:1)` });
for (let i = 0; i < 513; i++) capture.beginIdleTrackedWork()();
const state = capture.snapshot(); expect(capture.summary().sourceCatalog).toHaveLength(512); expect(state.sourceOverflow).toBe(1);
expect(actual.idleWorkSnapshot().active).toBe(0);
const failure = new Error('original finisher'), begin = actual.beginIdleTrackedWork;
actual.beginIdleTrackedWork = () => { const finish = begin(); return () => { finish(); throw failure; }; };
const observed = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = observed.beginIdleTrackedWork();
expect(() => finish()).toThrow(failure); expect(actual.idleWorkSnapshot().active).toBe(0); expect(observed.snapshot().observedActive).toBe(0);
});
it('bounds settle samples and stores callsites once, while missing paths forbid diagnostic completion', () => {
const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = capture.beginIdleTrackedWork();
const state = capture.snapshot(), catalog = state.newSources;
const samples = createSettledWorkSamples(() => catalog);
for (let i = 0; i < 2442; i++) samples.sample({ now: 1000 + i, idle: actual.idleWorkSnapshot(), workCapture: state }, 20 + i, 0);
const report = samples.report(); expect(report.samples).toHaveLength(2441); expect(report.sampleLimit).toBe(2441); expect(report.overflowSamples).toBe(1);
expect(report.samples[1]!.queryDelta).toBe(1); expect(report.sourceCatalog).toHaveLength(1);
expect(JSON.stringify(report.samples)).not.toContain('owned-work'); expect(() => samples.assertComplete()).toThrow('incomplete');
finish();
const missing = createSettledWorkSamples(() => []); missing.sample({ now: 0, idle: actual.idleWorkSnapshot() }, 0, 0);
expect(() => missing.assertComplete()).toThrow('incomplete');
});
it('intercepts the actual Node24 TS module and preserves every other original export and existing state', async () => {
const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)),
fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url))], { timeout: 15_000, maxBuffer: 128 * 1024 });
expect(JSON.parse(result.stdout.trim())).toMatchObject({ identity: true, originalStatePreserved: true, finalActive: 0 });
});
it('preserves actual Node unhandledRejection with the same original tracked promise and reason', async () => {
const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)),
fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url)), 'unhandled'], { timeout: 15_000, maxBuffer: 128 * 1024 });
expect(JSON.parse(result.stdout.trim())).toMatchObject({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 });
});
@@ -0,0 +1,100 @@
import { createServer, request } from "node:http";
import { setImmediate } from "node:timers/promises";
import { expect, it } from "vitest";
import { observeIncomingHttpWindow, observedNoIncomingHttp } from "../../../scripts/sleep-qualification/evidence.mjs";
import { observeMaintenanceIncomingHttp } from "./helpers/maintenance/incoming-http-observation.js";
import { withMaintenanceProvider } from "./helpers/maintenance/server-harness.js";
it("counts a direct SQL-free health fetch without retaining credentials or changing its response", async () => {
// This fixture has no database or outgoing client; its only work is answering HTTP.
const server = createServer((_req, res) => { res.setHeader("Content-Type", "application/json"); res.end('{"status":"ok"}'); });
server.on("upgrade", (_req, socket) => socket.destroy());
const capture = observeMaintenanceIncomingHttp(server);
try {
await withMaintenanceProvider(server, async origin => {
const before = capture.snapshot(), startedAt = new Date().toISOString();
expect(before).toMatchObject({ installed: true, valid: true, requests: 0 });
expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, before), { startedAt: before.observedAt, finishedAt: before.observedAt })).toBe(true);
const response = await fetch(`${origin}/api/health?fixture=private-query`, { signal: AbortSignal.timeout(5_000), headers: { Authorization: "Bearer fixture-private-token" } });
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ status: "ok" });
const finishedAt = new Date().toISOString(), after = capture.snapshot();
expect(after).toMatchObject({ installed: true, valid: true, captureId: before.captureId });
const observed = observeIncomingHttpWindow(before, after);
expect(observed.requests).toBe(1);
expect(observedNoIncomingHttp(observed, { startedAt, finishedAt })).toBe(false);
expect(JSON.stringify(observed)).not.toMatch(/private-query|fixture-private-token|Authorization|\/api\/health/);
});
} finally { capture.dispose(); }
});
it("observes a genuinely request-free interval without making snapshot reads into HTTP", async () => {
const server = createServer((_req, res) => res.end());
server.on("upgrade", (_req, socket) => socket.destroy());
const capture = observeMaintenanceIncomingHttp(server);
try {
await withMaintenanceProvider(server, async () => {
const before = capture.snapshot(), startedAt = new Date().toISOString();
capture.snapshot(); await setImmediate(); capture.snapshot();
const finishedAt = new Date().toISOString(), after = capture.snapshot();
expect(observeIncomingHttpWindow(before, after).requests).toBe(0);
expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, after), { startedAt, finishedAt })).toBe(true);
// This short boundary test does not claim the configured case's 90-second quiet proof.
});
} finally { capture.dispose(); }
});
it("counts an actual HTTP upgrade as incoming activity", async () => {
const server = createServer((_req, res) => res.end());
const upgrade = (_req: unknown, socket: import("node:stream").Duplex) => {
socket.end("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n");
};
server.on("upgrade", upgrade);
const capture = observeMaintenanceIncomingHttp(server);
try {
await withMaintenanceProvider(server, async origin => {
const before = capture.snapshot();
const client = request(origin, { signal: AbortSignal.timeout(5_000), headers: { Connection: "Upgrade", Upgrade: "fixture" } });
try {
await new Promise<void>((resolve, reject) => {
client.once("error", reject);
client.once("upgrade", (_response, socket) => { socket.destroy(); resolve(); });
client.end();
});
} finally { client.destroy(); }
expect(observeIncomingHttpWindow(before, capture.snapshot()).requests).toBe(1);
});
} finally { capture.dispose(); server.off("upgrade", upgrade); }
});
it("invalidates a detached listener and disposes only its own listeners", async () => {
const app = (_req: unknown, res: import("node:http").ServerResponse) => res.end("app");
const server = createServer(app);
const unrelatedUpgrade = () => {};
server.on("upgrade", unrelatedUpgrade);
const originalRequest = server.listeners("request"), originalUpgrade = server.listeners("upgrade");
const capture = observeMaintenanceIncomingHttp(server);
try {
await withMaintenanceProvider(server, async origin => {
const counter = server.listeners("request")[0]! as (...args: unknown[]) => void;
server.off("request", counter);
expect(capture.snapshot()).toMatchObject({ installed: true, valid: false });
// Reattaching cannot erase an unobserved interval.
server.prependListener("request", counter);
expect(capture.snapshot().valid).toBe(false);
capture.dispose(); capture.dispose();
expect(capture.snapshot()).toMatchObject({ installed: false, valid: false });
expect(server.listeners("request")).toEqual(originalRequest);
expect(server.listeners("upgrade")).toEqual(originalUpgrade);
expect(await (await fetch(origin, { signal: AbortSignal.timeout(5_000) })).text()).toBe("app");
});
} finally { capture.dispose(); }
});
it("leaves Node's unsupported-upgrade behavior alone when the app has no upgrade handler", () => {
const server = createServer((_req, res) => res.end());
const requestHandlers = server.listeners("request"), upgradeHandlers = server.listeners("upgrade");
expect(() => observeMaintenanceIncomingHttp(server)).toThrow("requires existing app request and upgrade handlers");
expect(server.listeners("request")).toEqual(requestHandlers);
expect(server.listeners("upgrade")).toEqual(upgradeHandlers);
});
@@ -0,0 +1,69 @@
import { afterEach, expect, it, vi } from 'vitest';
import { createMaintenanceS3Boundary, decodeMaintenanceAwsChunks } from './helpers/maintenance/s3-boundary.js';
import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js';
import { createS3StorageProvider } from '../storage/s3-provider.js';
import { classifyEarlyDiagnostic, stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js';
afterEach(() => vi.unstubAllEnvs());
it('uses real small-object SDK HTTP with conditional bytes, reads and deletion; unknown operations fail closed', async () => {
vi.stubEnv('AWS_ACCESS_KEY_ID', 'owned-fixture'); vi.stubEnv('AWS_SECRET_ACCESS_KEY', 'owned-fixture'); vi.stubEnv('AWS_SESSION_TOKEN', undefined);
const fixture = createMaintenanceS3Boundary();
await withMaintenanceProvider(fixture.server, async endpoint => {
const provider = createS3StorageProvider({ bucket: 'maintenance-test', region: 'us-east-1', endpoint, forcePathStyle: true });
const input = { objectKey: 'owned/value', contentType: 'text/plain', contentLength: 3, body: Buffer.from('one') };
const first = await provider.putObjectConditional!({ ...input, ifNoneMatch: '*' });
expect(first.etag).toBeTruthy();
await expect(provider.putObjectConditional!({ ...input, ifNoneMatch: '*' })).rejects.toMatchObject({ name: 'PreconditionFailed' });
await expect(provider.putObjectConditional!({ ...input, ifMatch: 'wrong' })).rejects.toMatchObject({ name: 'PreconditionFailed' });
await provider.putObjectConditional!({ ...input, body: Buffer.from('two'), ifMatch: first.etag });
const object = await provider.getObject({ objectKey: input.objectKey }); const chunks: Buffer[] = [];
for await (const chunk of object.stream) chunks.push(Buffer.from(chunk));
expect(Buffer.concat(chunks).toString()).toBe('two'); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(true);
expect((await provider.listObjects!({ prefix: 'owned/' })).keys).toEqual([input.objectKey]);
await provider.deleteObject({ objectKey: input.objectKey }); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(false);
expect(fixture.snapshot().errors).toEqual([]);
const unknown = await fetch(`${endpoint}/maintenance-test/key?uploads`, { method: 'POST' }); expect(unknown.status).toBe(501);
expect(fixture.snapshot().errors).toHaveLength(1);
});
});
it('decodes only complete bounded AWS checksum chunks and refuses malformed framing', () => {
expect(decodeMaintenanceAwsChunks(Buffer.from('3\r\none\r\n3\r\ntwo\r\n0\r\nx-amz-checksum-crc32:YWJj\r\n\r\n')).toString()).toBe('onetwo');
for (const wire of ['3\r\none', '3;signature=unknown\r\none\r\n0\r\n\r\n', '0\r\nunknown:secret\r\n\r\n']) expect(() => decodeMaintenanceAwsChunks(Buffer.from(wire))).toThrow();
});
it('retries only observed transient inspection states and rejects durable blockers even alongside them', () => {
const checks = [
{ reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' },
{ reason: 'backup.checkpoint', state: 'not_checked' }, { reason: 'inspection.hold', state: 'none' },
{ reason: 'inspection.generation', state: 'unknown' }, { reason: 'local.tracked_work', state: 'present' },
{ reason: 'plugins.runtime_drain', state: 'not_checked' }, { reason: 'durable.snapshot', state: 'none' },
{ reason: 'deadlines.catalog', state: 'none' },
];
const report = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: false, checks } } };
expect(classifyEarlyDiagnostic(report, 90_000)).toBe('transient');
const durable = structuredClone(report); durable.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' });
expect(() => classifyEarlyDiagnostic(durable, 90_000)).toThrow('chat_endpoints');
const unknown = structuredClone(report); unknown.idleSleepSafety.diagnostics.checks.push({ reason: 'deadlines.reader', state: 'unknown' });
expect(() => classifyEarlyDiagnostic(unknown, 90_000)).toThrow('deadlines.reader');
for (const check of report.idleSleepSafety.diagnostics.checks) if (['inspection.generation', 'local.tracked_work', 'plugins.runtime_drain'].includes(check.reason)) check.state = 'none';
report.idleSleepSafety.diagnostics.complete = true; expect(classifyEarlyDiagnostic(report, 90_000)).toBe('strict');
});
it('retains observer generation changes and still rejects a second durable report after a transient first', async () => {
const checks = [
{ reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' },
{ reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog'].map(reason => ({ reason, state: 'none' })),
];
const strict = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: true, checks } } };
const observations: unknown[][] = [];
const server = { settled: async () => ({ idle: { active: 0, generation: 10 }, managed: { apiActivityActive: true } }),
snapshot: async () => ({ idle: { active: 0, generation: 14 }, managed: { apiActivityActive: true } }) } as unknown as Parameters<typeof stabilizeEarlyDiagnostics>[0];
const accepted = await stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => structuredClone(strict), persist: async values => { observations.push(structuredClone(values)); } });
expect(accepted.rounds).toBe(1); expect(observations.at(-1)).toHaveLength(4);
const transient = structuredClone(strict); transient.idleSleepSafety.diagnostics.complete = false;
transient.idleSleepSafety.diagnostics.checks.find(check => check.reason === 'inspection.generation')!.state = 'unknown';
const blocked = structuredClone(transient); blocked.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' });
let reads = 0;
await expect(stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => reads++ ? blocked : transient,
persist: async values => { observations.push(structuredClone(values)); } })).rejects.toThrow('chat_endpoints');
expect(reads).toBe(2);
});
@@ -0,0 +1,99 @@
import { execFileSync } from 'node:child_process';
import { generateKeyPairSync, randomUUID, sign } from 'node:crypto';
import { createServer } from 'node:http';
import { writeFile } from 'node:fs/promises';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { beforeAll, describe, expect, it } from 'vitest';
import { createDb } from '@paperclipai/db';
import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js';
import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js';
import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js';
import { managedSleepConfiguration, managedSleepServerOptions, managedTenantActor, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js';
import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js';
import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js';
import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js';
const requested = process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT === '1';
const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_TEST_DATABASE_URL');
if (requested && !databaseUrl) throw new Error('Managed preflight requires an explicit dedicated external *_test URL; no embedded allocation');
if (requested && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Managed preflight requires retained raw evidence');
const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT, cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA;
if (requested && (!cloudCheckout || !/^[a-f0-9]{40}$/.test(cloudSha ?? ''))) throw new Error('Managed preflight requires the exact reviewed private Cloud checkout/SHA');
describe.skipIf(!requested)('actual managed fixture preflight (no browser, cooldown or quiet claim)', () => {
beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000);
it('provisions through real managed auth and completes a process run before strict activity diagnostics', async () => {
if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Cloud source does not match the reviewed preflight SHA');
const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href);
const broker = await startQualificationBroker(); const s3 = createMaintenanceS3Boundary();
const collectorRequests: string[] = [];
const collector = createServer(async (req, res) => { for await (const _chunk of req) { /* Fully consume actual exports. */ } collectorRequests.push(req.url!); res.setHeader('content-type', 'application/json'); res.end('{}'); });
try {
await withMaintenanceProvider(s3.server, async s3Origin => withMaintenanceProvider(collector, async origin => {
const pair = generateKeyPairSync('ed25519');
const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'managed-preflight', alg: 'EdDSA', use: 'sig' };
const configuration = managedSleepConfiguration(origin, true);
const server = await startMaintenanceServer(databaseUrl!, { ...managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker),
captureIdleWork: process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_DIAGNOSTICS === '1' });
const fixture = createDb(databaseUrl!, { maxConnections: 1 });
const control = (action: string, idleCapabilities?: unknown) => {
const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url'), now = Math.floor(Date.now() / 1000);
const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid });
const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) });
return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` };
};
let failure: { error: unknown } | undefined;
try {
await writeFile(path.join(server.directory, 'managed-preflight-startup.json'), JSON.stringify({ configuration, snapshot: await server.snapshot(), s3: s3.snapshot() }, null, 2));
expect(s3.snapshot().errors).toEqual([]);
const company = await provisionManagedSleepCompany(server, fixture);
await expect(server.api('POST', '/companies', { name: 'Must remain platform managed' })).rejects.toThrow('cloud_managed');
// Reuse the combined case's real vault/continuity/enrollment owners.
// The strict broker capability is unavailable before event-wake enrollment.
const github = await server.prepareGitHub(company.id);
await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true });
const githubContinuity = await server.settleGitHub(github);
await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding: await broker.binding(), boundary: broker.boundary }, null, 2));
await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1);
const brokerBinding = await broker.binding();
await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding, boundary: broker.boundary }, null, 2));
expect(brokerBinding?.instanceId).toBe(broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID);
expect(brokerBinding?.bindings).toEqual([[`${github.grantId}_101`, company.id, managedTenantActor.userId,
'github.code', '101', github.connectionId, github.grantId]]);
const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Managed preflight process', role: 'general', adapterType: 'process',
adapterConfig: { command: process.execPath, args: ['-e', "console.log('managed-preflight-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } });
const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Actual managed preflight key' });
const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'One actual managed process run', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` });
await expect.poll(async () => { const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; return run?.status; }, { timeout: 30_000 }).toBe('succeeded');
await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' });
const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${issue.id}`; expect(executions.count).toBe(1);
await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); const apiSucceededAt = performance.now();
const activity = await server.snapshot();
await writeFile(path.join(server.directory, 'managed-preflight-setup.json'), JSON.stringify({ company, githubContinuity, brokerBinding,
agentId: agent.id, keyId: key.id, issueId: issue.id, executions: executions.count, activity, s3: s3.snapshot() }, null, 2));
expect(activity.managed?.apiActivityActive).toBe(true);
await server.settled(500, 0, { managedSetupRecovery: true });
const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start'));
try {
const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' };
const read = () => server.api<any>('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities));
const proof = await stabilizeEarlyDiagnostics(server, { ownerId: drain.ownerId, apiSucceededAt, read,
persist: observations => writeFile(path.join(server.directory, 'managed-preflight-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) });
await writeFile(path.join(server.directory, 'managed-preflight-receipt.json'), JSON.stringify({ companyId: company.id, proof, s3: s3.snapshot(), collectorRequests,
boundary: 'Actual managed startup/provisioning/key activity/process outcome; no browser, real cooldown/quiet, eligibility, provider suspension or durable S3 claim' }, null, 2));
} finally { await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop')); }
} catch (error) { failure = { error }; throw error; }
finally {
const errors: unknown[] = [];
try { await writeFile(path.join(server.directory, 'managed-preflight-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); }
try { await server.stop(); } catch (error) { errors.push(error); }
try { await fixture.$client.end({ timeout: 1 }); } catch (error) { errors.push(error); }
try { await writeFile(path.join(server.directory, 'managed-preflight-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); }
if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`));
if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Managed preflight or owned cleanup failed');
}
}));
} finally { await broker.stop(); }
}, 180_000);
});
@@ -0,0 +1,43 @@
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { fileURLToPath } from 'node:url';
import { createServer } from 'node:http';
import { it, expect } from 'vitest';
import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js';
// Opt-in qualification must fail when the SDK falls back to no-op. The ordinary
// bootstrap tests separately retain coverage for the supported missing-peer mode.
it.skipIf(process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE !== '1')('exports an actual named span with no metric or log traffic over 90 real seconds', async () => {
const spans: string[] = [], paths: string[] = [];
const collector = createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
paths.push(req.url ?? 'missing-path');
const envelope = JSON.parse(Buffer.concat(chunks).toString('utf8'));
for (const resource of envelope.resourceSpans ?? []) for (const scope of resource.scopeSpans ?? []) for (const span of scope.spans ?? []) spans.push(span.name);
res.setHeader('Content-Type', 'application/json'); res.end('{}');
});
await withMaintenanceProvider(collector, async origin => {
const startedAt = performance.now();
const { stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)),
fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url)), '--observe-exporters'], {
env: { PATH: process.env.PATH, NODE_ENV: 'production', OTEL_EXPORTER_OTLP_ENDPOINT: origin, OTEL_EXPORTER_OTLP_PROTOCOL: 'http/json' },
timeout: 110_000, maxBuffer: 2 * 1024 * 1024,
});
expect(performance.now() - startedAt).toBeGreaterThanOrEqual(90_000);
expect(stderr).not.toContain('Continuing without tracing');
expect(spans).toContain('sleep-qualification-stimulus');
expect(paths).not.toContain('/v1/metrics');
expect(paths).not.toContain('/v1/logs');
});
}, 120_000);
it('keeps the optional SDK and exporters inactive without an endpoint', async () => {
const { stdout, stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)),
fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url))], {
env: { PATH: process.env.PATH, NODE_ENV: 'production' }, timeout: 15_000, maxBuffer: 2 * 1024 * 1024,
});
expect(stdout).not.toContain('OpenTelemetry service.version=');
expect(stderr).not.toContain('Continuing without tracing');
});
@@ -1892,8 +1892,28 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
});
it("retains one safe report per call and rejects history after access revocation", async () => {
const f = await nativeServiceFixture(), first = await f.service.start(f.input);
const foreign = await nativeServiceFixture();
const [foreignSession] = await db.select().from(chatVoiceSessions).where(eq(chatVoiceSessions.id, foreign.sessionId));
await db.insert(chatVoiceReports).values({ companyId: foreign.companyId, sessionId: foreign.sessionId, nextCheckAt: new Date(0) });
const foreignReports = await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId));
const reportProviderIds: string[] = [];
const fetchReport = f.transport.callReport as (providerSessionId: string) => ReturnType<typeof f.transport.callReport>;
const reports = voiceCallReports(db, session => {
reportProviderIds.push(session.providerSessionId!);
return fetchReport(session.providerSessionId!);
});
expect(await reports.load({ kind: "company", id: foreign.companyId })).toEqual([
expect.objectContaining({ id: foreign.sessionId, companyId: foreign.companyId, dueAt: 0 }),
]);
// This case owns report/history behavior for its company. Global recovery
// also visits every predecessor fixture's live calls, replies and reports.
const reconcileReports = async () => {
const due = (await reports.load({ kind: "company", id: f.companyId }))
.filter(row => row.dueAt <= Date.now()).sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id));
for (const row of due) await reports.check(row.id);
};
await f.service.end(f.companyId, first.session.id, f.input.caller);
await f.service.reconcile(1000);
await reconcileReports();
const detail = await f.service.callDetail(f.companyId, first.session.id, f.input.caller);
expect(detail.report).toMatchObject({ status: "available", costMicroUsd: "120000", transcript: [{ id: "turn1", text: "A test request" }] });
expect(await f.service.history(f.companyId, f.endpointId, f.input.caller)).toEqual(expect.arrayContaining([detail]));
@@ -1908,12 +1928,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
const baseline = await f.transport.callReport(), latest = new Date(Date.now() + 10000);
let response = { ...baseline, costMicroUsd: "240000", providerUpdatedAt: latest };
f.transport.callReport.mockImplementation(async (providerSessionId?: string) => providerSessionId === stored.providerSessionId ? response : baseline);
await f.service.reconcile(1000);
await reconcileReports();
expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000");
expect((await f.service.callDetail(f.companyId, other.session.id, f.input.caller)).report.costMicroUsd).toBe("120000");
await db.update(chatVoiceReports).set({ nextCheckAt: new Date(0) }).where(eq(chatVoiceReports.sessionId, first.session.id));
response = { ...baseline, costMicroUsd: "1", providerUpdatedAt: new Date(0) };
await f.service.reconcile(1000);
await reconcileReports();
expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000");
expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.sessionId, first.session.id))).toHaveLength(1);
await expect(f.service.callDetail(f.companyId, first.session.id, { id: randomUUID(), authority: "member" })).rejects.toMatchObject({ status: 403 });
@@ -1922,6 +1942,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
await expect(f.service.history(f.companyId, f.endpointId, f.input.caller)).rejects.toMatchObject({ status: 403 });
await expect(f.service.callDetail(f.companyId, first.session.id, f.input.caller)).rejects.toMatchObject({ status: 403 });
expect(f.transport.createPhoneSession).not.toHaveBeenCalled();
expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId))).toEqual(foreignReports);
expect(reportProviderIds).toContain(stored.providerSessionId);
expect(reportProviderIds).not.toContain(foreignSession.providerSessionId);
});
it("does not create provider media when membership is revoked during credential resolution", async () => {
const f = await nativeServiceFixture();
+18 -10
View File
@@ -14,6 +14,7 @@ import {
agents,
companies,
createDb,
closeRegisteredClients,
executionWorkspaces,
heartbeatRuns,
issueComments,
@@ -87,6 +88,7 @@ import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
const RUNTIME_OWNED_GIT_BRANCH_METADATA = {
createdByRuntime: true,
@@ -134,8 +136,12 @@ function workspaceBranchIncoherenceFingerprintForTest(input: {
}
const leasedRunIds = new Set<string>();
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
const externalRuntimeControlDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_RUNTIME_CONTROL_TEST_DATABASE_URL");
const embeddedPostgresSupport: Awaited<ReturnType<typeof getEmbeddedPostgresTestSupport>> = externalRuntimeControlDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
// The explicit fixture belongs only to the service-control suite. Other DB
// suites must not allocate an embedded or admin-owned database in this mode.
const describeEmbeddedPostgres = !externalRuntimeControlDatabaseUrl && embeddedPostgresSupport.supported ? describe : describe.skip;
const describeRuntimeControl = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
console.warn(
@@ -6672,16 +6678,18 @@ describeEmbeddedPostgres("workspace dirty quarantine branch repair", () => {
}, 20_000);
});
describeEmbeddedPostgres("workspace runtime service control persistence", () => {
describeRuntimeControl("workspace runtime service control persistence", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-");
db = createDb(tempDb.connectionString);
if (!externalRuntimeControlDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-");
db = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString,
externalRuntimeControlDatabaseUrl ? { maxConnections: 4 } : undefined);
}, 20_000);
afterAll(async () => {
if (externalRuntimeControlDatabaseUrl) await closeRegisteredClients(externalRuntimeControlDatabaseUrl, { scope: "database" });
await tempDb?.cleanup();
});
@@ -7284,14 +7292,14 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () =>
return result[0]!;
}));
// The two lanes claim ports concurrently in-process. The allocator guarantees
// distinct ports, but not which lane wins the base port. It depends on the
// scheduling order. So assert order-independent invariants, not array index.
// A free-port probe is not a reservation. Both siblings must own distinct
// live ports in the allocator's bounded range even if the preferred port
// becomes unavailable before allocation.
const lowerPort = Math.min(first.port!, second.port!);
const upperPort = Math.max(first.port!, second.port!);
const maxAllocatablePort = basePort + WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS - 1;
expect(first.port).not.toBe(second.port);
expect(lowerPort).toBe(basePort);
expect(lowerPort).toBeGreaterThanOrEqual(basePort);
expect(upperPort).toBeGreaterThan(basePort);
expect(upperPort).toBeLessThanOrEqual(maxAllocatablePort);
expect(first.url).toBe(`http://127.0.0.1:${first.port}`);
@@ -7333,7 +7341,7 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () =>
it("commits fixed-port reservations across the configured database idle timeout", async () => {
const fixture = await createRuntimeFixture();
const readinessDb = createDb(tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 });
const readinessDb = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 });
const [initialConnection] = await readinessDb.$client<{ pid: number }[]>`select pg_backend_pid() as pid`;
const cleanupRuntimeHome = await createRuntimeHome();
const workspace = fixture.workspaces[0]!;
+4
View File
@@ -499,6 +499,10 @@ async function bootstrapOtel(endpoint: string): Promise<void> {
console.log(`[paperclip] OpenTelemetry service.version=${serviceVersion}`);
const sdk = new NodeSDK({
// This integration exports traces only. NodeSDK otherwise creates
// default OTLP metric/log exporters even when none were configured.
metricReaders: [],
logRecordProcessors: [],
resource: resourceFromAttributes({
[ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME || "paperclip",
[ATTR_SERVICE_VERSION]: serviceVersion,