diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 5f6bb10206c..c194c4c202b 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -4744,6 +4744,102 @@ unavailable. Use a separate, disposable database whose name ends in `_test` for `PAPERCLIP_SKILL_SOURCES_TEST_DATABASE_URL`. The suites apply migrations and create test data; the synchronization suite truncates company data between tests. +#### Controlled default-cadence sleep acceptance + +The opt-in `domain_deadlines_v1` negotiation adds owner-bound skill deadline +handoff coverage. Legacy callers retain the conservative blockers described above. +The checked-in `scripts/test-skill-default-cadence.mjs` launcher checks this local +flow with the actual app, signed tenant routes, Cloud controller and PostgreSQL +alarm store. Git contents, S3 and physical child stop/start use controlled local +providers. A matching service checkout with +`src/sleep/fixtures/default-cadence-bridge.ts` and its normal compiled output is +required for the host flow. The PG0 controls do not require that checkout or a DB. +The launcher does not start a cluster, build artifacts, or contact live providers. + +Use Node **24.22.2**, pnpm **9.15.4**, and Node **22.23.4** for the matching service. +Prepare the server dependency closure with a frozen lockfile. Build only missing +TypeScript prerequisites explicitly; the launcher does not install or build them. +From the app repository root: + +```sh +export NODE24=/absolute/path/to/node-v24.22.2/bin/node +export PNPM9=/absolute/path/to/pnpm-9.15.4/bin/pnpm +export PATH="$(dirname "$NODE24"):$(dirname "$PNPM9"):$PATH" +"$PNPM9" --filter @paperclipai/server... install --frozen-lockfile --ignore-scripts +for package in packages/shared packages/plugins/sdk packages/paperclip-eval-kernel packages/paperclip-runner; do + (cd "$package" && "$NODE24" node_modules/typescript/bin/tsc -p tsconfig.json) +done +(cd packages/paperclip-runner && "$NODE24" node_modules/typescript/bin/tsc -p tsconfig.surfaces.json) +export SKILL_SLEEP_RESULTS="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-skill-sleep.XXXXXX")" +export PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG="$SKILL_SLEEP_RESULTS/controls-guard.jsonl" +"$NODE24" scripts/test-skill-default-cadence.mjs --controls +``` + +This PG0 lane runs 44 controls. It explicitly defers the single real-PostgreSQL +observer case before its support probe. It requires the actual worker's preloaded +guard marker and matching receipt. Ordinary Vitest runs keep all subprocess +controls and defer only that launcher-specific parent assertion. An explicit +prewire mode or guard-log setting makes the unchanged assertion mandatory; a +missing or invalid marker fails. The launcher never installs the guard late. + +For host acceptance, separately create empty, owned databases on an existing local +PostgreSQL 17 server. Use distinct names ending in `_test`. The test applies normal +migrations and writes fixtures; it never starts an embedded cluster. Use a fresh +pair for each attempt, including the full run after bootstrap. Prepare the matching +service dependencies separately and compile its normal `tsconfig.build.json` with +Node 22. The recorded local runtime used service source +`2cb54f904031dcf69a32902cd853b43f74d6f301`. Current service publication +`389878364b42dd129835d263f91882c5774d5123` carries that measured helper source through +its normal upstream merge; the runtime was not repeated on that service merge. +An arbitrary service revision is not an interchangeable prerequisite. + +```sh +export PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT=/absolute/path/to/matching-service +export PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE=/absolute/path/to/node-v22.23.4/bin/node +export PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA=389878364b42dd129835d263f91882c5774d5123 +test "$(git -C "$PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT" rev-parse HEAD)" = "$PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA" +(cd "$PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT" && "$PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE" node_modules/typescript/bin/tsc -p tsconfig.build.json) +export PAPERCLIP_TEST_POSTGRES_ADMIN_URL=postgres://paperclip@127.0.0.1:55448/postgres +export PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL=postgres://paperclip@127.0.0.1:55448/skill_sleep_bootstrap_app_test +export PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL=postgres://paperclip@127.0.0.1:55448/skill_sleep_bootstrap_cloud_test +export PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE="$SKILL_SLEEP_RESULTS/bootstrap" +mkdir -p "$PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE" +unset PAPERCLIP_MAINTENANCE_PG0_CONTROLS +export PAPERCLIP_SKILL_DEFAULT_CADENCE_STAGE=bootstrap +export PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG="$SKILL_SLEEP_RESULTS/bootstrap-guard.jsonl" +"$NODE24" scripts/test-skill-default-cadence.mjs +``` + +Bootstrap checks revision 1, the legacy refusal, the signed bounded handoff and the +exact durable ACK before child stop. It does not check a due wake. For the complete +run, first create a different empty pair, then run: + +```sh +export PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL=postgres://paperclip@127.0.0.1:55448/skill_sleep_cycles_app_test +export PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL=postgres://paperclip@127.0.0.1:55448/skill_sleep_cycles_cloud_test +export PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE="$SKILL_SLEEP_RESULTS/two-cycle" +mkdir -p "$PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE" +export PAPERCLIP_SKILL_DEFAULT_CADENCE_STAGE=two-cycle +export PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG="$SKILL_SLEEP_RESULTS/two-cycle-guard.jsonl" +"$NODE24" scripts/test-skill-default-cadence.mjs +``` + +Use a new guard-log path for each attempt. The full mode takes about 10-14 minutes. +It retains the original 300-second source cadence, real app/SQL clocks, D-120 alarm +eligibility and the existing 850-second test timeout. Do not shorten the cadence, +write deadlines, or substitute a fake clock. The launcher uses one worker and no +file parallelism. Budget runtime pools for the app's potential 10 connections plus +Cloud's shared 2 and one transient observer; each pool is at most 4. The recorded +server limit was 40. Run this fixture without other heavy tests or builds. + +Receipts retain both original deadlines, durable alarm generations, distinct cold +processes and persisted revisions 2 and 3. Each cold start also checks a measured +10-second app SQL/upstream/outgoing-HTTP quiet window. Awake guards can still show +2 PostgreSQL sockets: quiet SQL does not mean all connections are closed. This is +controlled local acceptance, not strict 90-second host acceptance, live-provider +qualification or proof of deployed Railway/Neon suspension. Keep failed attempts +and audit owned sessions/processes before database cleanup. + ### Public MCP and Dot event delivery Public MCP task events and Dot mailbox events use separate topics in the shared diff --git a/scripts/fixtures/skill-default-cadence-network-guard.cjs b/scripts/fixtures/skill-default-cadence-network-guard.cjs new file mode 100644 index 00000000000..64e77e8aa89 --- /dev/null +++ b/scripts/fixtures/skill-default-cadence-network-guard.cjs @@ -0,0 +1,97 @@ + +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const net = require('node:net'), tls = require('node:tls'), http = require('node:http'), https = require('node:https'), dns = require('node:dns'); +const { createHash } = require('node:crypto'); +const { threadId, isMainThread } = require('node:worker_threads'); +const key = Symbol.for('paperclip.skill.default-cadence.prewire'); +const log = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG; +if (!log) throw new Error('Default cadence preload requires an explicit guard log'); +if (globalThis[key]) { if (globalThis[key].log !== log) throw new Error('Default cadence guard log changed'); return; } +const loopback = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']); +const listeners = new Set(), endpoints = new Set(), pgEndpoints = new Set(), attempts = [], active = new Set(); +const tracked = new WeakSet(); let peak = 0; +const identity = 'skill-default-cadence-guard-v1'; +const sourceHash = createHash('sha256').update(fs.readFileSync(__filename)).digest('hex'); +const record = (event, value = {}) => fs.appendFileSync(log, JSON.stringify({ event, identity, sourceHash, pid: process.pid, ppid: process.ppid, threadId, isMainThread, uid: process.getuid?.(), at: Date.now(), ...value }) + '\n'); +const deny = reason => { attempts.push(reason); process.exitCode = 98; record('blocked', { reason }); throw new Error(`Default cadence guard blocked ${reason} before transport`); }; +const local = host => { if (!loopback.has(String(host).toLowerCase())) deny(`offhost ${host}`); }; +function allowOrigin(value) { + const url = new URL(value); local(url.hostname); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash || url.pathname !== '/') deny('invalid owned HTTP origin'); + endpoints.add(`${url.hostname}:${url.port || (url.protocol === 'https:' ? 443 : 80)}`); +} +function allowDatabase(value) { + const url = new URL(value); local(url.hostname); + if (!['postgres:', 'postgresql:'].includes(url.protocol) || !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1))) + || ['options', 'database', 'dbname'].some(k => url.searchParams.has(k))) deny('invalid dedicated database'); + const endpoint = `${url.hostname}:${url.port || 5432}`; endpoints.add(endpoint); pgEndpoints.add(endpoint); +} +for (const name of ['PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL', 'DATABASE_URL']) if (process.env[name]) allowDatabase(process.env[name]); +for (const value of JSON.parse(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_ORIGINS || '[]')) allowOrigin(value); +function check(host = 'localhost', port, protocol) { + local(host); + if (protocol && !['http:', 'https:'].includes(protocol)) deny(`protocol ${protocol}`); + if (!port || !endpoints.has(`${host}:${port}`) && !listeners.has(Number(port))) deny(`unowned loopback ${host}:${port}`); +} +function checkArgs(args, httpProtocol) { + if (Array.isArray(args[0])) return checkArgs(args[0], httpProtocol); + const first = args[0]; + if (first instanceof URL || typeof first === 'string' && /^https?:/.test(first)) { + const url = first instanceof URL ? first : new URL(first); check(url.hostname, url.port || (url.protocol === 'https:' ? 443 : 80), url.protocol); + if (args[1] && typeof args[1] === 'object') { + const override = args[1]; if (override.hostname || override.host) check(override.hostname || override.host, override.port || url.port || (url.protocol === 'https:' ? 443 : 80), override.protocol || url.protocol); + } + } else if (first && typeof first === 'object') { + if (first.socketPath || !httpProtocol && first.path && !first.port) deny('unowned Unix socket'); + const port = first.port || (httpProtocol === 'https:' ? 443 : httpProtocol === 'http:' ? 80 : undefined); + check(first.hostname || first.host, port, first.protocol || httpProtocol); + } else if (typeof first === 'number') check(typeof args[1] === 'string' ? args[1] : undefined, first); + else deny('unknown connection arguments'); +} +const listen = net.Server.prototype.listen; +net.Server.prototype.listen = function(...args) { + this.once('listening', () => { const address = this.address(); if (address && typeof address === 'object' && loopback.has(address.address)) { + listeners.add(address.port); this.once('close', () => listeners.delete(address.port)); + } }); + return Reflect.apply(listen, this, args); +}; +for (const module of [http, https]) for (const name of ['request', 'get']) { + const original = module[name]; module[name] = function(...args) { checkArgs(args, module === https ? 'https:' : 'http:'); return Reflect.apply(original, this, args); }; +} +for (const module of [dns, dns.promises]) for (const name of ['lookup', 'resolve', 'resolve4', 'resolve6', 'resolveAny', 'resolveCname', 'resolveMx', 'resolveNaptr', 'resolveNs', 'resolvePtr', 'resolveSoa', 'resolveSrv', 'resolveTxt', 'reverse']) if (typeof module[name] === 'function') { + const original = module[name]; module[name] = function(...args) { local(args[0]); if (name !== 'lookup') deny(`unowned DNS ${name}`); return Reflect.apply(original, this, args); }; +} +const connect = net.Socket.prototype.connect; +net.Socket.prototype.connect = function(...args) { + const normalized = Array.isArray(args[0]) ? args[0] : args; + const first = normalized[0]; const socketPath = typeof first === 'string' ? first : first?.port ? undefined : first?.path; + if (socketPath) { + const expected = path.join(os.tmpdir(), `tsx-${process.getuid?.()}`, `${process.ppid}.pipe`); + let live = false; try { process.kill(process.ppid, 0); live = true; } catch {} + if (!live || path.resolve(socketPath) !== path.resolve(expected) || fs.existsSync(expected)) deny('unowned or existing Unix socket'); + record('current-parent-pipe-probe', { path: expected, parent: process.ppid }); + this.once('connect', () => { attempts.push('unexpected parent pipe connection'); process.exitCode = 98; record('blocked', { reason: 'unexpected parent pipe connection' }); this.destroy(); }); + this.once('error', error => { + if (error.code !== 'ENOENT') { attempts.push(`parent pipe ${error.code}`); process.exitCode = 98; record('blocked', { reason: `parent pipe ${error.code}` }); } + else record('current-parent-pipe-ENOENT', { path: expected, parent: process.ppid }); + }); + return Reflect.apply(connect, this, args); + } + checkArgs(normalized); + const host = first && typeof first === 'object' ? first.host || first.hostname || 'localhost' : typeof normalized[1] === 'string' ? normalized[1] : 'localhost'; + const port = first && typeof first === 'object' ? first.port : first; + if (pgEndpoints.has(`${host}:${port}`) && !tracked.has(this)) { + tracked.add(this); this.once('connect', () => { active.add(this); peak = Math.max(peak, active.size); }); this.once('close', () => active.delete(this)); + } + return Reflect.apply(connect, this, args); +}; +const tlsConnect = tls.connect; tls.connect = function(...args) { checkArgs(args); return Reflect.apply(tlsConnect, this, args); }; +const fetch = globalThis.fetch; if (fetch) globalThis.fetch = function(input, init) { checkArgs([input instanceof Request ? input.url : input]); return fetch(input, init); }; +const exit = process.exit; +process.exit = function(code) { return exit.call(process, attempts.length ? 98 : code); }; +const stats = () => ({ loaded: true, identity, sourceHash, pid: process.pid, ppid: process.ppid, threadId, isMainThread, uid: process.getuid?.(), preload: __filename, + peakPgSockets: peak, remainingPgSockets: active.size, unexpectedAttempts: attempts.slice() }); +const state = Object.freeze({ log, stats, allowOrigin, allowDatabase }); +globalThis[key] = state; process[key] = state; +record('preloaded', { node: process.version, execPath: process.execPath, preload: __filename }); +process.on('exit', () => record('exit', { ...stats(), exitCode: process.exitCode || 0 })); diff --git a/scripts/test-skill-default-cadence.mjs b/scripts/test-skill-default-cadence.mjs new file mode 100644 index 00000000000..4d1807e530a --- /dev/null +++ b/scripts/test-skill-default-cadence.mjs @@ -0,0 +1,58 @@ +import { createRequire } from 'node:module'; +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const controls = process.argv.slice(2).join(' ') === '--controls'; +if (!controls && process.argv.length > 2) throw new Error('Use no arguments for the witness or --controls for finite PG0 controls'); +const names = ['PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL']; +if (!controls && process.env.PAPERCLIP_MAINTENANCE_PG0_CONTROLS) throw new Error('PG0 controls flag is forbidden for the witness'); +if (!controls) { + const urls = names.map(name => { + if (!process.env[name]) throw new Error(`${name} requires an explicit dedicated test database`); + let url; try { url = new URL(process.env[name]); } catch { throw new Error(`${name} requires a loopback dedicated test database`); } + if (!['postgres:', 'postgresql:'].includes(url.protocol) || !['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname) + || !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1))) + || ['options', 'dbname', 'database'].some(key => url.searchParams.has(key))) throw new Error(`${name} requires a loopback dedicated test database`); + return url; + }); + if (urls[0].pathname === urls[1].pathname) throw new Error('App and Cloud require distinct test databases'); + const cloud = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT; + if (!cloud || !path.isAbsolute(cloud) || !existsSync(path.join(cloud, 'dist/sleep/fixtures/default-cadence-bridge.js'))) throw new Error('Set the exact compiled Cloud checkout'); + if (!path.isAbsolute(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE || '')) throw new Error('Set the actual Node22 executable'); + if (!/^[a-f0-9]{40}$/.test(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA || '')) throw new Error('Set exact Cloud source SHA'); +} +const log = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG; +if (!log || !path.isAbsolute(log) || !existsSync(path.dirname(log))) throw new Error('Set an absolute guard log in the existing evidence directory'); +if (existsSync(log) && statSync(log).size !== 0) throw new Error('Use a fresh per-attempt guard log'); +if (!process.version.startsWith('v24.')) throw new Error('Launch acceptance with the actual Node24 executable'); +const require = createRequire(new URL('../server/package.json', import.meta.url)); +const guard = path.join(root, 'scripts/fixtures/skill-default-cadence-network-guard.cjs'); + + +require(guard); +const vitest = path.join(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); +const env = { PATH: process.env.PATH, HOME: process.env.HOME, LANG: process.env.LANG, TMPDIR: process.env.TMPDIR, + NODE_ENV: 'test', NODE_OPTIONS: `--require=${guard}`, PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG: log }; +if (controls) env.PAPERCLIP_MAINTENANCE_PG0_CONTROLS = '1'; +if (!controls) { + for (const name of [...names, 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE', + 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_STAGE', + 'PAPERCLIP_TEST_POSTGRES_ADMIN_URL']) if (process.env[name]) env[name] = process.env[name]; + env.PAPERCLIP_SKILL_DEFAULT_CADENCE_ACCEPTANCE = '1'; env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS = '1'; +} +const files = controls ? ['server/src/__tests__/skill-default-cadence-prewire.test.ts', 'server/src/__tests__/maintenance-harness-resources.test.ts', + 'server/src/__tests__/maintenance-postgres-proxy.test.ts', 'server/src/services/skill-source-idle-schedule.test.ts'] + : ['server/src/__tests__/skill-source-default-cadence-acceptance.test.ts']; +const child = spawn(process.execPath, ['--require', guard, vitest, 'run', ...files, + '--maxWorkers=1', '--no-file-parallelism', ...(controls ? ['--testNamePattern=^(?!.*maintenance observer against PostgreSQL)'] : []), `--execArgv=--require=${guard}`], { cwd: root, env, stdio: 'inherit' }); +for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); +child.once('error', error => { console.error(error); process.exitCode = 1; }); +child.once('close', (code, signal) => { + const events = readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + const violation = events.some(event => event.event === 'blocked' || event.unexpectedAttempts?.length); + if (violation) console.error('Default cadence guard recorded an unexpected attempt; caught exceptions do not qualify'); + process.exitCode = violation ? 98 : signal ? 1 : code ?? 1; +}); diff --git a/server/src/__tests__/helpers/maintenance/default-cadence-bridge.ts b/server/src/__tests__/helpers/maintenance/default-cadence-bridge.ts new file mode 100644 index 00000000000..a559f35f81a --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/default-cadence-bridge.ts @@ -0,0 +1,89 @@ +import { fileURLToPath } from 'node:url'; +import { defaultCadenceNetwork } from './default-cadence-network.js'; +import { fork } from 'node:child_process'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + + +export async function startDefaultCadenceBridge(options: { + checkout: string; node: string; + physicalStop(): Promise; physicalWake(): Promise<{ origin: string }>; +}) { + const child = fork(path.join(options.checkout, 'dist/sleep/fixtures/default-cadence-bridge.js'), [], { + cwd: options.checkout, execPath: options.node, execArgv: ["--require", fileURLToPath(new URL("../../../../../scripts/fixtures/skill-default-cadence-network-guard.cjs", import.meta.url))], + env: { PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: 'test', PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG: process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG }, stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }); + let output = '', id = 0; + child.stdout?.on('data', bytes => { output += String(bytes); }); + child.stderr?.on('data', bytes => { output += String(bytes); }); + const pending = new Map(); + const physicalWork = new Set>(); + const physicalCleanupFailures: unknown[] = []; + let closing = false; + let readyResolve!: () => void, readyReject!: (error: Error) => void; + const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; }); + child.on('message', (message: any) => { + if (message.ready) { + const guard = message.defaultCadenceGuard; + if (!guard?.loaded || guard.pid !== child.pid || guard.ppid !== process.pid || guard.sourceHash !== defaultCadenceNetwork().sourceHash) { + readyReject(new Error('Actual Cloud child did not report the expected preloaded guard identity')); return; + } + readyResolve(); return; + } + if (message.callbackId) { + if (closing) { + if (child.connected) child.send({ callbackId: message.callbackId, error: 'Default cadence bridge is closing' }); + return; + } + const operation = message.physical === 'stop' ? options.physicalStop() : options.physicalWake(); + const handled = operation.then(async result => { + if (closing && message.physical === 'wake') await options.physicalStop(); + if (child.connected) child.send({ callbackId: message.callbackId, result }); + }, error => { + if (child.connected) child.send({ callbackId: message.callbackId, error: error instanceof Error ? error.stack : String(error) }); + }); + physicalWork.add(handled); + void handled.then(() => physicalWork.delete(handled), error => { + physicalWork.delete(handled); physicalCleanupFailures.push(error); + }); + return; + } + const request = pending.get(message.id); pending.delete(message.id); + if (message.error) { + output += `\nCloud RPC failure: ${message.error}`; + console.error(`[default-cadence-bridge] ${message.error}`); + request?.reject(new Error(message.error)); + } else request?.resolve(message.result); + }); + child.once('error', readyReject); + const exited = new Promise(resolve => child.once('exit', () => { + const failure = new Error(`Default cadence bridge exited: ${output.slice(-8000)}`); + readyReject(failure); for (const request of pending.values()) request.reject(failure); resolve(); + })); + async function bounded(work: Promise, ms: number) { + const abort = new AbortController(); + try { return await Promise.race([work, delay(ms, undefined, { signal: abort.signal }).then(() => { throw new Error(`Default cadence bridge deadline: ${output.slice(-8000)}`); })]); } + finally { abort.abort(); } + } + try { await bounded(ready, 15_000); } catch (error) { child.kill("SIGKILL"); await exited; throw error; } + async function rpc(method: string, input?: unknown, budgetMs = 210_000): Promise { + const requestId = ++id; + const response = new Promise((resolve, reject) => pending.set(requestId, { resolve, reject })); + child.send({ id: requestId, method, input }); + try { return await bounded(response, budgetMs); } finally { pending.delete(requestId); } + } + return { rpc: (method: string, input?: unknown) => rpc(method, input), pid: child.pid, get output() { return output; }, async close() { + closing = true; + const deadline = performance.now() + 210_000; + const remaining = () => Math.max(0, deadline - performance.now()); + try { + await bounded(Promise.all([...physicalWork]), remaining()); + if (physicalCleanupFailures.length) throw new AggregateError(physicalCleanupFailures, 'Physical callback cleanup failed'); + if (child.exitCode === null && child.connected) { await rpc('close', undefined, remaining()); console.info(`[default-cadence-bridge] close acknowledged pid=${child.pid}`); } + } finally { + console.info(`[default-cadence-bridge] disconnect pid=${child.pid} exit=${child.exitCode} signal=${child.signalCode}`); + if (child.connected) child.disconnect(); + await bounded(exited, 10_000).catch(async error => { child.kill('SIGKILL'); await exited; throw error; }); + } + } }; +} diff --git a/server/src/__tests__/helpers/maintenance/default-cadence-child.ts b/server/src/__tests__/helpers/maintenance/default-cadence-child.ts new file mode 100644 index 00000000000..dd81c178bbf --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/default-cadence-child.ts @@ -0,0 +1,35 @@ +import { installDefaultCadenceNetwork, defaultCadenceNetwork } from './default-cadence-network.js'; +import assert from 'node:assert/strict'; +import { mock } from 'node:test'; +import { githubFixture } from '../github-skills.js'; +import type { openGitSkillSnapshot } from '../../../services/skill-source-git-snapshot.js'; + + + + +installDefaultCadenceNetwork(); +const send = process.send; +if (send) process.send = function(message: unknown, ...args: unknown[]) { + const value = message && typeof message === 'object' ? { ...message, defaultCadenceGuard: defaultCadenceNetwork() } : message; + return Reflect.apply(send, process, [value, ...args]); +} as NonNullable; +const origin = process.env.PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN; +assert(origin && new URL(origin).hostname === '127.0.0.1'); +const actual = await import('../../../services/skill-source-git-snapshot.js'); +const open: typeof openGitSkillSnapshot = async (input, options = {}) => { + assert.equal(input.repositoryUrl, 'https://github.com/acme/skills'); + assert(['HEAD', 'main'].includes(input.ref) || /^[a-f0-9]{40}$/i.test(input.ref)); + options.signal?.throwIfAborted(); + options.beforeDownload?.(); + const response = await fetch(new URL('/__skill_snapshot', origin), { signal: options.signal }); + assert(response.ok); + const snapshot = await response.json() as { commit: string; files: Record }; + assert.match(snapshot.commit, /^[a-f0-9]{40}$/); + if (input.commitSha) assert.equal(input.commitSha, snapshot.commit); + return githubFixture(snapshot.files, {}, snapshot.commit).openSnapshot!(input, options); +}; +mock.module(new URL('../../../services/skill-source-git-snapshot.ts', import.meta.url).href, + { namedExports: { ...actual, openGitSkillSnapshot: open } }); +assert.equal((await import('../../../services/skill-source-git-snapshot.js')).openGitSkillSnapshot, open, + 'Module loader bypassed the named external Git boundary'); +await import('./server-child.js'); diff --git a/server/src/__tests__/helpers/maintenance/default-cadence-network.ts b/server/src/__tests__/helpers/maintenance/default-cadence-network.ts new file mode 100644 index 00000000000..5a28c17ca58 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/default-cadence-network.ts @@ -0,0 +1,12 @@ +type GuardStats = { loaded: true; identity: string; sourceHash: string; preload: string; pid: number; ppid: number; unexpectedAttempts: string[]; peakPgSockets: number; remainingPgSockets: number }; +type GuardState = { stats(): GuardStats; allowOrigin(origin: string): void; allowDatabase(url: string): void }; +function current(): GuardState { + const state = (globalThis as unknown as Record)[Symbol.for('paperclip.skill.default-cadence.prewire')]; + if (!state || state.stats().pid !== process.pid || state.stats().identity !== 'skill-default-cadence-guard-v1') { + throw new Error('Default cadence requires the checked-in preload before module operations'); + } + return state; +} +export function installDefaultCadenceNetwork() { current(); } +export function defaultCadenceNetwork() { const stats = current().stats(); return { ...stats, offhostAttempts: stats.unexpectedAttempts }; } +export function allowDefaultCadenceOrigin(origin: string) { current().allowOrigin(origin); } diff --git a/server/src/__tests__/helpers/maintenance/server-harness.ts b/server/src/__tests__/helpers/maintenance/server-harness.ts index 15e1c6ca04e..13d015e6b7f 100644 --- a/server/src/__tests__/helpers/maintenance/server-harness.ts +++ b/server/src/__tests__/helpers/maintenance/server-harness.ts @@ -61,6 +61,9 @@ export interface MaintenanceConfiguration { storageProvider?: "local_disk" | "s3"; } export interface MaintenanceServerOptions { + childEntry?: URL; + childPreload?: URL; + databasePoolMax?: number; artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string; instanceId?: string }; cloud?: { jwks: string; s3Origin: string; standby: boolean }; @@ -71,6 +74,9 @@ export interface MaintenanceServerOptions { } export async function startMaintenanceServer(databaseUrl: string, options: MaintenanceServerOptions = {}) { const configuration = options.configuration; + if (options.databasePoolMax !== undefined && (!Number.isInteger(options.databasePoolMax) || options.databasePoolMax < 1 || options.databasePoolMax > 4)) { + throw new Error('Maintenance database pool must be an integer from 1 through 4'); + } if (options.captureIdleWork && options.clock) throw new Error("Passive setup diagnostics require the real clock"); if (options.tenantActor && (!options.cloud || !configuration?.managedInstance || configuration.deploymentMode !== 'authenticated' || configuration.storageProvider !== 's3')) { throw new Error('Managed tenant fixture requires the supported authenticated S3 profile'); @@ -132,18 +138,22 @@ export async function startMaintenanceServer(databaseUrl: string, options: Maint const captureStartedAt = options.captureIdleWork ? Date.now() : undefined; const captureSources = options.captureIdleWork ? new Map() : undefined; let settledDiagnosticId = 0; - const child = fork(fileURLToPath(new URL("./server-child.ts", import.meta.url)), [], { + const child = fork(fileURLToPath(options.childEntry ?? new URL("./server-child.ts", import.meta.url)), [], { cwd: fileURLToPath(new URL("../../../../..", import.meta.url)), - execArgv: [...((configuration?.announcements || options.captureIdleWork) ? ['--experimental-test-module-mocks'] : []), "--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], + execArgv: [...(options.childPreload ? ["--require", fileURLToPath(options.childPreload)] : []), ...((configuration?.announcements || options.captureIdleWork || options.childEntry) ? ['--experimental-test-module-mocks'] : []), "--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], env: { ...(captureStartedAt === undefined ? {} : { PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE: '1', PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW: `${new Date(captureStartedAt).toISOString()}/${new Date(captureStartedAt + 60_000).toISOString()}`, }), + ...(options.childPreload ? { + PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG: process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG, + PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_ORIGINS: JSON.stringify([options.githubOrigin, options.cloud?.s3Origin].filter(Boolean)), + } : {}), PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: "production", PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test", PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"), - DATABASE_URL: proxy.url, DATABASE_POOL_MAX: configuration ? "4" : "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", + DATABASE_URL: proxy.url, DATABASE_POOL_MAX: options.databasePoolMax === undefined ? (configuration ? "4" : "6") : String(options.databasePoolMax), DATABASE_APPLICATION_NAME: "maintenance-integration-test", PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: String(configuration?.announcements ?? false), @@ -180,6 +190,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: Maint const log = createWriteStream(path.join(directory, "server.log")); stopLog = () => new Promise(resolve => log.end(resolve)); let serverReady = false; + let childGuard: unknown; child.stdout?.on("data", chunk => { output += String(chunk); log.write(chunk); }); child.stderr?.on("data", chunk => { output += String(chunk); log.write(chunk); }); const requests = new Map void; reject: (error: Error) => void }>(); @@ -187,7 +198,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: Maint const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; }); const exited = new Promise(resolve => child.once("exit", () => resolve())); child.on("message", (message: any) => { - if (message.kind === "ready") { serverReady = true; readyResolve(message.port); } + if (message.kind === "ready") { childGuard = message.defaultCadenceGuard; serverReady = true; readyResolve(message.port); } else if (message.id) { const request = requests.get(message.id); requests.delete(message.id); if (message.kind === "error") request?.reject(new Error(message.error)); else request?.resolve(message); @@ -227,7 +238,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: Maint }; const port = await withDeadline(ready, 90_000, () => `Maintenance boot timed out: ${output.slice(-12_000)}`); return { - proxy, directory, retainedDirectory, port, stop, + proxy, directory, retainedDirectory, port, stop, pid: child.pid, get childGuard() { return childGuard; }, async api(method: string, route: string, body?: unknown, headers: Record = {}): Promise { if (options.clock) await request("realtime"); try { diff --git a/server/src/__tests__/maintenance-postgres-proxy.test.ts b/server/src/__tests__/maintenance-postgres-proxy.test.ts index c09e99b0a5c..3a79110f35f 100644 --- a/server/src/__tests__/maintenance-postgres-proxy.test.ts +++ b/server/src/__tests__/maintenance-postgres-proxy.test.ts @@ -58,7 +58,9 @@ describe("passive maintenance PostgreSQL observer", () => { let databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_MAINTENANCE_TEST_DATABASE_URL"); let temporary: Awaited> | undefined; -const databaseSupport = await maintenanceDatabaseSupport(databaseUrl); +const databaseSupport = process.env.PAPERCLIP_MAINTENANCE_PG0_CONTROLS === '1' + ? { supported: false, reason: 'Explicit PG0 controls defer the complete real-PostgreSQL observer suite without probing native support.' } + : await maintenanceDatabaseSupport(databaseUrl); const describeDatabase = databaseSupport.supported ? describe : describe.skip; if (!databaseSupport.supported) console.warn(`Skipping maintenance PostgreSQL tests: ${databaseSupport.reason ?? "unsupported environment"}`); describeDatabase("maintenance observer against PostgreSQL", () => { diff --git a/server/src/__tests__/skill-default-cadence-prewire.test.ts b/server/src/__tests__/skill-default-cadence-prewire.test.ts new file mode 100644 index 00000000000..289700d2e8c --- /dev/null +++ b/server/src/__tests__/skill-default-cadence-prewire.test.ts @@ -0,0 +1,196 @@ +import { threadId, isMainThread } from 'node:worker_threads'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; + +const guard = fileURLToPath(new URL('../../../scripts/fixtures/skill-default-cadence-network-guard.cjs', import.meta.url)); +const launcher = fileURLToPath(new URL('../../../scripts/test-skill-default-cadence.mjs', import.meta.url)); +const prewireRequested = Object.hasOwn(process.env, 'PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG') + || process.env.PAPERCLIP_MAINTENANCE_PG0_CONTROLS === '1' + || process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_ACCEPTANCE === '1'; +it.skipIf(!prewireRequested)('the actual Vitest worker is preloaded with its own PID before fixture execution', () => { + const guardState = (globalThis as any)[Symbol.for('paperclip.skill.default-cadence.prewire')]; + expect(guardState).toBeDefined(); + const observed = guardState.stats(); + expect(observed.pid).toBe(process.pid); expect(observed.preload).toBe(guard); + expect(observed.threadId).toBe(threadId); expect(observed.isMainThread).toBe(isMainThread); + const receipts = readFileSync(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG!, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + expect(receipts).toContainEqual(expect.objectContaining({ event: 'preloaded', pid: process.pid, threadId, isMainThread, sourceHash: observed.sourceHash })); + expect(observed.unexpectedAttempts).toEqual([]); + console.info(JSON.stringify({ actualVitestWorkerGuard: observed })); +}); +function child(code: string, extra: Record = {}, nodeArgs: string[] = []) { + const root = mkdtempSync(path.join(tmpdir(), 'skill-prewire-')); + const log = path.join(root, 'guard.jsonl'); + try { + const result = spawnSync(process.execPath, ['--require', guard, ...nodeArgs, '-e', code], { timeout: 5_000, encoding: 'utf8', + env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: root, NODE_OPTIONS: '', PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG: log, ...extra } }); + const events = readFileSync(log, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + return { result, events }; + } finally { rmSync(root, { recursive: true, force: true }); } +} +const stats = "globalThis[Symbol.for('paperclip.skill.default-cadence.prewire')].stats()"; +it.each([ + "require('node:http').get('http://example.invalid/')", + "require('node:https').get('https://example.invalid/')", + "require('node:net').connect(443,'example.invalid')", + "require('node:tls').connect(443,'example.invalid')", + "fetch('https://example.invalid/')", + "require('node:dns').lookup('example.invalid',()=>{})", + "require('node:dns').promises.resolve4('example.invalid')", + "new(require('node:net').Socket)().connect([{host:'example.invalid',port:443},()=>{}])", + "require('node:net').connect(55448,'127.0.0.1')", + "require('node:net').connect(require('node:path').join(require('node:os').tmpdir(),'tsx-'+process.getuid(),(process.ppid+12345)+'.pipe'))", +])('preload rejects and records a caught unowned transport: %s', operation => { + const { result, events } = child(`try { ${operation}; } catch {} console.log(JSON.stringify(${stats}));`); + expect(result.error).toBeUndefined(); expect(result.status).toBe(98); + const observed = JSON.parse(result.stdout.trim()); + expect(observed.pid).toBe(result.pid); expect(observed.unexpectedAttempts).toHaveLength(1); + expect(observed.peakPgSockets).toBe(0); + expect(events[0]).toMatchObject({ event: 'preloaded', pid: result.pid }); + expect(events.filter(event => event.event === 'blocked')).toHaveLength(1); + expect(events.at(-1)).toMatchObject({ event: 'exit', exitCode: 98 }); +}); +it('checks an overriding HTTP hostname even when the URL names a real owned listener', () => { + const { result, events } = child(`const http=require('node:http');let dnsCalls=0; + const server=http.createServer();server.listen(0,'127.0.0.1',()=>{ + require('node:dns').lookup=()=>{dnsCalls++;throw Error('DNS must not run');}; + try{http.get(new URL('http://127.0.0.1:'+server.address().port),{hostname:'example.invalid'});}catch{} + server.close(()=>console.log(JSON.stringify({...${stats},dnsCalls}))); + });`); + expect(result.error).toBeUndefined(); expect(result.status).toBe(98); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ dnsCalls: 0, unexpectedAttempts: ['offhost example.invalid'] }); + expect(events.filter(event => event.event === 'blocked')).toHaveLength(1); +}); +it('never permits an existing Unix path even for the exact current parent', () => { + const { result, events } = child(`const fs=require('node:fs'),path=require('node:path'),os=require('node:os'); + const dir=path.join(os.tmpdir(),'tsx-'+process.getuid());fs.mkdirSync(dir,{recursive:true}); + const pipe=path.join(dir,process.ppid+'.pipe');fs.writeFileSync(pipe,'not a socket'); + try{require('node:net').connect(pipe);}catch{}console.log(JSON.stringify(${stats}));`); + expect(result.error).toBeUndefined(); expect(result.status).toBe(98); + expect(events.filter(event => event.event === 'blocked')).toHaveLength(1); +}); +it('permits actual owned loopback HTTP only after the real listener binds', () => { + const { result, events } = child(`const http=require('node:http'); const server=http.createServer((req,res)=>res.end('useful response')); + server.listen(0,'127.0.0.1',()=>http.get('http://127.0.0.1:'+server.address().port,res=>{ + let body='';res.on('data',chunk=>body+=chunk);res.on('end',()=>server.close(()=>{ + if(body!=='useful response')throw Error('missing useful body');console.log(JSON.stringify(${stats})); + })); + }));`); + expect(result.error).toBeUndefined(); expect(result.status).toBe(0); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ pid: result.pid, loaded: true, unexpectedAttempts: [] }); + expect(events.at(-1)).toMatchObject({ event: 'exit', exitCode: 0 }); +}); +it('permits only an absent current live-parent TSX probe and records the actual ENOENT', () => { + const { result, events } = child(`const path=require('node:path'),os=require('node:os'); + const socket=require('node:net').connect(path.join(os.tmpdir(),'tsx-'+process.getuid(),process.ppid+'.pipe')); + socket.on('error',error=>{if(error.code!=='ENOENT')throw error;console.log(JSON.stringify(${stats}));});`); + expect(result.error).toBeUndefined(); expect(result.status).toBe(0); + expect(JSON.parse(result.stdout.trim()).unexpectedAttempts).toEqual([]); + expect(events.filter(event => event.event === 'current-parent-pipe-ENOENT')).toHaveLength(1); +}); +it.each([{}, + { PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL: 'postgres://u@127.0.0.1/live', PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL: 'postgres://u@127.0.0.1/cloud_test' }, + { PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL: 'postgres://u@127.0.0.1/same_test', PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL: 'postgres://u@127.0.0.1/same_test' }, + { PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL: 'postgres://u@example.invalid/app_test', PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL: 'postgres://u@127.0.0.1/cloud_test' }, +])('launcher refuses absent/unsafe settings before loading Vitest', settings => { + const result = spawnSync(process.execPath, [launcher], { timeout: 5_000, encoding: 'utf8', env: { PATH: process.env.PATH, NODE_OPTIONS: '', ...settings } }); + expect(result.error).toBeUndefined(); expect(result.status).toBe(1); + expect(result.stderr).toMatch(/test database|loopback|distinct/); expect(result.stdout).not.toContain('RUN'); +}); + +it('permits approved HTTP options with a request path and default port through an owned agent', () => { + const { result } = child(`const http=require('node:http'),net=require('node:net'); + const state=globalThis[Symbol.for('paperclip.skill.default-cadence.prewire')];let agentCalls=0; + const server=http.createServer((req,res)=>res.end('approved options response')); + server.listen(0,'127.0.0.1',()=>{ + state.allowOrigin('http://127.0.0.1'); + const agent=new http.Agent();agent.createConnection=()=>{agentCalls++;return net.connect(server.address().port,'127.0.0.1');}; + http.get({host:'127.0.0.1',path:'/useful',agent},res=>{let body='';res.on('data',chunk=>body+=chunk); + res.on('end',()=>{agent.destroy();server.close(()=>console.log(JSON.stringify({...state.stats(),body,agentCalls})));});}); + });`); + expect(result.error).toBeUndefined(); expect(result.status).toBe(0); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ body: 'approved options response', agentCalls: 1, unexpectedAttempts: [] }); +}); +it.each([ + ['unknown.example.invalid', 'offhost unknown.example.invalid'], + ['127.0.0.1', 'unowned loopback 127.0.0.1:80'], +])('rejects an unknown HTTP options origin before agent work: %s', (host, reason) => { + const { result } = child(`const http=require('node:http');let agentCalls=0;const agent=new http.Agent(); + agent.createConnection=()=>{agentCalls++;throw Error('Agent must not run');}; + try{http.get({host:${JSON.stringify(host)},path:'/',agent});}catch{} + console.log(JSON.stringify({...${stats},agentCalls}));`); + expect(result.error).toBeUndefined(); expect(result.status).toBe(98); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ agentCalls: 0, unexpectedAttempts: [reason] }); +}); +it('joins an admitted deferred wake and closes its real child listener before bridge close returns', () => { + const bridgeSource = process.env.PAPERCLIP_SKILL_BRIDGE_CONTROL_SOURCE + ?? fileURLToPath(new URL('./helpers/maintenance/default-cadence-bridge.ts', import.meta.url)); + const networkSource = fileURLToPath(new URL('./helpers/maintenance/default-cadence-network.ts', import.meta.url)); + const loader = fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)); + const { result, events } = child(String.raw`(async()=>{ + const assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),os=require('node:os'); + const {fork}=require('node:child_process'),{pathToFileURL}=require('node:url'); + const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); + const ipcErrors=[];process.on('uncaughtException',error=>{if(error.code!=='ERR_IPC_CHANNEL_CLOSED')throw error;ipcErrors.push({code:error.code,message:error.message});}); + const root=fs.mkdtempSync(path.join(os.tmpdir(),'bridge-close-')),app=path.join(root,'app'); + const helpers=path.join(app,'server/src/__tests__/helpers/maintenance');fs.mkdirSync(helpers,{recursive:true}); + fs.writeFileSync(path.join(app,'package.json'),JSON.stringify({type:'module'})); + const source=path.join(helpers,'default-cadence-bridge.ts');fs.copyFileSync(process.env.BRIDGE_SOURCE,source); + fs.copyFileSync(process.env.NETWORK_SOURCE,path.join(helpers,'default-cadence-network.ts')); + const preload=path.join(app,'scripts/fixtures/skill-default-cadence-network-guard.cjs');fs.mkdirSync(path.dirname(preload),{recursive:true});fs.copyFileSync(process.env.GUARD_SOURCE,preload); + const checkout=path.join(root,'cloud'),entry=path.join(checkout,'dist/sleep/fixtures/default-cadence-bridge.js');fs.mkdirSync(path.dirname(entry),{recursive:true}); + fs.writeFileSync(entry,[ + "process.on('message',message=>{if(message.method==='trigger'){process.send({callbackId:'wake',physical:'wake'},()=>process.exit(0));}else if(message.method==='close'){process.send({id:message.id,result:true});}});", + "process.send({ready:true,pid:process.pid,defaultCadenceGuard:globalThis[Symbol.for('paperclip.skill.default-cadence.prewire')].stats()});" + ].join('\n')); + const physicalFile=path.join(root,'physical-app.cjs');fs.writeFileSync(physicalFile,[ + "const http=require('node:http');const server=http.createServer((req,res)=>res.end('late real child'));", + "server.listen(0,'127.0.0.1',()=>process.send({origin:'http://127.0.0.1:'+server.address().port}));", + "process.on('message',message=>{if(message.stop)server.close(()=>process.disconnect());});" + ].join('\n')); + let release,start,finish,fail,physical,physicalExit,origin,body,physicalOutput='',stops=0; + const released=new Promise(resolve=>{release=resolve;}),started=new Promise(resolve=>{start=resolve;}),finished=new Promise((resolve,reject)=>{finish=resolve;fail=reject;}); + const state=globalThis[Symbol.for('paperclip.skill.default-cadence.prewire')]; + const stop=async()=>{stops++;if(physical&&physical.exitCode===null){ + if(physical.connected)physical.send({stop:true});else physical.kill('SIGTERM'); + await Promise.race([physicalExit,delay(1000).then(async()=>{if(physical.exitCode===null)physical.kill('SIGKILL');await physicalExit;})]); + }}; + const {startDefaultCadenceBridge}=await import(pathToFileURL(source).href);let bridge; + try{ + bridge=await startDefaultCadenceBridge({checkout,node:process.execPath,physicalStop:stop,physicalWake:async()=>{ + start();await released; + try{ + physical=fork(physicalFile,[],{execArgv:['--require',preload],env:process.env,stdio:['ignore','ignore','pipe','ipc']}); + physical.stderr.on('data',bytes=>{physicalOutput+=String(bytes);}); + physicalExit=new Promise(resolve=>physical.once('exit',resolve)); + origin=await new Promise((resolve,reject)=>{physical.once('message',message=>resolve(message.origin));physical.once('error',reject);physical.once('exit',code=>reject(Error('Physical child exited before origin: '+code+' '+physicalOutput)));}); + state.allowOrigin(origin);body=await(await fetch(origin)).text();finish();return{origin}; + }catch(error){fail(error);throw error;} + }}); + const trigger=bridge.rpc('trigger').then(()=>null,error=>String(error));await started;await trigger; + const closing=bridge.close().then(()=>({ok:true}),error=>({ok:false,error:String(error)})); + const closedEarly=await Promise.race([closing.then(()=>true),delay(40).then(()=>false)]); + release();await finished;const closed=await closing; + let reachable=false;try{reachable=(await fetch(origin)).ok;}catch{} + console.log(JSON.stringify({...state.stats(),body,closedEarly,closed,reachable,stops,origin,ipcErrors,physicalPid:physical.pid,physicalExitCode:physical.exitCode})); + assert.equal(body,'late real child');assert.equal(reachable,false,'Late child listener remained reachable after bridge.close'); + assert.equal(physical.exitCode,0);assert.equal(stops,1);assert.equal(closedEarly,false);assert.equal(closed.ok,true);assert.deepEqual(ipcErrors,[]); + }finally{ + release();if(physical&&physical.exitCode===null)await stop();fs.rmSync(root,{recursive:true,force:true}); + } + })().catch(error=>{console.error(error.stack);process.exitCode=1;});`, { + BRIDGE_SOURCE: bridgeSource, NETWORK_SOURCE: networkSource, GUARD_SOURCE: guard, + }, ['--import', loader]); + const diagnostics = JSON.stringify({ stdout: result.stdout, stderr: result.stderr, events }); + expect(result.error, diagnostics).toBeUndefined(); expect(result.status, diagnostics).toBe(0); + const observation = result.stdout.trim().split('\n').at(-1)!; + expect(observation, diagnostics).toMatch(/^\{/); + console.info(JSON.stringify({ cleanupObservation: JSON.parse(observation), guardEvents: events })); + expect(JSON.parse(observation)).toMatchObject({ + body: 'late real child', closedEarly: false, closed: { ok: true }, reachable: false, stops: 1, physicalExitCode: 0, ipcErrors: [], unexpectedAttempts: [], + }); +}); diff --git a/server/src/__tests__/skill-source-default-cadence-acceptance.test.ts b/server/src/__tests__/skill-source-default-cadence-acceptance.test.ts new file mode 100644 index 00000000000..86617fcc334 --- /dev/null +++ b/server/src/__tests__/skill-source-default-cadence-acceptance.test.ts @@ -0,0 +1,186 @@ +import { defaultCadenceNetwork, installDefaultCadenceNetwork, allowDefaultCadenceOrigin } from './helpers/maintenance/default-cadence-network.js'; +import assert from 'node:assert/strict'; +import { generateKeyPairSync } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { describe, expect, it } from 'vitest'; +import { applyPendingMigrations, createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { githubFixture } from './helpers/github-skills.js'; +import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { provisionManagedSleepCompany, managedTenantActor } from './helpers/maintenance/managed-profile.js'; +import { startDefaultCadenceBridge } from './helpers/maintenance/default-cadence-bridge.js'; + +const enabled = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_ACCEPTANCE === '1'; +const stage = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_STAGE ?? 'two-cycle'; +if (enabled && !['bootstrap', 'two-cycle'].includes(stage)) throw new Error('Default cadence stage must be bootstrap or two-cycle'); +if (enabled) installDefaultCadenceNetwork(); +const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL'); +const cloudDatabaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL'); +const cloudCheckout = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT; +const cloudSha = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA; +const cloudNode = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE; +const evidence = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE; +if (enabled && (!databaseUrl || !cloudDatabaseUrl || !cloudCheckout || !cloudNode || !evidence || !/^[a-f0-9]{40}$/.test(cloudSha ?? '') + || process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1')) throw new Error('Default cadence witness requires two explicit owned DBs, pinned Cloud checkout/Node22, and retained evidence'); + + + +describe.skipIf(!enabled)('default skill cadence actual signed host handoff', () => { + it(stage === 'bootstrap' ? 'qualifies actual producer, signed legacy refusal and original-D durable ACK only' + : 'sleeps and cold-reactivates twice under each original default300s deadline', async () => { + assert.equal(execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), cloudSha); + assert.match(execFileSync(cloudNode!, ['--version'], { encoding: 'utf8' }), /^v22\./); + await applyPendingMigrations(databaseUrl!); + + let upstream = 1; + const files = () => ({ 'workflow/SKILL.md': `---\nname: default-cadence\ndescription: Useful default cadence workflow\n---\nPersisted useful revision ${upstream}.\n` }); + const commit = () => String(upstream).repeat(40); + const requests: Array<{ at: number; path: string }> = []; + const upstreamServer = createServer(async (req, res) => { + requests.push({ at: Date.now(), path: req.url! }); + try { + const body = req.url === '/__skill_snapshot' ? { files: files(), commit: commit() } + : await githubFixture(files(), {}, commit())(req.url!); + res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify(body)); + } catch (error) { res.statusCode = 500; res.end(JSON.stringify({ error: String(error) })); } + }); + const s3 = createMaintenanceS3Boundary(); + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'default-cadence', alg: 'EdDSA', use: 'sig' }; + const privateJwk = JSON.stringify({ ...pair.privateKey.export({ format: 'jwk' }), kid: jwk.kid, alg: 'EdDSA', use: 'sig' }); + const jwks = JSON.stringify({ keys: [jwk] }); + const cycles: unknown[] = []; + const childPids: number[] = []; + let server: Awaited> | undefined; + let bridge: Awaited> | undefined; + try { + await withMaintenanceProvider(s3.server, s3Origin => withMaintenanceProvider(upstreamServer, async githubOrigin => { + try { + const boot = async () => { + const started = await startMaintenanceServer(databaseUrl!, { + childEntry: new URL('./helpers/maintenance/default-cadence-child.ts', import.meta.url), + childPreload: new URL('../../../scripts/fixtures/skill-default-cadence-network-guard.cjs', import.meta.url), + databasePoolMax: 2, artifactRoot: evidence, githubOrigin, clock: false, controlJwks: jwks, + cloud: { jwks, s3Origin, standby: false }, tenantActor: managedTenantActor, + configuration: { managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3', announcements: false }, + }); + assert(started.pid); childPids.push(started.pid); + const guard = started.childGuard as ReturnType; + expect(guard.loaded).toBe(true); expect(guard.pid).toBe(started.pid); expect(guard.ppid).toBe(process.pid); + expect(guard.sourceHash).toBe(defaultCadenceNetwork().sourceHash); + allowDefaultCadenceOrigin(`http://127.0.0.1:${started.port}`); + return started; + }; + server = await boot(); + + + const setupDb = createDb(databaseUrl!, { maxConnections: 1 }); + const company = await provisionManagedSleepCompany(server, setupDb).finally(() => setupDb.$client.end({ timeout: 1 })); + await server.settled(); + bridge = await startDefaultCadenceBridge({ checkout: cloudCheckout!, node: cloudNode!, + physicalStop: async () => { assert(server); await server.stop(); server = undefined; upstream++; }, + physicalWake: async () => { assert(!server); server = await boot(); return { origin: `http://127.0.0.1:${server.port}` }; } }); + assert(bridge.pid); childPids.push(bridge.pid); + await bridge.rpc('initialize', { databaseUrl: cloudDatabaseUrl, migrationsPath: path.join(cloudCheckout!, 'migrations'), + appOrigin: `http://127.0.0.1:${server.port}`, privateJwk, jwks }); + await server.settled(); + + await server.api('POST', `/companies/${company.id}/projects`, { name: 'Default cadence source', + workspace: { name: 'Repository', repoUrl: 'https://github.com/acme/skills', defaultRef: 'HEAD' } }); + const persisted = async () => { + const observerUrl = new URL(databaseUrl!); observerUrl.searchParams.set('application_name', 'skill-default-readonly-observer'); + const observer = createDb(observerUrl.href, { maxConnections: 1 }); + try { + const rows = await observer.$client.begin('read only', tx => tx`select s.id,s.current_version_id,s.markdown,c.next_sync_at,c.last_success_at,c.last_scan_commit,c.lease_token,c.automatic_updates,c.enabled + from company_skills s join company_skill_source_entries e on e.skill_id=s.id and e.company_id=s.company_id + join company_skill_sources c on c.id=e.source_id and c.company_id=s.company_id where s.company_id=${company.id}`); + assert.equal(rows.length, 1); return rows[0]!; + } finally { await observer.$client.end({ timeout: 1 }); } + }; + await expect.poll(async () => { + try { return (await persisted()).markdown; } catch { return ''; } + }, { timeout: 30_000 }).toContain('Persisted useful revision 1.'); + const initial = await persisted(); + let priorVersion = initial.current_version_id; + const legacy = await bridge.rpc('legacy'); + expect(legacy.idleSleepSafe).not.toBe(true); + for (let cycle = 1; cycle <= 2; cycle++) { + assert(server); + await server.settled(); + const source = await persisted(); + const originalD = new Date(source.next_sync_at).getTime(); + expect(originalD - new Date(source.last_success_at).getTime()).toBeGreaterThanOrEqual(300_000); + expect(originalD - new Date(source.last_success_at).getTime()).toBeLessThan(300_050); + expect(source.automatic_updates && source.enabled).toBe(true); + expect(source.lease_token).toBeNull(); + const refreshRequestStart = requests.length; + const sleep = await bridge.rpc('sleep'); + expect(server).toBeUndefined(); + expect(new Date(sleep.alarm.next_wake_at).getTime()).toBe(originalD); + const wakeAt = new Date(sleep.alarm.next_attempt_at).getTime(); + expect(wakeAt).toBe(originalD - 120_000); + await writeFile(path.join(evidence!, `cycle-${cycle}-durable-ack.json`), JSON.stringify({ originalD, sleep, requests }, null, 2)); + if (stage === 'bootstrap') { + await writeFile(path.join(evidence!, 'bootstrap-only-receipt.json'), JSON.stringify({ stage, originalD, sleep, + initial, requests, boundary: 'Actual initial useful producer/signed gate/durable ACK/physical child stop only. No due wake or second cycle executed.' }, null, 2)); + return; + } + + await delay(Math.max(0, wakeAt - Date.now()) + 25); + const wake = await bridge.rpc('wake'); + assert(server, 'Actual controller provider callback must cold-start a new child'); + await server.settled(); + + const before = await persisted(); + expect(before.current_version_id).toBe(priorVersion); + const snapBefore = await server.snapshot(), queryCount = server.proxy.queries.length, providerCount = requests.length; + await delay(10_000); + const snapAfter = await server.snapshot(); + expect(server.proxy.queries.length).toBe(queryCount); + expect(requests.length).toBe(providerCount); + expect(snapAfter.outgoingHttpRequests).toBe(snapBefore.outgoingHttpRequests); + await delay(Math.max(0, originalD - Date.now()) + 25); + await expect.poll(async () => (await persisted()).markdown, { timeout: 30_000 }).toContain(`Persisted useful revision ${cycle + 1}.`); + const changed = await persisted(); + expect(changed.current_version_id).not.toBe(priorVersion); + expect(new Date(changed.last_success_at).getTime()).toBeGreaterThanOrEqual(originalD); + const refreshRequests = requests.slice(refreshRequestStart); + expect(refreshRequests.length).toBeGreaterThan(0); + expect(Math.min(...refreshRequests.map(request => request.at))).toBeGreaterThanOrEqual(originalD); + priorVersion = changed.current_version_id; + await server.settled(); + cycles.push({ cycle, originalD, wakeAt, sleep, wake, changed, refreshRequestStart, refreshRequests, quiet: { snapBefore, snapAfter, queryCount, providerCount } }); + await writeFile(path.join(evidence!, 'two-cycle-receipt.json'), JSON.stringify({ cycles, requests, s3: s3.snapshot(), + boundary: 'Actual app index/routes/source owner and signed Cloud gate/controller/Postgres alarms; controlled localhost Git/S3/process stop only; no Railway/Neon claim' }, null, 2)); + } + } catch (error) { + await writeFile(path.join(evidence!, 'primary-failure-before-cleanup.json'), JSON.stringify({ + error: error instanceof Error ? error.stack : String(error), bridgeOutput: bridge?.output, serverOutput: server?.output, + }, null, 2)); + throw error; + } finally { + try { const current = bridge; bridge = undefined; await current?.close(); } + finally { const current = server; server = undefined; await current?.stop(); } + } + })); + } catch (error) { + await writeFile(path.join(evidence!, 'attempt-failure.json'), JSON.stringify({ at: new Date().toISOString(), + error: error instanceof Error ? error.stack : String(error), cycles, requests, bridgeOutput: bridge?.output, + serverOutput: server?.output, boundary: 'Actual first failure; no automatic retry or assertion/time policy change' }, null, 2)); + throw error; + } finally { + try { await bridge?.close(); } finally { await server?.stop(); } + expect(defaultCadenceNetwork().offhostAttempts).toEqual([]); + const events = (await readFile(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG!, 'utf8')).trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + expect(events.filter(event => event.event === 'blocked')).toEqual([]); + for (const pid of [process.pid, process.ppid, ...childPids]) { + expect(events).toContainEqual(expect.objectContaining({ event: 'preloaded', pid, sourceHash: defaultCadenceNetwork().sourceHash })); + } + } + }, 850_000); +}); diff --git a/server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts b/server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts index 403963ec504..2335df20e67 100644 --- a/server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts @@ -128,7 +128,6 @@ describeEmbeddedPostgres("native ingress bootstrap retry through the real Runner const address = portProbe.address(); if (!address || typeof address === "string") throw new Error("Missing fixture port"); const runnerPort = address.port; - await new Promise(done => portProbe.close(() => done())); let healthy = false, acquisitions = 0, previewUrl = ""; const preview = createHttpsServer({ key: await readFile(key), cert: await readFile(certificate) }, (request, response) => { if (request.url !== "/preview") { response.writeHead(404).end(); return; } @@ -159,9 +158,12 @@ describeEmbeddedPostgres("native ingress bootstrap retry through the real Runner }); }); preview.on("tlsClientError", error => { diagnostics.push(`fixture TLS: ${error.message}`); }); - await new Promise(done => preview.listen(0, "127.0.0.1", done)); + try { + await new Promise(done => preview.listen(0, "127.0.0.1", done)); + } finally { await new Promise(done => portProbe.close(() => done())); } const previewAddress = preview.address(); if (!previewAddress || typeof previewAddress === "string") throw new Error("Missing preview address"); + expect(previewAddress.port).not.toBe(runnerPort); previewUrl = `wss://127.0.0.1:${previewAddress.port}/api/runner/v1/connect/${runId}`; const bundles: ReturnType[] = []; const handles: RunnerProcessHandle[] = [];