diff --git a/scripts/__tests__/status-object-signed-sleep-preflight.test.mjs b/scripts/__tests__/status-object-signed-sleep-preflight.test.mjs new file mode 100644 index 00000000000..2af40900ca5 --- /dev/null +++ b/scripts/__tests__/status-object-signed-sleep-preflight.test.mjs @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +const launcher = path.join(root, 'scripts/test-status-object-signed-sleep.mjs'); +const boundarySource = ` +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const { appendFileSync } = require('node:fs'); +const { syncBuiltinESMExports } = require('node:module'); +require('node:net').Socket.prototype.connect = function() { + appendFileSync(process.env.PREFLIGHT_NETWORK_RECEIPT, 'blocked\\n'); + process.exitCode = 98; + throw new Error('Preflight controls forbid all sockets'); +}; +require('node:child_process').spawn = function(command, args, options) { + assert.equal(command, process.execPath); + const path = require('node:path'); + const expectedRoot = process.env.PREFLIGHT_EXPECTED_ROOT; + const guard = path.join(expectedRoot, 'scripts/fixtures/skill-default-cadence-network-guard.cjs'); + const vitest = path.join(path.dirname(require.resolve('vitest/package.json', { + paths: [path.join(expectedRoot, 'server')], + })), 'vitest.mjs'); + assert.deepEqual(args, ['--require', guard, vitest, 'run', + 'server/src/__tests__/status-object-signed-sleep-acceptance.test.ts', + '--maxWorkers=1', '--no-file-parallelism', '--execArgv=--require=' + guard]); + assert.equal(options.cwd, expectedRoot); + assert.equal(options.env.NODE_OPTIONS, '--require=' + guard); + appendFileSync(process.env.PREFLIGHT_SPAWN_RECEIPT, 'vitest-boundary\\n'); + const child = new EventEmitter(); + child.kill = () => false; + queueMicrotask(() => child.emit('close', 0, null)); + return child; +}; +syncBuiltinESMExports(); +`; + +function fixture(run) { + const directory = mkdtempSync(path.join(os.tmpdir(), 'status-owner-preflight-')); + const evidence = path.join(directory, 'evidence'), cloud = path.join(directory, 'cloud'); + mkdirSync(evidence); + mkdirSync(path.join(cloud, 'dist/sleep/fixtures'), { recursive: true }); + writeFileSync(path.join(cloud, 'dist/sleep/fixtures/default-cadence-bridge.js'), ''); + const preload = path.join(directory, 'boundary.cjs'), log = path.join(directory, 'guard.jsonl'); + const spawnReceipt = path.join(directory, 'spawn.txt'), networkReceipt = path.join(directory, 'network.txt'); + writeFileSync(preload, boundarySource); + writeFileSync(log, ''); + const env = { + PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, + PREFLIGHT_EXPECTED_ROOT: root, PREFLIGHT_SPAWN_RECEIPT: spawnReceipt, PREFLIGHT_NETWORK_RECEIPT: networkReceipt, + PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL: 'postgres://paperclip@127.0.0.1:55448/owner_preflight_app_test', + PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL: 'postgres://paperclip@127.0.0.1:55448/owner_preflight_cloud_test', + PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT: cloud, PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE: process.execPath, + PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA: '0'.repeat(40), + PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE: evidence, PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG: log, + }; + const execute = () => { + const result = spawnSync(process.execPath, ['--require', preload, launcher, '--owner=status', '--stage=identity'], { + cwd: root, env, encoding: 'utf8', timeout: 10_000, + }); + console.log(JSON.stringify({ event: 'launcher-preflight-child', pid: result.pid, status: result.status, + spawnEntered: existsSync(spawnReceipt), socketAttempted: existsSync(networkReceipt), + guardEvents: existsSync(env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG) + ? readFileSync(env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [] })); + assert.equal(result.error, undefined); + assert(!existsSync(networkReceipt), 'No socket may be entered during preflight controls'); + return result; + }; + try { run({ evidence, directory, log, env, execute, spawnReceipt }); } + finally { rmSync(directory, { recursive: true, force: true }); } +} + +function refused(context, message) { + const result = context.execute(); + assert.notEqual(result.status, 0); + assert.match(result.stderr, message); + assert(!existsSync(context.spawnReceipt), 'Refusal must precede the Vitest spawn boundary'); + assert.equal(readFileSync(context.log, 'utf8'), '', 'Refusal must precede guard installation'); +} + +test('fresh evidence reaches the actual launcher Vitest boundary without transport', () => fixture(context => { + const result = context.execute(); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(context.spawnReceipt, 'utf8'), 'vitest-boundary\n'); +})); + +test('fresh evidence may contain only its explicit empty regular guard log', () => fixture(context => { + const log = path.join(context.evidence, 'guard.jsonl'); + writeFileSync(log, ''); + context.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG = log; + const result = context.execute(); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(context.spawnReceipt, 'utf8'), 'vitest-boundary\n'); +})); + +for (const name of ['status-provider-state.json', 'status-provider-state.json.runs.jsonl', + 'status-provider-state.json.pending', 'status-provider-state.json.api-origin', 'other-evidence.json']) { + test(`rejects existing ${name} before guard or spawn and preserves its bytes`, () => fixture(context => { + const file = path.join(context.evidence, name), prior = 'retained evidence\n'; + writeFileSync(file, prior); + refused(context, /requires a fresh evidence directory/); + assert.equal(readFileSync(file, 'utf8'), prior); + })); +} + +test('rejects a symlink evidence directory before guard or spawn', () => fixture(context => { + const link = path.join(context.directory, 'evidence-link'); + symlinkSync(context.evidence, link); + context.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE = link; + refused(context, /evidence directory without a symlink/); + assert(lstatSync(link).isSymbolicLink()); +})); + +test('rejects a symlink guard log without writing its target', () => fixture(context => { + const link = path.join(context.evidence, 'guard-link.jsonl'); + symlinkSync(context.log, link); + context.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG = link; + refused(context, /fresh absolute guard log/); + assert(lstatSync(link).isSymbolicLink()); +})); diff --git a/scripts/test-status-object-signed-sleep.mjs b/scripts/test-status-object-signed-sleep.mjs new file mode 100644 index 00000000000..19a609d7e82 --- /dev/null +++ b/scripts/test-status-object-signed-sleep.mjs @@ -0,0 +1,76 @@ +import { createRequire } from 'node:module'; +import { spawn } from 'node:child_process'; +import { existsSync, lstatSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +const root = fileURLToPath(new URL('../', import.meta.url)); +const args = process.argv.slice(2); +if (args.length !== 2 || !/^--owner=(status|object)$/.test(args[0]) || !/^--stage=(bootstrap|cycle|identity)$/.test(args[1])) { + throw new Error('Use --owner=status|object --stage=bootstrap|cycle, or --owner=status --stage=identity'); +} +if (args[1] === '--stage=identity' && args[0] !== '--owner=status') throw new Error('Identity control requires the status owner'); +const diagnostic = process.env.PAPERCLIP_STATUS_OBJECT_SLEEP_DIAGNOSTIC; +const diagnosticOwner = args[0] === '--owner=object' && args[1] === '--stage=bootstrap' + || args[0] === '--owner=status' && args[1] === '--stage=identity'; +if (diagnostic !== undefined && (diagnostic !== '1' || !diagnosticOwner)) { + throw new Error('Diagnostic capture requires object bootstrap or status identity and the explicit value 1'); +} +if (!process.version.startsWith('v24.')) throw new Error('Use the actual Node24 executable'); +if (process.env.PAPERCLIP_MAINTENANCE_PG0_CONTROLS) throw new Error('PG0 flag is forbidden for signed owner acceptance'); +const names = ['PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL']; +const urls = names.map(name => { + const value = process.env[name]; + if (!value) throw new Error(`${name} requires an explicit dedicated database`); + const url = new URL(value); + if (!['postgres:', 'postgresql:'].includes(url.protocol) || url.hostname !== '127.0.0.1' || url.port !== '55448' + || !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1))) || url.search || url.hash) { + throw new Error(`${name} requires the exact owned loopback database without overrides`); + } + return url; +}); +if (urls[0].pathname === urls[1].pathname) throw new Error('App and Cloud fixtures must differ'); +if (process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL) { + const admin = new URL(process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL); + if (!['postgres:', 'postgresql:'].includes(admin.protocol) || admin.hostname !== '127.0.0.1' || admin.port !== '55448' + || admin.pathname !== '/postgres' || admin.search || admin.hash) throw new Error('Admin URL must be the same loopback55448/postgres cluster'); +} +const cloud = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT; +if (!cloud || !path.isAbsolute(cloud) || !existsSync(path.join(cloud, 'dist/sleep/fixtures/default-cadence-bridge.js'))) throw new Error('Set exact compiled Cloud checkout'); +if (!path.isAbsolute(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE || '')) throw new Error('Set exact Node22 executable'); +if (!/^[a-f0-9]{40}$/.test(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA || '')) throw new Error('Set exact Cloud source SHA'); +const evidence = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE; +if (!evidence || !path.isAbsolute(evidence) || !existsSync(evidence) || !lstatSync(evidence).isDirectory()) { + throw new Error('Set an existing absolute evidence directory without a symlink'); +} +const log = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG; +const logStat = log && path.isAbsolute(log) ? lstatSync(log, { throwIfNoEntry: false }) : undefined; +if (!log || !path.isAbsolute(log) || !existsSync(path.dirname(log)) + || logStat && (!logStat.isFile() || logStat.size !== 0)) throw new Error('Set a fresh absolute guard log'); +for (const entry of readdirSync(evidence)) { + const existing = path.join(evidence, entry); + if (path.resolve(existing) !== path.resolve(log) || !lstatSync(existing).isFile() || statSync(existing).size !== 0) { + throw new Error('Owner acceptance requires a fresh evidence directory; existing artifacts must be preserved'); + } +} +const require = createRequire(new URL('../server/package.json', import.meta.url)); +const guard = path.join(root, 'scripts/fixtures/skill-default-cadence-network-guard.cjs'); +delete process.env.DATABASE_URL; +delete process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_ORIGINS; +require(guard); +const vitest = path.join(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); +const env = { PATH: process.env.PATH, HOME: process.env.HOME, LANG: process.env.LANG, TMPDIR: process.env.TMPDIR, + NODE_ENV: 'test', NODE_OPTIONS: `--require=${guard}`, PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG: log, + PAPERCLIP_STATUS_OBJECT_SLEEP_ACCEPTANCE: '1', PAPERCLIP_STATUS_OBJECT_SLEEP_OWNER: args[0].slice(8), + PAPERCLIP_STATUS_OBJECT_SLEEP_STAGE: args[1].slice(8), PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS: '1' }; +for (const name of [...names, 'PAPERCLIP_STATUS_OBJECT_SLEEP_DIAGNOSTIC', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE', + 'PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA', 'PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE', 'PAPERCLIP_TEST_POSTGRES_ADMIN_URL']) { + if (process.env[name]) env[name] = process.env[name]; +} +const child = spawn(process.execPath, ['--require', guard, vitest, 'run', 'server/src/__tests__/status-object-signed-sleep-acceptance.test.ts', + '--maxWorkers=1', '--no-file-parallelism', `--execArgv=--require=${guard}`], { cwd: root, env, stdio: 'inherit' }); +for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); +child.once('error', error => { console.error(error); process.exitCode = 1; }); +child.once('close', (code, signal) => { + const events = readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + process.exitCode = events.some(event => event.event === 'blocked' || event.unexpectedAttempts?.length) ? 98 : signal ? 1 : code ?? 1; +}); diff --git a/server/src/__tests__/helpers/maintenance/postgres-proxy.ts b/server/src/__tests__/helpers/maintenance/postgres-proxy.ts index d80aa2fc689..d68a2990786 100644 --- a/server/src/__tests__/helpers/maintenance/postgres-proxy.ts +++ b/server/src/__tests__/helpers/maintenance/postgres-proxy.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { createConnection, createServer, type Socket } from "node:net"; -export type ObservedQuery = { protocol: "simple" | "extended"; sql: string; fingerprint: string }; +export type ObservedQuery = { protocol: "simple" | "extended"; sql: string; fingerprint: string; at?: number }; export function createPostgresProtocolObserver(onQuery: (query: ObservedQuery) => void, onFrame: (type: string) => void = () => {}) { let frontend = Buffer.alloc(0), backend = Buffer.alloc(0), startup = true, ready = false, pending = 1; @@ -67,7 +67,7 @@ export function createPostgresProtocolObserver(onQuery: (query: ObservedQuery) = }; } -export async function startMaintenancePostgresProxy(databaseUrl: string) { +export async function startMaintenancePostgresProxy(databaseUrl: string, captureQueryTimes = false) { const upstream = new URL(databaseUrl), queries: ObservedQuery[] = [], errors: Error[] = []; const frames = { Query: 0, Parse: 0, Bind: 0, Execute: 0, Sync: 0, Terminate: 0 }; const transport = { connectionsOpened: 0, connectionsClosed: 0, frontendBytes: 0, backendBytes: 0 }; @@ -76,7 +76,7 @@ export async function startMaintenancePostgresProxy(databaseUrl: string) { transport.connectionsOpened++; client.once("close", () => { transport.connectionsClosed++; }); const target = createConnection({ host: upstream.hostname, port: Number(upstream.port) || 5432 }); - const observer = createPostgresProtocolObserver(query => queries.push(query), type => { + const observer = createPostgresProtocolObserver(query => queries.push(captureQueryTimes ? { ...query, at: Date.now() } : query), type => { if (type === "Q") frames.Query++; else if (type === "P") frames.Parse++; else if (type === "B") frames.Bind++; else if (type === "E") frames.Execute++; else if (type === "S") frames.Sync++; else if (type === "X") frames.Terminate++; }); diff --git a/server/src/__tests__/helpers/maintenance/server-harness.ts b/server/src/__tests__/helpers/maintenance/server-harness.ts index 13d015e6b7f..4f68d99e0e1 100644 --- a/server/src/__tests__/helpers/maintenance/server-harness.ts +++ b/server/src/__tests__/helpers/maintenance/server-harness.ts @@ -1,3 +1,4 @@ +import type { ManagedExperimentalFeatureKey } from "@paperclipai/shared"; import type { IdleWorkCaptureSnapshot } from './idle-work-capture.js'; import type { IncomingHttpSnapshot } from '../../../../../scripts/sleep-qualification/evidence.mjs'; import { fork } from "node:child_process"; @@ -51,6 +52,7 @@ type Snapshot = { idle: { active: number; generation: number }; startup: { phase announcement?: { intercepted: boolean; requests: string[]; unmatched: string[]; boundary: string } }; export interface MaintenanceConfiguration { + managedFeatures?: Readonly>>; automaticBackups?: { intervalMinutes: number; idleCheckpoint: boolean }; announcements?: boolean; telemetryOrigin?: string; @@ -60,7 +62,15 @@ export interface MaintenanceConfiguration { deploymentMode?: "local_trusted" | "authenticated"; storageProvider?: "local_disk" | "s3"; } +export interface MaintenanceRuntimeSecrets { + PAPERCLIP_AGENT_JWT_SECRET: string; + PAPERCLIP_DECISION_SIGNING_SECRET: string; + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: string; + BETTER_AUTH_SECRET: string; + PAPERCLIP_SECRETS_MASTER_KEY: string; +} export interface MaintenanceServerOptions { + runtimeSecrets?: Readonly; childEntry?: URL; childPreload?: URL; databasePoolMax?: number; @@ -74,6 +84,16 @@ export interface MaintenanceServerOptions { } export async function startMaintenanceServer(databaseUrl: string, options: MaintenanceServerOptions = {}) { const configuration = options.configuration; + const runtimeSecrets = options.runtimeSecrets; + if (runtimeSecrets !== undefined) { + const keys: Array = ['PAPERCLIP_AGENT_JWT_SECRET', 'PAPERCLIP_DECISION_SIGNING_SECRET', + 'PAPERCLIP_TOOL_ACTION_SIGNING_SECRET', 'BETTER_AUTH_SECRET', 'PAPERCLIP_SECRETS_MASTER_KEY']; + if (!runtimeSecrets || Object.keys(runtimeSecrets).length !== keys.length || keys.some(key => + typeof runtimeSecrets[key] !== 'string' || runtimeSecrets[key].trim().length < 32) + || !/^[a-f0-9]{64}$/.test(runtimeSecrets.PAPERCLIP_SECRETS_MASTER_KEY)) { + throw new Error('Maintenance runtimeSecrets requires exactly five complete generated fixture secrets'); + } + } if (options.databasePoolMax !== undefined && (!Number.isInteger(options.databasePoolMax) || options.databasePoolMax < 1 || options.databasePoolMax > 4)) { throw new Error('Maintenance database pool must be an integer from 1 through 4'); } @@ -117,7 +137,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: Maint })(); } try { - const proxy = await startMaintenancePostgresProxy(databaseUrl); + const proxy = await startMaintenancePostgresProxy(databaseUrl, options.captureIdleWork); stopProxy = proxy.stop; await new Promise((resolve, reject) => { reservation.once("error", reject); reservation.listen(0, "127.0.0.1", resolve); }); const address = reservation.address(); @@ -154,8 +174,8 @@ export async function startMaintenanceServer(databaseUrl: string, options: Maint PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test", PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"), DATABASE_URL: proxy.url, DATABASE_POOL_MAX: options.databasePoolMax === undefined ? (configuration ? "4" : "6") : String(options.databasePoolMax), DATABASE_APPLICATION_NAME: "maintenance-integration-test", - PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), - PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), + PAPERCLIP_AGENT_JWT_SECRET: runtimeSecrets?.PAPERCLIP_AGENT_JWT_SECRET ?? randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: runtimeSecrets?.PAPERCLIP_DECISION_SIGNING_SECRET ?? randomUUID(), + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: runtimeSecrets?.PAPERCLIP_TOOL_ACTION_SIGNING_SECRET ?? randomUUID(), BETTER_AUTH_SECRET: runtimeSecrets?.BETTER_AUTH_SECRET ?? randomUUID(), CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: String(configuration?.announcements ?? false), ...(configuration?.announcements ? { PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE: '1' } : {}), PAPERCLIP_DB_BACKUP_ENABLED: String(Boolean(configuration?.automaticBackups)), PAPERCLIP_UI_DEV_MIDDLEWARE: "false", @@ -174,8 +194,8 @@ export async function startMaintenanceServer(databaseUrl: string, options: Maint } : {}), ...(options.controlJwks ? { PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.controlJwks } : {}), ...(options.cloud ? { - PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } }), - PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"), + PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: configuration?.managedFeatures ?? {}, plugins: { autoInstall: [] } }), + PAPERCLIP_SECRETS_MASTER_KEY: runtimeSecrets?.PAPERCLIP_SECRETS_MASTER_KEY ?? randomBytes(32).toString("hex"), PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "maintenance-tenant-token", PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.cloud.jwks, PAPERCLIP_CLOUD_WARM_STANDBY: options.cloud.standby ? "1" : "0", PAPERCLIP_PUBLIC_URL: options.tenantActor ? `http://127.0.0.1:${address.port}` : "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: options.tenantActor ? "127.0.0.1" : "pool-maintenance.staging.paperclip.app", diff --git a/server/src/__tests__/helpers/maintenance/status-interval-agent.mjs b/server/src/__tests__/helpers/maintenance/status-interval-agent.mjs new file mode 100644 index 00000000000..83a347fbfc0 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/status-interval-agent.mjs @@ -0,0 +1,84 @@ +import assert from 'node:assert/strict'; +import { readFile, appendFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +const statePath = process.argv[2]; +assert(statePath && path.isAbsolute(statePath)); +const guard = globalThis[Symbol.for('paperclip.skill.default-cadence.prewire')]; +assert(guard?.stats().loaded && guard.stats().pid === process.pid, 'Actual agent process requires prewire before imports'); +const originPath = process.env.STATUS_OBJECT_EXPECTED_API_ORIGIN_FILE; +assert(originPath && path.isAbsolute(originPath), 'Expected fixture API origin file is required'); +const expectedOrigin = await readFile(originPath, 'utf8'); +const expected = new URL(expectedOrigin); +assert(expected.protocol === 'http:' && expected.hostname === '127.0.0.1' && expected.port + && expected.pathname === '/' && !expected.username && !expected.password && !expected.search && !expected.hash); +assert.equal(expectedOrigin, expected.origin, 'Expected fixture API origin must be canonical'); +const base = new URL(process.env.PAPERCLIP_API_URL); +assert(base.protocol === 'http:' && base.hostname === '127.0.0.1' && base.pathname === '/' + && !base.username && !base.password && !base.search && !base.hash); +assert.equal(base.origin, expected.origin, 'Product-provided API origin must match the actual fixture server port'); +const token = process.env.PAPERCLIP_API_KEY, runId = process.env.PAPERCLIP_RUN_ID; +assert(token && runId && process.env.PAPERCLIP_AGENT_ID && process.env.PAPERCLIP_COMPANY_ID); +guard.allowOrigin(base.origin); +const receipt = async event => appendFile(statePath + '.runs.jsonl', JSON.stringify({ event, at: Date.now(), pid: process.pid, runId, providedApiOrigin: base.origin, expectedApiOrigin: expected.origin, + companyId: process.env.PAPERCLIP_COMPANY_ID, agentId: process.env.PAPERCLIP_AGENT_ID, guard: guard.stats() }) + '\n'); +await receipt('started'); +let state; +const readyBy = Date.now() + 10_000; +while (!state?.cardId && Date.now() < readyBy) { + try { state = JSON.parse(await readFile(statePath, 'utf8')); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + if (!state?.cardId) await delay(25); +} +assert(state?.cardId && Number.isSafeInteger(state.revision) && state.revision > 0, 'Producer must publish the actual returned card id'); +async function api(method, route, body) { + const response = await fetch(new URL('/api' + route, base), { + method, signal: AbortSignal.timeout(10_000), + headers: { Authorization: `Bearer ${token}`, 'X-Paperclip-Run-Id': runId, 'Content-Type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + assert(response.ok, `Authorized status writer API ${method} returned ${response.status}`); + return response.json(); +} +const card = await api('GET', `/status-cards/${state.cardId}`); +assert.equal(card.companyId, process.env.PAPERCLIP_COMPANY_ID); +assert.equal(card.agentId, process.env.PAPERCLIP_AGENT_ID); +assert(card.generatingIssueId); +const generation = await api('GET', `/issues/${card.generatingIssueId}`); +assert.equal(generation.assigneeAgentId, process.env.PAPERCLIP_AGENT_ID); +assert(generation.checkoutRunId === runId || generation.executionRunId === runId, 'Normal dispatcher must own the generation run'); +if (card.state === 'compiling') await api('PUT', `/status-cards/${card.id}/query`, { + queries: [{ scope: 'issues', status: ['blocked'] }], title: 'Current blockers', + changeSummary: 'Compiled the normal blocked-issue query', generationIssueId: generation.id, +}); +const preview = await api('GET', `/status-cards/${card.id}/dry-run`); +assert.equal(preview.cardId, card.id); +assert(Array.isArray(preview.queries)); +const facts = preview.queries.flatMap(page => { + assert(Array.isArray(page.result?.results)); + return page.result.results; +}).map(result => { + assert.equal(result.type, 'issue'); + assert(result.issue && typeof result.issue.id === 'string' && typeof result.issue.title === 'string'); + assert(typeof result.issue.identifier === 'string'); + return `${result.issue.identifier}: ${result.issue.title}`; +}); +assert(facts.length > 0, 'Useful summary must describe actual normal company-search results'); +const markdown = `Useful status revision ${state.revision}.\n\n${facts.join('\n')}`; +const summary = await api('PUT', `/status-cards/${card.id}/summary`, { + markdown, + changeSummary: `Observed revision ${state.revision} through the actual query`, + generationIssueId: generation.id, model: 'controlled-local-status-model', +}); +assert.equal(summary.card.id, card.id); +assert.equal(summary.card.companyId, process.env.PAPERCLIP_COMPANY_ID); +assert.equal(summary.card.state, 'active'); +assert.equal(summary.card.generatingIssueId, null); +assert.equal(summary.document.latestBody, markdown); +const completed = await api('PATCH', `/issues/${generation.id}`, { status: 'done' }); +assert.equal(completed.id, generation.id); +assert.equal(completed.companyId, process.env.PAPERCLIP_COMPANY_ID); +assert.equal(completed.status, 'done'); +await receipt('completed'); +console.log(JSON.stringify({ event: 'status-interval-agent-completed', at: Date.now(), pid: process.pid, + runId, generationIssueId: generation.id, cardId: card.id, revision: state.revision, guard: guard.stats() })); diff --git a/server/src/__tests__/helpers/maintenance/status-object-child.ts b/server/src/__tests__/helpers/maintenance/status-object-child.ts new file mode 100644 index 00000000000..86d84abd001 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/status-object-child.ts @@ -0,0 +1,8 @@ +import { installDefaultCadenceNetwork, defaultCadenceNetwork } from './default-cadence-network.js'; +installDefaultCadenceNetwork(); +const send = process.send; +if (send) process.send = function(message: unknown, ...args: unknown[]) { + const value = message && typeof message === 'object' ? { ...message, defaultCadenceGuard: defaultCadenceNetwork() } : message; + return Reflect.apply(send, process, [value, ...args]); +} as NonNullable; +await import('./server-child.js'); diff --git a/server/src/__tests__/helpers/maintenance/status-object-producers.ts b/server/src/__tests__/helpers/maintenance/status-object-producers.ts new file mode 100644 index 00000000000..59f09ab161d --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/status-object-producers.ts @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { INSTANCE_FEATURE_CATALOG, PAPERCLIP_CLOUD_MANAGED_BY, type InstanceExperimentalSettingsWithManaged } from '@paperclipai/shared'; +import { createServer } from 'node:http'; +import { writeFile, rename } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import type { startMaintenanceServer } from './server-harness.js'; +type Server = Awaited>; +async function assertManagedOwnerFeature(server: Server, chosen: 'enableStatusCards' | 'enableExternalObjects') { + const settings = await server.api('GET', '/instance/settings/experimental'); + assert.equal(settings[chosen], true); + assert.deepEqual(settings.managedKeys?.[chosen], { managed: true, managedBy: PAPERCLIP_CLOUD_MANAGED_BY }); + const unchosen = chosen === 'enableStatusCards' ? 'enableExternalObjects' : 'enableStatusCards'; + assert.equal(settings[unchosen], INSTANCE_FEATURE_CATALOG[unchosen].cloudDefault); + assert.equal(settings.managedKeys?.[unchosen], undefined); +} +export async function prepareStatusInterval(server: Server, companyId: string, statePath: string) { + const writeState = async (value: { cardId?: string; revision: number }) => { + await writeFile(statePath + '.pending', JSON.stringify(value), { mode: 0o600 }); + await rename(statePath + '.pending', statePath); + }; + const originPath = statePath + '.api-origin'; + const writeExpectedApiOrigin = async (currentServer: Server) => { + await writeFile(originPath + '.pending', `http://127.0.0.1:${currentServer.port}`, { mode: 0o600 }); + await rename(originPath + '.pending', originPath); + }; + await writeExpectedApiOrigin(server); + await writeState({ revision: 1 }); + await assertManagedOwnerFeature(server, 'enableStatusCards'); + const agent = await server.api<{ id: string }>('POST', `/companies/${companyId}/agents`, { + name: 'Controlled interval summarizer', adapterType: 'process', runner: 'legacy', + adapterConfig: { env: { STATUS_OBJECT_EXPECTED_API_ORIGIN_FILE: originPath, DATABASE_URL: '', PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG: process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG }, command: process.execPath, args: ['--require', + fileURLToPath(new URL('../../../../../scripts/fixtures/skill-default-cadence-network-guard.cjs', import.meta.url)), + fileURLToPath(new URL('./status-interval-agent.mjs', import.meta.url)), statePath], timeoutSec: 30, graceSec: 5 }, + }); + const blocker = await server.api<{ id: string }>('POST', `/companies/${companyId}/issues`, { title: 'A real persisted blocker', status: 'blocked', priority: 'medium' }); + const card = await server.api<{ id: string }>('POST', `/companies/${companyId}/status-cards`, { + interestPrompt: 'Summarize current blocked issues', agentId: agent.id, + refreshPolicy: { mode: 'interval', intervalMinutes: 15 }, + }); + const revision = (value: number) => { + assert(Number.isSafeInteger(value) && value > 0); + return writeState({ cardId: card.id, revision: value }); + }; + await revision(1); + return { agentId: agent.id, cardId: card.id, blockerId: blocker.id, revision, writeExpectedApiOrigin }; +} +export function statusObjectGitHubBoundary() { + let revision = 1; + const requests: Array<{ at: number; path: string }> = []; + const server = createServer((request, response) => { + requests.push({ at: Date.now(), path: request.url ?? '' }); + const body = request.url === '/repos/acme/sleep-object' + ? { id: 731, full_name: 'acme/sleep-object', private: false, owner: { login: 'acme' } } + : request.url === '/repos/acme/sleep-object/issues/7' + ? { id: 732, number: 7, title: `Useful GitHub object revision ${revision}`, state: 'open', + html_url: 'https://github.com/acme/sleep-object/issues/7', updated_at: new Date().toISOString(), labels: [] } + : undefined; + response.setHeader('Content-Type', 'application/json'); + response.statusCode = body ? 200 : 404; + response.end(JSON.stringify(body ?? { message: 'Unexpected owned GitHub path' })); + }); + return { server, requests, revision(value: number) { assert(Number.isSafeInteger(value) && value > revision); revision = value; } }; +} +export async function prepareGitHubObject(server: Server, companyId: string) { + await assertManagedOwnerFeature(server, 'enableExternalObjects'); + const issue = await server.api<{ id: string; companyId: string; description: string | null }>('POST', `/companies/${companyId}/issues`, { + title: 'Track the ordinary supported GitHub issue', status: 'todo', priority: 'medium', + }); + assert.equal(issue.companyId, companyId); + assert(!issue.description?.includes('https://github.com/acme/sleep-object/issues/7')); + return { issueId: issue.id, async addReference() { + const description = 'Reference https://github.com/acme/sleep-object/issues/7'; + const updated = await server.api<{ id: string; companyId: string; description: string }>('PATCH', `/issues/${issue.id}`, { description }); + assert.equal(updated.id, issue.id); + assert.equal(updated.companyId, companyId); + assert.equal(updated.description, description); + return updated; + } }; +} diff --git a/server/src/__tests__/status-object-signed-sleep-acceptance.test.ts b/server/src/__tests__/status-object-signed-sleep-acceptance.test.ts new file mode 100644 index 00000000000..ba52eecd2d4 --- /dev/null +++ b/server/src/__tests__/status-object-signed-sleep-acceptance.test.ts @@ -0,0 +1,470 @@ +import { defaultCadenceNetwork, installDefaultCadenceNetwork, allowDefaultCadenceOrigin } from './helpers/maintenance/default-cadence-network.js'; +import assert from 'node:assert/strict'; +import { generateKeyPairSync, randomBytes, randomUUID, createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { readFile, writeFile, mkdtemp, chmod, stat, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { describe, expect, it } from 'vitest'; +import { applyPendingMigrations, createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { startMaintenanceServer, withMaintenanceProvider, type MaintenanceRuntimeSecrets } from './helpers/maintenance/server-harness.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { provisionManagedSleepCompany, managedTenantActor } from './helpers/maintenance/managed-profile.js'; +import { startDefaultCadenceBridge } from './helpers/maintenance/default-cadence-bridge.js'; +import { prepareStatusInterval, prepareGitHubObject, statusObjectGitHubBoundary } from './helpers/maintenance/status-object-producers.js'; + +const enabled = process.env.PAPERCLIP_STATUS_OBJECT_SLEEP_ACCEPTANCE === '1'; +const owner = process.env.PAPERCLIP_STATUS_OBJECT_SLEEP_OWNER; +const stage = process.env.PAPERCLIP_STATUS_OBJECT_SLEEP_STAGE; +const diagnostic = process.env.PAPERCLIP_STATUS_OBJECT_SLEEP_DIAGNOSTIC === '1'; +const appUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SKILL_DEFAULT_CADENCE_TEST_DATABASE_URL'); +const cloudUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_TEST_DATABASE_URL'); +const cloudCheckout = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_CHECKOUT; +const cloudSha = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_SHA; +const cloudNode = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_CLOUD_NODE; +const evidence = process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_EVIDENCE; +if (enabled) { + installDefaultCadenceNetwork(); + assert(owner === 'status' || owner === 'object'); assert(stage === 'bootstrap' || stage === 'cycle' || (stage === 'identity' && owner === 'status')); + if (diagnostic) assert((owner === 'object' && stage === 'bootstrap') || (owner === 'status' && stage === 'identity')); + assert(appUrl && cloudUrl && cloudCheckout && cloudNode && evidence && /^[a-f0-9]{40}$/.test(cloudSha ?? '')); + assert.equal(process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS, '1'); +} +type Observation = { id: string; next_at: Date; last_at: Date; value: string; version: string; claim: string | null; pending: number; policy: unknown; last_run_id: string | null; generation_issue_id: string | null; run_started_at: Date | null; run_status: string | null; generation_status: string | null }; +type ObservedRow = Omit & { + last_at: Date | string | null; next_at: Date | string | null; run_started_at: Date | string | null; value: string | null; +}; +function producedTimestamp(value: unknown, field: string): Date { + assert(value instanceof Date || (typeof value === 'string' && value.trim().length > 0), `Invalid produced ${field}`); + const parsed = value instanceof Date ? value : new Date(value); + assert(Number.isFinite(parsed.getTime()), `Invalid produced ${field}`); + return parsed; +} +type AgentReceipt = { event: string; at: number; pid: number; runId: string; guard: ReturnType }; + +describe.skipIf(!enabled)('status and external-object actual signed cold-owner handoff', () => { + const runAcceptance = async (): Promise => { + assert(appUrl && cloudUrl && cloudCheckout && cloudNode && evidence); + assert.equal(execFileSync('git', ['-C', cloudCheckout, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), cloudSha); + assert.match(execFileSync(cloudNode, ['--version'], { encoding: 'utf8' }), /^v22\./); + await applyPendingMigrations(appUrl); + const upstream = statusObjectGitHubBoundary(), s3 = createMaintenanceS3Boundary(); + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'signed-owner', alg: 'EdDSA', use: 'sig' }; + const privateJwk = JSON.stringify({ ...pair.privateKey.export({ format: 'jwk' }), kid: jwk.kid, alg: 'EdDSA', use: 'sig' }); + const jwks = JSON.stringify({ keys: [jwk] }); + const statePath = path.join(evidence, 'status-provider-state.json'); + const childPids: number[] = []; + let boundaryAdvanced = false; + const physical: Array<{ kind: string; at: number; pid?: number }> = []; + let server: Awaited> | undefined; + let bridge: Awaited> | undefined; + const phases: Array<{ name: string; at: number; queryCount?: number; pending?: number }> = []; + const phase = (name: string) => { + phases.push({ name, at: Date.now(), queryCount: server?.proxy.queries.length, pending: server?.proxy.pending }); + }; + let secretsDirectory: string | undefined; + let primary: { error: unknown } | undefined; + const failures: Array<{ stage: string; error: unknown }> = []; + let status: Awaited> | undefined; + const agentReceipts = async (): Promise => { + if (owner !== 'status') return []; + try { return (await readFile(statePath + '.runs.jsonl', 'utf8')).trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); } + catch (error) { if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return []; throw error; } + }; + const record = async (name: string, value: unknown) => writeFile(path.join(evidence, name + '.json'), JSON.stringify(value, null, 2)); + const proxyEvidence = () => server ? { queries: [...server.proxy.queries], frames: server.proxy.frames, transport: server.proxy.transport, + pending: server.proxy.pending, starting: server.proxy.starting, errors: server.proxy.errors.map(error => error.stack ?? String(error)) } : undefined; + try { + secretsDirectory = await mkdtemp(path.join(tmpdir(), 'paperclip-owner-runtime-secrets-')); + await chmod(secretsDirectory, 0o700); + const secretsPath = path.join(secretsDirectory, 'runtime-secrets.json'); + const runtimeSecrets: MaintenanceRuntimeSecrets = { + PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), + PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString('hex'), + }; + const retainRuntimeSecrets = async (value: MaintenanceRuntimeSecrets, label: string, exclusive = false) => { + const bytes = JSON.stringify(value); + await writeFile(secretsPath, bytes, { mode: 0o600, flag: exclusive ? 'wx' : 'w' }); + const mode = (await stat(secretsPath)).mode & 0o777; + assert.equal(mode, 0o600); + await record(label, { names: Object.keys(value).sort(), sha256: createHash('sha256').update(bytes).digest('hex'), + changedNames: Object.keys(value).filter(key => value[key as keyof MaintenanceRuntimeSecrets] !== runtimeSecrets[key as keyof MaintenanceRuntimeSecrets]).sort(), + fileMode: mode, directoryMode: (await stat(secretsDirectory!)).mode & 0o777, values: 'Not retained in evidence; private runtime file deleted after all owners stop' }); + }; + await retainRuntimeSecrets(runtimeSecrets, 'runtime-secret-profile', true); + if (diagnostic) await record('diagnostic-mode', { instrumented: true, capture: 'Existing passive idle-work capture and timestamped proxy query metadata', + initialPostproductionSettleDeadlineMs: 30_000, identityPostgenerationSettleDeadlineMs: 30_000, preproductionSetupObservationMs: 61_000, quietMs: 500, + owner, stage, boundary: 'Diagnostic evidence only; normal product startup, work, promises and admission retained' }); + await withMaintenanceProvider(s3.server, s3Origin => withMaintenanceProvider(upstream.server, async githubOrigin => { + const boot = async () => { + const retainedSecrets = JSON.parse(await readFile(secretsPath, 'utf8')) as MaintenanceRuntimeSecrets; + const started = await startMaintenanceServer(appUrl, { + childEntry: new URL('./helpers/maintenance/status-object-child.ts', import.meta.url), + childPreload: new URL('../../../scripts/fixtures/skill-default-cadence-network-guard.cjs', import.meta.url), + databasePoolMax: 2, runtimeSecrets: retainedSecrets, artifactRoot: evidence, githubOrigin, clock: false, controlJwks: jwks, captureIdleWork: diagnostic, + cloud: { jwks, s3Origin, standby: false }, tenantActor: managedTenantActor, + configuration: { managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3', announcements: false, + managedFeatures: owner === 'status' ? { enableStatusCards: true } : { enableExternalObjects: true } }, + }); + assert(started.pid); childPids.push(started.pid); + try { + expect(started.childGuard).toMatchObject({ loaded: true, pid: started.pid, ppid: process.pid, sourceHash: defaultCadenceNetwork().sourceHash }); + allowDefaultCadenceOrigin(`http://127.0.0.1:${started.port}`); + if (status) await status.writeExpectedApiOrigin(started); + return started; + } catch (error) { + try { await started.stop(); } catch (error) { failures.push({ stage: 'boot-origin-cleanup', error }); } + throw error; + } + }; + try { + server = await boot(); phase('app-ready'); + const setupDb = createDb(appUrl, { maxConnections: 1 }); + let company: Awaited>; + try { company = await provisionManagedSleepCompany(server, setupDb); } + catch (error) { primary ??= { error }; throw error; } + finally { + try { await setupDb.$client.end({ timeout: 1 }); } + catch (error) { if (primary) failures.push({ stage: 'setup-close', error }); else throw error; } + } + phase('normal-producer-entered'); + if (owner === 'status') status = await prepareStatusInterval(server, company.id, statePath); + else { + const object = await prepareGitHubObject(server, company.id); + phase('object-plain-issue-created'); + const setupAgents = await server.api('GET', `/companies/${company.id}/agents`); + expect(setupAgents).toHaveLength(0); + phase('object-setup-reconciliation-entered'); + const setup = await server.settled(500, 0, { managedSetupRecovery: true }); + expect(setup.startup.phase).toBe('ready'); expect(setup.idle.active).toBe(0); + expect(upstream.requests).toHaveLength(0); + phase('object-setup-reconciliation-returned'); + await record('object-preproduction-setup', { issueId: object.issueId, observationBudgetMs: 61_000, quietMs: 500, + setup, agentCount: setupAgents.length, phases: [...phases], requests: [...upstream.requests], producer: 'Normal plain-issue POST then description-only URL PATCH after real setup reconciliation' }); + phase('object-reference-patch-entered'); + await object.addReference(); + phase('object-reference-patch-returned'); + } + phase('normal-producer-returned'); + const readProduced = async (): Promise => { + const url = new URL(appUrl); url.searchParams.set('application_name', 'status-object-readonly-observer'); + const db = createDb(url.href, { maxConnections: 1 }); + let readFailure: { error: unknown } | undefined; + try { + const rows = await db.$client.begin('read only', tx => owner === 'status' + ? tx`select c.id,c.next_eval_at as next_at,c.last_generated_at as last_at,d.latest_body as value,d.latest_revision_number::text as version, + c.generating_issue_id as claim,c.refresh_policy as policy, + (select count(*)::int from status_card_updates u where u.card_id=c.id and u.status='running') as pending, + u.run_id as last_run_id,u.generation_issue_id,r.started_at as run_started_at,r.status as run_status,gi.status as generation_status + from status_cards c join documents d on d.id=c.document_id and d.company_id=c.company_id + left join lateral (select run_id,generation_issue_id from status_card_updates u where u.card_id=c.id and u.kind!='compile' and u.status='ok' order by u.finished_at desc limit 1) u on true + left join heartbeat_runs r on r.id=u.run_id and r.company_id=c.company_id + left join issues gi on gi.id=u.generation_issue_id and gi.company_id=c.company_id where c.company_id=${company.id} and c.id=${status!.cardId}` + : tx`select o.id,o.next_refresh_at as next_at,o.last_resolved_at as last_at,o.display_title as value,o.xmin::text as version, + o.refresh_token::text as claim,(select count(*)::int from external_object_mentions m where m.object_id=o.id and m.company_id=o.company_id) as pending, + o.data as policy,null::text as last_run_id,null::text as generation_issue_id,null::timestamptz as run_started_at,null::text as run_status,null::text as generation_status + from external_objects o where o.company_id=${company.id} and o.provider_key='github' and o.external_id='acme/sleep-object#issues/7'`); + if (rows.length === 0) return undefined; + assert.equal(rows.length, 1); + const row = rows[0]!; + assert(Object.hasOwn(row, 'last_at'), 'Produced projection is missing last_at'); + if (row.last_at === null) return undefined; + const last_at = producedTimestamp(row.last_at, 'timestamp'); + const next_at = producedTimestamp(row.next_at, 'deadline'); + const run_started_at = row.run_started_at === null ? null : producedTimestamp(row.run_started_at, 'run start'); + assert(typeof row.value === 'string', 'Invalid produced value'); + assert.equal(typeof row.version, 'string', 'Invalid produced version'); + return { ...row, last_at, next_at, run_started_at, value: row.value }; + } catch (error) { readFailure = { error }; throw error; } + finally { + try { await db.$client.end({ timeout: 1 }); } + catch (error) { if (readFailure) failures.push({ stage: 'observer-close', error }); else throw error; } + } + }; + const persisted = async () => { const row = await readProduced(); assert(row, 'No produced row'); return row; }; + const marker = (revision: number) => owner === 'status' ? `Useful status revision ${revision}.` : `Useful GitHub object revision ${revision}`; + const initialBy = Date.now() + 30_000; + let produced = await readProduced(); + while (!produced && Date.now() < initialBy) { await delay(50); produced = await readProduced(); } + assert(produced, 'No produced row within 30000ms'); + expect(produced.value).toContain(marker(1)); phase('useful-v1-observed'); + if (stage === 'identity') { + assert(status); + const identityDb = createDb(appUrl, { maxConnections: 1 }); + let controlPrimary: { error: unknown } | undefined; + try { + const identity = async () => { + const rows = await identityDb.$client.begin('read only', tx => tx>` + select agent_id,company_id,algorithm,key_id,md5(private_key_material::text) as ciphertext_hash + from agent_identity_keys where company_id=${company.id} and agent_id=${status!.agentId}`); + assert.equal(rows.length, 1); return rows[0]!; + }; + const useful = async (revision: number) => { + const deadline = Date.now() + 30_000; + let row = await persisted(); + while ((!row.value.includes(marker(revision)) || row.run_status !== 'succeeded' || row.generation_status !== 'done') && Date.now() < deadline) { + await delay(50); row = await persisted(); + } + expect(row.value).toContain(marker(revision)); expect(row.run_status).toBe('succeeded'); + expect(row.generation_status).toBe('done'); expect(row.claim).toBeNull(); + return row; + }; + const first = await useful(1), before = await identity(); + const restart = async (kind: string) => { + assert(server); const old = server; + await old.stop(); server = undefined; physical.push({ kind: `${kind}-stopped`, at: Date.now(), pid: old.pid }); + server = await boot(); physical.push({ kind: `${kind}-started`, at: Date.now(), pid: server.pid }); + expect(server.pid).not.toBe(old.pid); + }; + await restart('identity-same-key'); + await status.revision(2); + await server!.api('POST', `/status-cards/${status.cardId}/refresh`, { full: true }); + const second = await useful(2); + phase('identity-useful-v2-observed'); + if (diagnostic) await record('diagnostic-useful-v2', { second, phases: [...phases], proxy: proxyEvidence() }); + const retained = await identity(); + expect(second.last_run_id).not.toBe(first.last_run_id); expect(retained).toEqual(before); + const starts = (await agentReceipts()).filter(event => event.event === 'started'); + expect(starts).toHaveLength(2); expect(starts[1]!.runId).toBe(second.last_run_id); + if (diagnostic) { + phase('identity-postgeneration-settle-entered'); + await record('diagnostic-postgeneration-settle-entered', { phases: [...phases], proxy: proxyEvidence() }); + const firstStartedAt = Date.now(); + const settled = await server!.settled().catch(async firstError => { + phase('identity-postgeneration-first-settle-failed'); + try { await record('diagnostic-first-settle-failure', { error: firstError instanceof Error ? firstError.stack : String(firstError), + startedAt: firstStartedAt, endedAt: Date.now(), elapsedMs: Date.now() - firstStartedAt, phases: [...phases], proxy: proxyEvidence() }); } + catch (error) { failures.push({ stage: 'diagnostic-first-failure-evidence', error }); } + phase('identity-postgeneration-continuation-entered'); + const continuationStartedAt = Date.now(); + try { + const continuation = await server!.settled(); + phase('identity-postgeneration-continuation-returned'); + try { await record('diagnostic-settle-continuation', { firstAssertionFailed: true, outcome: 'naturally_settled_after_failed_bound', + startedAt: continuationStartedAt, endedAt: Date.now(), elapsedMs: Date.now() - continuationStartedAt, + continuation, phases: [...phases], proxy: proxyEvidence() }); } + catch (error) { failures.push({ stage: 'diagnostic-continuation-evidence', error }); } + } catch (continuationError) { + phase('identity-postgeneration-continuation-failed'); + try { await record('diagnostic-settle-continuation', { firstAssertionFailed: true, outcome: 'failed_again', + error: continuationError instanceof Error ? continuationError.stack : String(continuationError), + startedAt: continuationStartedAt, endedAt: Date.now(), elapsedMs: Date.now() - continuationStartedAt, + phases: [...phases], proxy: proxyEvidence() }); } + catch (error) { failures.push({ stage: 'diagnostic-continuation-evidence', error }); } + } + throw firstError; + }); + phase('identity-postgeneration-settle-returned'); + await record('diagnostic-postgeneration-settled', { settled, phases: [...phases], proxy: proxyEvidence() }); + } + const alteredSecrets = { ...runtimeSecrets, PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString('hex') }; + expect(alteredSecrets.PAPERCLIP_SECRETS_MASTER_KEY).not.toBe(runtimeSecrets.PAPERCLIP_SECRETS_MASTER_KEY); + await retainRuntimeSecrets(alteredSecrets, 'runtime-secret-negative-profile'); + await restart('identity-wrong-master'); + await status.revision(3); + await server!.api('POST', `/status-cards/${status.cardId}/refresh`, { full: true }); + const latestRun = async () => identityDb.$client.begin('read only', tx => tx>` + select id,status,error_code,error from heartbeat_runs where company_id=${company.id} and agent_id=${status!.agentId} + order by created_at desc limit 1`).then(rows => rows[0]); + const failedBy = Date.now() + 30_000; + let rejected = await latestRun(); + while ((!rejected || rejected.id === second.last_run_id || rejected.status !== 'failed') && Date.now() < failedBy) { + await delay(50); rejected = await latestRun(); + } + assert(rejected); expect(rejected.id).not.toBe(second.last_run_id); expect(rejected.status).toBe('failed'); + expect(rejected.error_code).toBe('setup_failed'); expect(rejected.error).toContain('Secret decryption failed'); + expect((await agentReceipts()).filter(event => event.event === 'started')).toEqual(starts); + const afterRejected = await identity(); expect(afterRejected).toEqual(before); + await record('cold-identity-control', { first, second, before, retained, afterRejected, rejected, physical, + agentReceipts: await agentReceipts(), boundary: 'Actual normal manual refresh generations and cold process restarts; no scheduled-alarm/cadence claim; wrong master key must fail closed before agent entry' }); + return; + } catch (error) { controlPrimary = { error }; throw error; } + finally { + try { await identityDb.$client.end({ timeout: 1 }); } + catch (error) { if (controlPrimary) failures.push({ stage: 'identity-observer-close', error }); else throw error; } + } + } + phase('initial-settled-entered'); + if (owner === 'status') await server.settled(500, 0, { managedSetupRecovery: true }); + else await server.settled(); + phase('initial-settled-returned'); + if (diagnostic) await record('diagnostic-initial-settled', { phases, proxy: { queries: [...server.proxy.queries], + frames: server.proxy.frames, transport: server.proxy.transport, pending: server.proxy.pending, starting: server.proxy.starting } }); + const initial = await persisted(); + assert(initial.next_at && initial.last_at); + const originalD = new Date(initial.next_at).getTime(); + expect(originalD - new Date(initial.last_at).getTime()).toBe(owner === 'status' ? 900_000 : 300_000); + expect(initial.claim).toBeNull(); + if (owner === 'status') { + expect(initial.policy).toMatchObject({ mode: 'interval', intervalMinutes: 15 }); + expect(initial.pending).toBe(0); expect(initial.run_status).toBe('succeeded'); expect(initial.generation_status).toBe('done'); + await server.api('POST', `/companies/${company.id}/issues`, { title: 'A changed real persisted blocker', status: 'blocked', priority: 'medium' }); + await server.settled(500, 0, { managedSetupRecovery: true }); + expect((await persisted()).value).toBe(initial.value); + expect(new Date((await persisted()).next_at).getTime()).toBe(originalD); + } else expect(initial.pending).toBeGreaterThan(0); + bridge = await startDefaultCadenceBridge({ checkout: cloudCheckout, node: cloudNode, + physicalStop: async () => { + assert(server); const old = server; + physical.push({ kind: 'stop-entered', at: Date.now(), pid: old.pid }); + await old.stop(); server = undefined; physical.push({ kind: 'stopped', at: Date.now(), pid: old.pid }); + if (!boundaryAdvanced) { + if (status) await status.revision(2); else upstream.revision(2); + boundaryAdvanced = true; + } + }, + physicalWake: async () => { + assert(!server); server = await boot(); physical.push({ kind: 'cold-started', at: Date.now(), pid: server.pid }); + return { origin: `http://127.0.0.1:${server.port}` }; + } }); + assert(bridge.pid); childPids.push(bridge.pid); + await bridge.rpc('initialize', { databaseUrl: cloudUrl, migrationsPath: path.join(cloudCheckout, 'migrations'), + appOrigin: `http://127.0.0.1:${server.port}`, privateJwk, jwks }); + await server.settled(); + if (owner === 'object') expect((await bridge.rpc('legacy')).idleSleepSafe).not.toBe(true); + const requestStart = upstream.requests.length, agentStart = (await agentReceipts()).length; + const beforeSleepSnapshot = await server.snapshot(); + const sleep = await bridge.rpc('sleep'); + expect(server).toBeUndefined(); + expect(new Date(sleep.alarm.next_wake_at).getTime()).toBe(originalD); + const wakeAt = new Date(sleep.alarm.next_attempt_at).getTime(); + expect(wakeAt).toBe(originalD - 120_000); + const ack = sleep.events.find((event: { kind: string }) => event.kind === 'durable-alarm-before-stop'); + assert(ack); expect(typeof sleep.alarm.operation_lock_id).toBe('string'); expect(sleep.alarm.operation_lock_id.length).toBeGreaterThan(0); + expect(ack.row.operation_lock_id).toBe(sleep.alarm.operation_lock_id); + expect(ack.row.generation).toBe(sleep.alarm.generation); + expect(ack.at).toBeLessThanOrEqual(physical.find(event => event.kind === 'stop-entered')!.at); + await record('bootstrap-receipt', { owner, stage, originalD, wakeAt, initial, beforeSleepSnapshot, sleep, physical, requests: upstream.requests, agentReceipts: await agentReceipts() }); + if (stage === 'bootstrap') return; + await delay(Math.max(0, wakeAt - Date.now()) + 25); + const wake = await bridge.rpc('wake'); assert(server); + expect(childPids.filter(pid => pid !== bridge!.pid)).toHaveLength(2); + expect(physical.find(event => event.kind === 'cold-started')!.pid).not.toBe(physical.find(event => event.kind === 'stopped')!.pid); + await server.settled(); + const feature = await server.api>('GET', '/instance/settings/experimental'); + expect(feature[owner === 'status' ? 'enableStatusCards' : 'enableExternalObjects']).toBe(true); + await server.settled(); + const quiet = async () => { + assert(server); const before = await server.snapshot(); + const queries = server.proxy.queries.length, requests = upstream.requests.length, agents = (await agentReceipts()).length; + const started = Date.now(); await delay(10_000); + const after = await server.snapshot(); + expect(server.proxy.queries.length).toBe(queries); expect(upstream.requests.length).toBe(requests); + expect((await agentReceipts()).length).toBe(agents); expect(after.outgoingHttpRequests).toBe(before.outgoingHttpRequests); + return { started, ended: Date.now(), queries, requests, agents, before, after }; + }; + expect((await persisted()).value).toBe(initial.value); + expect(new Date((await persisted()).next_at).getTime()).toBe(originalD); + const earlyQuiet = await quiet(); + expect(earlyQuiet.ended).toBeLessThan(originalD); + expect((await persisted()).version).toBe(initial.version); + await delay(Math.max(0, originalD - Date.now()) + 25); + const changedBy = Date.now() + 30_000; + let observed = await persisted(); + while (!observed.value.includes(marker(2)) && Date.now() < changedBy) { await delay(50); observed = await persisted(); } + expect(observed.value).toContain(marker(2)); + if (owner === 'status') { + const startedAt = Date.now(), queriesBefore = server.proxy.queries.length, pendingBefore = server.proxy.pending; + const settled = await server.settled(500, 0, { managedSetupRecovery: true }); + const endedAt = Date.now(); + await record('status-postgeneration-observation', { startedAt, endedAt, elapsedMs: endedAt - startedAt, + timeoutMs: 61_000, quietMs: 500, queriesBefore, queriesAfter: server.proxy.queries.length, + pendingBefore, pendingAfter: server.proxy.pending, settled, + boundary: 'Bounded recovery reconciliation observation; the subsequent real 10s SQL/network/producer quiet window is separate' }); + } else await server.settled(); + const changed = await persisted(); + expect(changed.version).not.toBe(initial.version); expect(new Date(changed.last_at).getTime()).toBeGreaterThanOrEqual(originalD); + expect(new Date(changed.last_at).getTime()).toBeLessThan(originalD + 30_000); + expect(changed.claim).toBeNull(); + if (owner === 'status') { + expect(changed.value).toContain('A changed real persisted blocker'); + expect(changed.policy).toEqual(initial.policy); expect(changed.pending).toBe(0); expect(changed.run_status).toBe('succeeded'); expect(changed.generation_status).toBe('done'); + expect(new Date(changed.run_started_at!).getTime()).toBeGreaterThanOrEqual(originalD); + const starts = (await agentReceipts()).slice(agentStart).filter(event => event.event === 'started'); + expect(starts).toHaveLength(1); expect(starts[0]!.at).toBeGreaterThanOrEqual(originalD); + expect(starts[0]!.runId).toBe(changed.last_run_id); + } else { + const requests = upstream.requests.slice(requestStart); expect(requests.length).toBeGreaterThan(0); + expect(Math.min(...requests.map(request => request.at))).toBeGreaterThanOrEqual(originalD); + expect(changed.pending).toBeGreaterThan(0); + } + expect(new Date(changed.next_at).getTime() - new Date(changed.last_at).getTime()).toBe(owner === 'status' ? 900_000 : 300_000); + const settledQuiet = await quiet(); + await record('cold-owner-cycle-receipt', { owner, originalD, wakeAt, initial, sleep, wake, changed, physical, earlyQuiet, settledQuiet, + requests: upstream.requests, agents: await agentReceipts(), s3: s3.snapshot(), boundary: 'Actual signed app/Cloud/SQL alarm and owner; controlled localhost GitHub, deterministic process model, S3 and physical child only' }); + } catch (error) { + primary ??= { error }; phase('primary-caught'); + try { await record('primary-before-cleanup', { error: error instanceof Error ? error.stack : String(error), serverOutput: server?.output, bridgeOutput: bridge?.output, physical, diagnostic, phases, + proxy: proxyEvidence() }); } + catch (error) { failures.push({ stage: 'primary-evidence', error }); } + throw error; + } finally { + if (diagnostic) { + phase('diagnostic-owner-close-entered'); + try { await record('diagnostic-before-owner-close', { phases: [...phases], proxy: proxyEvidence() }); } + catch (error) { failures.push({ stage: 'diagnostic-close-evidence', error }); } + } + const currentBridge = bridge; bridge = undefined; + try { await currentBridge?.close(); } catch (error) { failures.push({ stage: 'bridge-close', error }); } + const currentServer = server; server = undefined; + try { await currentServer?.stop(); } catch (error) { failures.push({ stage: 'server-stop', error }); } + } + })); + } catch (error) { + if (!primary) primary = { error }; + else if (error !== primary.error) failures.push({ stage: 'provider-cleanup', error }); + try { await record('attempt-failure', { error: primary.error instanceof Error ? primary.error.stack : String(primary.error), physical }); } + catch (error) { failures.push({ stage: 'attempt-evidence', error }); } + } finally { + const currentBridge = bridge; bridge = undefined; + try { await currentBridge?.close(); } catch (error) { failures.push({ stage: 'final-bridge-close', error }); } + const currentServer = server; server = undefined; + try { await currentServer?.stop(); } catch (error) { failures.push({ stage: 'final-server-stop', error }); } + try { + const uncertainOwners = failures.filter(({ stage }) => ['boot-origin-cleanup', 'bridge-close', 'server-stop', + 'final-bridge-close', 'final-server-stop'].includes(stage)).map(({ stage }) => stage); + if (secretsDirectory && uncertainOwners.length) { + const ownedPath = path.join(secretsDirectory, 'runtime-secrets.json'); + let sha256: string | undefined; + try { sha256 = createHash('sha256').update(await readFile(ownedPath)).digest('hex'); } + catch (error) { failures.push({ stage: 'runtime-secret-retained-hash', error }); } + await record('runtime-secret-retained', { ownedPath, sha256, uncertainOwners, + reason: 'Owner closure failed; private profile retained for exact owner audit and normal cleanup' }); + } else if (secretsDirectory) await rm(secretsDirectory, { recursive: true, force: true }); + } catch (error) { failures.push({ stage: 'runtime-secret-cleanup', error }); } + try { + const agents = await agentReceipts(); + const events = (await readFile(process.env.PAPERCLIP_SKILL_DEFAULT_CADENCE_GUARD_LOG!, 'utf8')).trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + expect(events.filter(event => event.event === 'blocked')).toEqual([]); + for (const pid of [process.pid, process.ppid, ...childPids, ...agents.map(agent => agent.pid)]) { + expect(events).toContainEqual(expect.objectContaining({ event: 'preloaded', pid, sourceHash: defaultCadenceNetwork().sourceHash })); + } + await record('final-parent-guard', { stats: defaultCadenceNetwork(), physical, childPids, agents }); + expect(defaultCadenceNetwork().offhostAttempts).toEqual([]); + } catch (error) { failures.push({ stage: 'guard-evidence', error }); } + try { await record('cleanup-guard-failures', failures.map(({ stage, error }) => ({ stage, error: error instanceof Error ? error.stack : String(error) }))); } + catch (error) { failures.push({ stage: 'cleanup-evidence', error }); } + } + if (primary) throw primary.error; + if (failures.length) throw new AggregateError(failures.map(failure => failure.error), 'Owner acceptance cleanup or guard failed'); + }; + const registrations = [ + { owner: 'status', stage: 'bootstrap', diagnostic: false, name: 'status bootstrap: publishes a signed original-deadline alarm before physical stop' }, + { owner: 'status', stage: 'cycle', diagnostic: false, name: 'status cycle: publishes useful work at the original interval after cold wake' }, + { owner: 'status', stage: 'identity', diagnostic: false, name: 'status identity: retains encrypted identity across cold boots and rejects a changed master key' }, + { owner: 'object', stage: 'bootstrap', diagnostic: false, name: 'object bootstrap: publishes a signed built-in GitHub deadline before physical stop' }, + { owner: 'object', stage: 'cycle', diagnostic: false, name: 'object cycle: refreshes built-in GitHub data at the original deadline after cold wake' }, + { owner: 'status', stage: 'identity', diagnostic: true, name: 'status identity diagnostic: observes post-generation recovery and preserves the first failed bound' }, + { owner: 'object', stage: 'bootstrap', diagnostic: true, name: 'object bootstrap diagnostic: retains passive setup SQL and work evidence' }, + ] as const; + for (const registration of registrations) { + it.skipIf(owner !== registration.owner || stage !== registration.stage || diagnostic !== registration.diagnostic)( + registration.name, runAcceptance, + registration.stage === 'identity' ? 240_000 : registration.stage === 'bootstrap' ? 850_000 : 1_250_000, + ); + } +});