From ab2d2098a2764c511978d1c3e7a3fc5dfcef07f4 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:41:15 -0600 Subject: [PATCH] test: verify sleep producer readiness and CI contracts (#18795) Reviewed and verified at 9f87b490d1d25553c49570ba5074c4143b8e3f7d. Co-Authored-By: Paperclip --- doc/observability.md | 3 +- doc/sleep-qualification.md | 244 ++++++++++++++ scripts/sleep-qualification/evidence.d.mts | 54 +++ scripts/sleep-qualification/evidence.mjs | 231 +++++++++++++ scripts/sleep-qualification/setup-otel.mjs | 125 +++++++ server/scripts/sleep-otel-setup.test.mjs | 67 ++++ server/scripts/sleep-qualification.test.mjs | 233 +++++++++++++ .../maintenance/announcement-boundary.ts | 36 ++ .../helpers/maintenance/backup-observation.ts | 54 +++ .../maintenance/browser-observation.ts | 82 +++++ .../maintenance/chat-history-producer.ts | 21 ++ .../maintenance/configuration-producers.ts | 48 +++ .../early-diagnostic-stabilization.ts | 52 +++ .../maintenance/early-sleep-diagnostics.ts | 18 + .../github-configuration-producer.ts | 47 +++ .../maintenance/idle-work-capture-probe.ts | 50 +++ .../helpers/maintenance/idle-work-capture.ts | 137 ++++++++ .../maintenance/incoming-http-observation.ts | 40 +++ .../helpers/maintenance/managed-profile.ts | 34 ++ .../helpers/maintenance/native-browser.ts | 65 ++++ .../helpers/maintenance/otel-child.ts | 10 + .../helpers/maintenance/s3-boundary.ts | 79 +++++ .../helpers/maintenance/server-child.ts | 74 ++++- .../helpers/maintenance/server-harness.ts | 144 ++++++-- .../__tests__/instrumentation-signals.test.ts | 30 ++ .../maintenance-harness-resources.test.ts | 8 + .../sleep-backup-observation.test.ts | 61 ++++ .../sleep-configuration-acceptance.test.ts | 312 ++++++++++++++++++ .../__tests__/sleep-early-diagnostics.test.ts | 15 + .../__tests__/sleep-idle-work-capture.test.ts | 90 +++++ .../sleep-incoming-http-observation.test.ts | 100 ++++++ .../__tests__/sleep-managed-fixture.test.ts | 69 ++++ .../__tests__/sleep-managed-preflight.test.ts | 99 ++++++ .../src/__tests__/sleep-otel-export.test.ts | 43 +++ .../voice-session-store.integration.test.ts | 29 +- .../src/__tests__/workspace-runtime.test.ts | 28 +- server/src/instrumentation.ts | 4 + tests/e2e/disconnect-write-recovery.spec.ts | 5 +- tests/e2e/task-loading-frame.spec.ts | 29 +- 39 files changed, 2823 insertions(+), 47 deletions(-) create mode 100644 doc/sleep-qualification.md create mode 100644 scripts/sleep-qualification/evidence.d.mts create mode 100644 scripts/sleep-qualification/evidence.mjs create mode 100644 scripts/sleep-qualification/setup-otel.mjs create mode 100644 server/scripts/sleep-otel-setup.test.mjs create mode 100644 server/scripts/sleep-qualification.test.mjs create mode 100644 server/src/__tests__/helpers/maintenance/announcement-boundary.ts create mode 100644 server/src/__tests__/helpers/maintenance/backup-observation.ts create mode 100644 server/src/__tests__/helpers/maintenance/browser-observation.ts create mode 100644 server/src/__tests__/helpers/maintenance/chat-history-producer.ts create mode 100644 server/src/__tests__/helpers/maintenance/configuration-producers.ts create mode 100644 server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts create mode 100644 server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts create mode 100644 server/src/__tests__/helpers/maintenance/github-configuration-producer.ts create mode 100644 server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts create mode 100644 server/src/__tests__/helpers/maintenance/idle-work-capture.ts create mode 100644 server/src/__tests__/helpers/maintenance/incoming-http-observation.ts create mode 100644 server/src/__tests__/helpers/maintenance/managed-profile.ts create mode 100644 server/src/__tests__/helpers/maintenance/native-browser.ts create mode 100644 server/src/__tests__/helpers/maintenance/otel-child.ts create mode 100644 server/src/__tests__/helpers/maintenance/s3-boundary.ts create mode 100644 server/src/__tests__/instrumentation-signals.test.ts create mode 100644 server/src/__tests__/sleep-backup-observation.test.ts create mode 100644 server/src/__tests__/sleep-configuration-acceptance.test.ts create mode 100644 server/src/__tests__/sleep-early-diagnostics.test.ts create mode 100644 server/src/__tests__/sleep-idle-work-capture.test.ts create mode 100644 server/src/__tests__/sleep-incoming-http-observation.test.ts create mode 100644 server/src/__tests__/sleep-managed-fixture.test.ts create mode 100644 server/src/__tests__/sleep-managed-preflight.test.ts create mode 100644 server/src/__tests__/sleep-otel-export.test.ts diff --git a/doc/observability.md b/doc/observability.md index 3c0ba327c29..300743ae057 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -210,7 +210,8 @@ diagnostic line on boot and continues without tracing — your server stays up. ## Scope The OpenTelemetry export carries **traces only**. Metrics and log exporters -are out of scope and intentionally not configured here. Auto-instrumentations +are out of scope. The SDK receives explicit empty metric-reader and log-processor +lists so its default OTLP exporters cannot emit unrequested metrics or logs. Auto-instrumentations for `fs`, `dns`, and `net` are disabled by default because they are too chatty for this workload; everything else from `@opentelemetry/auto-instrumentations-node` is on (HTTP, Express, PG, etc.). diff --git a/doc/sleep-qualification.md b/doc/sleep-qualification.md new file mode 100644 index 00000000000..4bbacc05d00 --- /dev/null +++ b/doc/sleep-qualification.md @@ -0,0 +1,244 @@ +# Sleep configuration qualification + +This opt-in harness boots the actual server against an explicitly supplied fresh +test database. It measures local SQL, connection and network quiet. A local pass +does not establish Railway or Neon suspension. Keep a separate verdict for each +layer: quiet, eligibility, database suspension, host suspension, backup persistence, +inbound wake, timed wake and return to idle. + +## Local configured services + +Use a reviewed integrated source containing the incoming-wake and domain-deadline +contracts. Provision and migrate a fresh dedicated PostgreSQL database ending in +`_test` through the normal test operator. The suite does not allocate a database, +start embedded PostgreSQL or delete a database. It preserves its fixture rows. +Never supply a live tenant database. + +The configured exporter needs the exact optional peer versions already declared +in `server/package.json`. With Node 24 and npm available, explicitly install the +fixture peers into a new temporary prefix and link only missing packages in this +checkout: + +```sh +node scripts/sleep-qualification/setup-otel.mjs +``` + +The setup leaves application manifests and lockfiles unchanged. Retain the printed +setup receipt and temporary package lock. A version mismatch fails without +replacing an installed package. An opt-in CI job must run the same setup command; +setting an OTLP endpoint alone does not qualify an exporter. + +The helper records its exact symlink targets in +`server/node_modules/.paperclip-sleep-otel-links.json`. If an owned temporary prefix +disappears, rerunning repairs only those matching recorded symlinks. Existing +matching packages are left alone. An unrecorded dangling link or other entry is +preserved with a recovery error; its pathname is not evidence of ownership. +`node scripts/sleep-qualification/setup-otel.mjs --cleanup` unlinks only matching +owned links. It preserves foreign replacements and keeps temporary install +prefixes for setup evidence. Remove those owned prefixes separately when their +receipts are no longer needed. + +Set `PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE=1`, +`PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL` to the owned migrated fixture and +`PAPERCLIP_SLEEP_CLOUD_SHA` to the exact reviewed Cloud source SHA. Set +`PAPERCLIP_SLEEP_CLOUD_CHECKOUT` to that built private checkout; the suite verifies +its HEAD and imports its production broker adapter without copying private source. +Retain artifacts +with `PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS=1`. Run the two scoped Vitest files with +one worker and no file parallelism: + +```sh +pnpm exec vitest run server/src/__tests__/sleep-otel-export.test.ts \ + server/src/__tests__/sleep-configuration-acceptance.test.ts \ + --maxWorkers=1 --no-file-parallelism +``` + +The first case requires a successful named span at the real local OTLP collector +and no metric or log export during 90 real seconds, beyond the SDK's default +60-second metric cadence. It fails if the production SDK falls back to no-op. +The assembled case keeps +normal automatic backups, announcements, first-party telemetry and OTel enabled. +It creates an agent, useful completed process execution, API credential and future +routine through normal APIs. It waits the real ten-minute authenticated-use +cooldown, then measures at least 90 real seconds without tenant HTTP/SQL observer +polling. A passive listener on the actual app HTTP server counts incoming requests +and WebSocket upgrades, including direct clients outside the harness API helper. +It requires the app's existing request and upgrade handlers before installing; +it cannot become the sole upgrade listener and change unsupported-upgrade handling. +IPC snapshots record its capture identity, installation/validity, cumulative safe-integer +counts and observation times. Both snapshots must enclose the complete quiet interval, +remain valid and yield an exact zero delta. No URL, headers or body are recorded; +snapshot reads themselves make no HTTP requests or SQL queries. Missing or detached +capture, malformed counts and the legacy empty `observerRequests` list cannot prove +quiet. The passive protocol proxy separately measures database effects. It obtains +two owned signed safety reports after the window and preserves fixture history. +After ordinary API reactivation it waits another full real ten-minute cooldown, +measures another 90 seconds of quiet and requests fresh owned eligibility proof. +The helper keeps its legacy defaults for existing maintenance tests. + +Announcements are enabled and the actual route/feed service validates and caches +the known fixture. An owned test-child module wrapper injects the existing feed +transport seam for the exact default HTTPS URL; unmatched requests fail. Startup +fails if the wrapper did not intercept the actual app constructor. The receipt +requires one named fetch and no subsequent calls during quiet. Browser mode also +requires the fixture copy to render. The production DNS/HTTPS address guard stays +unchanged and is not exercised in this combined case; its owning tests cover that +boundary separately. Active user activity can legitimately keep an instance awake. + +The assembled scenario includes real tool approval rejection/delivery retirement +and signed chat-webhook completion followed by ordinary endpoint removal. That +removed-endpoint subcase is separate from the retained enabled GitHub connection: +the latter uses actual encrypted user credentials, refresh/continuity owners, +signed broker bindings and the production event delivery worker. The host wake +capability comes from the production connector store. Local in-memory broker +storage and synthetic GitHub HTTP responses remain explicit external boundaries; +this does not establish deployed broker durability or host suspension. These +assembled paths require a successful run against the reviewed integration before +their receipts can qualify them; source presence alone is not evidence. +Unknown plugins, persistent provider sessions, unfinished delivery/accounting, +uncertain writes and cleanup remain real blockers. Every inventory family needs +an explicit disposition before claiming all configurations qualified. + +## Evaluate a receipt + +```sh +pnpm exec vitest run server/scripts/sleep-qualification.test.mjs --maxWorkers=1 +node scripts/sleep-qualification/evidence.mjs receipt.json expected.json report.json +``` + +`expected.json` binds `appSha`, `cloudSha`, `target` and a SHA-256 `configDigest`. +The digest uses `logicalSleepConfiguration(configuration)`: explicit backup interval +and checkpoint policy, announcements, telemetry activation, OTLP activation/protocol, +and UI activation in a fixed order. It excludes ephemeral loopback ports. Compute +the expected logical contract independently before starting the fixture; the +evaluator recomputes the receipt digest. Keep actual `collectorOrigins.telemetry` +and `.otel` separately, matching the actual configuration and optional expected +origin bindings. + +Configured coverage must also have recorded observations. Announcements require +actual interception, a request for the expected feed, and no unmatched URL. +`signalObservation` records telemetry requests, trace requests, actual OTLP span +records and the named qualification span separately. Metric, log, malformed-trace +and unexpected request counts must be explicitly zero. The assembled case freezes +these observations after each quiet window; feature booleans are derived from +them and the observed backup checkpoint. Missing or malformed counts fail local +qualification even when a feature flag claims success. + +Before the normal cooldown, a signed diagnostic-only inventory saves both raw +reports. It explicitly expects the active API cooldown aggregate and the skipped +backup diagnostic; every other reported blocker must be absent. Aggregate local +probes cannot identify which probe is present, so this early inventory never +proves eligibility. Its hold is released before the full unchanged cooldown. +After quiet, raw ordinary reports are saved before assertions; a refusal also saves +a diagnostic report under that hold. + +Automatic backup coverage requires a new archive from the normal idle checkpoint, +not an enabled flag. After quiet counters freeze, the harness reads every new +owned gzip through its trailer, hashes compressed bytes, checks nonempty SQL for +the fixture identifier without retaining contents, and observes the wake marker +and absence of backup failure markers. This local evidence carries no provider +volume or restore claim. The owning backup suite separately qualifies restoration. +Archive provenance uses a positive finite opened-file birthtime, falling back to +positive finite content mtime only when the filesystem reports birthtime as zero. +The selected timestamp and its source are recorded. Existing names, old positive +birthtimes, malformed timestamps and future chronology remain disqualifying. +Original receipts are retained as produced; the new evidence fields are not +retroactively added to earlier runs. +The historical Full Y at `487e450b` and short composition at `57744f4e` wrote an +unmeasured empty observer-request list. Their recorded SQL, connection, outbound, +browser and producer observations remain pinned to those runs, but that list does +not meet the corrected inbound-observation contract. A new combined qualification +requires both real cooldowns and quiet windows with the installed incoming counter; +a short injected-poll regression cannot retroactively establish observer absence. +Receipt finalization follows all observations and runs the same evaluator before +writing the final receipt. A return-to-idle claim also needs explicit in-run quiet +start/end timestamps spanning at least 90 real seconds after the latest wake; +a duration field alone cannot prove it. +The evaluator rejects mismatched identities and chronology. Missing evidence stays +`INCONCLUSIVE`; a measured failed condition is `NOT VERIFIED`, including an +identity-verified active Neon endpoint or exact Railway service with running deployments. It does not produce +provider proof from local counters, fake time, a registry sleep flag or a successful +control response. Preserve original failures and their raw logs after a retry. + +## Deployed boundary + +No deployment target is authorized by this document. The private Cloud companion +provides a gated read-only provider observer. An operator must first approve one +exact owned disposable staging target, revisions, configuration, provider refs, +fixture operations and finite observation deadline. A named connector or shared +QA workspace is not that authorization. + +Capture app/Cloud identity and effective policy while awake. Use only provider and +central control-plane metadata during sleep; tenant health/API/SQL probes can wake +the target. Confirm the exact Neon endpoint is idle and all owned Railway services +have stopped deployments. Observe ordinary authorized API, broker and deadline +wake cycles separately. Verify useful persisted results and return to idle. +Automatic backup persistence additionally requires an exact attached volume +covering the configured home/backup directory and a fsynced sentinel/archive +continuity receipt across the owned stop/wake. A writable local fallback directory +is insufficient. Missing live access leaves these layers `INCONCLUSIVE`. + +For actual foreground/background browser coverage, first build the shipped UI into +`server/ui-dist` in the same reviewed checkout. Install the normal Playwright +Chromium runtime and run on a display-capable worker. Set +`PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE=1` for the configured suite. The harness loads +the actual UI, records an ordinary active reload, opens another real foreground +tab and requires Chromium to report the app tab hidden. It observes 90 real +seconds without app API traffic, then closes the browser before the server quiet +window. Failure to establish real visibility fails that mode; it never substitutes +a fake visibility event or frozen page. Without this explicit mode, the receipt +claims closed-browser coverage only. + +The browser uses a fresh owned Chromium profile and loopback debugging endpoint. +It attaches with `connectOverCDP({ noDefaults: true })` to the native default +context. Normal Playwright contexts install focus emulation that kept real +background tabs visible in the measured failure. This path avoids that override. +The receipt includes native window IDs, initial/final visibility and passive +visibility events for the entire 90-second interval. An intervening activation +fails the observation. Cleanup disconnects, terminates only the owned process and +removes its temporary profile. It never attaches to a user's existing browser. + + +## Managed fixture preflight + +Configured qualification uses the supported managed authenticated profile. The +actual app runs on loopback, with a lifetime-stable fixture vault key and the +normal managed configuration. A fixture Cloud front door supplies trusted tenant +headers to invoke the real primary-company/user/membership provisioning path; +ordinary company creation remains forbidden. Bearer-key and signed-control +requests omit tenant headers. Native browser headers can reach only the owned +app origin; foreign HTTP/WebSocket transports are blocked and fail qualification. + +Managed application storage requires S3. A bounded loopback protocol fixture +supplies the remote object service for actual SDK requests; unsupported operations, +framing and conditional requests fail closed. This establishes neither provider +IAM nor durable S3 storage. The instance home and database archives remain real +local filesystem paths. Normal managed migration, historical cleanup and routine +owners remain enabled. The logical digest binds managed activation, auth mode and +storage provider; actual ephemeral transport origins are recorded separately. + +Before another long campaign, run the opt-in `sleep-managed-preflight.test.ts` on +its own root-leased fresh external database using +`PAPERCLIP_SLEEP_MANAGED_PREFLIGHT=1` and an explicit dedicated +`PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_TEST_DATABASE_URL`. Retain artifacts with +`PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS=1`. It uses the same profile and negotiated +broker receiver, provisions real authority, admits and completes one process run, +and proves real key activity through signed diagnostics. It performs no browser, +ten-minute wait or quiet-window claim. No embedded allocation or automatic drop. +Loopback execution requires the host tool's explicit network/runtime permission. + +Setup and successful browser receipts are written immediately before later +assertions. Early diagnostics first use the existing owner-settled contract; they +reject durable blockers immediately and retry only observed local in-flight / +generation inspection incompleteness, up to three rounds. Every attempt is saved. +Both accepted reports must be strict, complete, structurally matching and under +the exact live hold, with each internal `inspection.generation=none` and no active +work before/after. Real HTTP authentication itself changes the global work +counter, so equality across deliberate observer requests is not asserted. + +A fresh successful opaque-key request anchors the later full real cooldown. +Neither setup stabilization nor the fast preflight substitutes for both real +600100ms cooldowns, both untouched 90000ms windows and useful reactivation. S3 +observations are saved before and after owned app cleanup, including failures; +cleanup preserves the original error. Full provider sleep remains a separate +unqualified layer until an exact deployed target is authorized and observed. diff --git a/scripts/sleep-qualification/evidence.d.mts b/scripts/sleep-qualification/evidence.d.mts new file mode 100644 index 00000000000..25a091fe9d0 --- /dev/null +++ b/scripts/sleep-qualification/evidence.d.mts @@ -0,0 +1,54 @@ +export interface SleepQualificationIdentity { + appSha: string; + cloudSha: string; + target: string; + configDigest: string; + collectorOrigins?: { telemetry: string; otel: string }; +} +export interface SleepQualificationReport { + version: number; + target: string; + appSha: string; + cloudSha: string; + configDigest: string; + layers: Record; +} +export function logicalSleepConfiguration(configuration: unknown): { + automaticBackups: { intervalMinutes: number; idleCheckpoint: boolean }; + announcements: boolean; + telemetry: boolean; + otel: { enabled: boolean; protocol: 'http/json' }; + serveUi: boolean; + managedInstance: boolean; + deploymentMode: 'authenticated' | 'local_trusted'; + storageProvider: 's3' | 'local_disk'; +}; +export function sleepConfigurationDigest(configuration: unknown): string; +export interface SleepSignalObservation { + telemetryRequests: number; traceRequests: number; exportedSpans: number; qualificationSpans: number; + metricRequests: number; logRequests: number; malformedTraceRequests: number; unexpectedRequests: number; +} +export interface IncomingHttpSnapshot { + source: 'node-http-server-events'; + captureId: string; + installed: boolean; + valid: boolean; + requests: number; + observedAt: string; +} +export interface IncomingHttpWindow { + before: IncomingHttpSnapshot; + after: IncomingHttpSnapshot; + requests: number; +} +export function observeIncomingHttpWindow(before: IncomingHttpSnapshot, after: IncomingHttpSnapshot): IncomingHttpWindow; +export function observedNoIncomingHttp(observation: unknown, quiet: { startedAt: string; finishedAt: string }): boolean; +export function observeConfiguredSignals(requests: ReadonlyArray<{ path: string; body: unknown }>): SleepSignalObservation; +export function configuredSleepFeatures(observations: Record): { + telemetry: boolean; announcements: boolean; automaticBackups: boolean; otel: boolean; exportedSpans: number; +}; +export function evaluateSleepQualification(receipt: unknown, expected: SleepQualificationIdentity): SleepQualificationReport; +export function finalizeLocalSleepReceipt>(observations: T, expected: SleepQualificationIdentity, now?: () => string): { + receipt: T & { finishedAt: string }; + report: SleepQualificationReport; +}; diff --git a/scripts/sleep-qualification/evidence.mjs b/scripts/sleep-qualification/evidence.mjs new file mode 100644 index 00000000000..d19b54fb034 --- /dev/null +++ b/scripts/sleep-qualification/evidence.mjs @@ -0,0 +1,231 @@ +import { createHash } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +const sha = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value); +const finite = value => typeof value === 'number' && Number.isFinite(value) && value >= 0; +const stamp = value => typeof value === 'string' && Number.isFinite(Date.parse(value)); +const layers = ['workerQuiet', 'eligibility', 'databaseSuspension', 'hostSuspension', 'backupPersistence', 'inboundWake', 'timedWake', 'returnToIdle']; +const result = (verdict, reason) => ({ verdict, reason }); +const announcementFeed = 'https://pages.paperclip.ing/announcements/v1/current.json'; +const observedAnnouncement = announcement => announcement?.intercepted === true + && Array.isArray(announcement.requests) && announcement.requests.length > 0 + && announcement.requests.every(url => url === announcementFeed) + && Array.isArray(announcement.unmatched) && announcement.unmatched.length === 0; +const signalKeys = ['telemetryRequests', 'traceRequests', 'exportedSpans', 'qualificationSpans', 'metricRequests', 'logRequests', 'malformedTraceRequests', 'unexpectedRequests']; +const knownSignalCounts = signal => signal && signalKeys.every(key => Number.isSafeInteger(signal[key]) && signal[key] >= 0); + +/** Preserve actual cumulative app-server snapshots; IPC reads create no HTTP traffic. */ +export function observeIncomingHttpWindow(before, after) { + return { before, after, requests: after?.requests - before?.requests }; +} +const knownIncomingHttpSnapshot = snapshot => snapshot?.source === 'node-http-server-events' + && typeof snapshot.captureId === 'string' && /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/.test(snapshot.captureId) + && snapshot.installed === true && snapshot.valid === true + && Number.isSafeInteger(snapshot.requests) && snapshot.requests >= 0 && stamp(snapshot.observedAt); +export const observedNoIncomingHttp = (observation, quiet) => knownIncomingHttpSnapshot(observation?.before) + && knownIncomingHttpSnapshot(observation?.after) && observation.before.captureId === observation.after.captureId + && stamp(quiet?.startedAt) && stamp(quiet?.finishedAt) && Date.parse(quiet.finishedAt) >= Date.parse(quiet.startedAt) + && Date.parse(observation.before.observedAt) <= Date.parse(quiet.startedAt) + && Date.parse(observation.after.observedAt) >= Date.parse(quiet.finishedAt) + && Number.isSafeInteger(observation.requests) && observation.requests === 0 + && observation.after.requests - observation.before.requests === observation.requests; + +/** Count actual collector requests and OTLP span records, never request-as-span estimates. */ +export function observeConfiguredSignals(requests) { + const observed = Object.fromEntries(signalKeys.map(key => [key, 0])); + for (const request of requests) { + if (typeof request?.path !== 'string' || !request.path.startsWith('/') || request.path.startsWith('//')) { observed.unexpectedRequests++; continue; } + const pathname = new URL(request.path, 'http://owned-collector').pathname; + if (pathname === '/telemetry') { observed.telemetryRequests++; continue; } + if (pathname === '/v1/metrics') { observed.metricRequests++; continue; } + if (pathname === '/v1/logs') { observed.logRequests++; continue; } + if (pathname !== '/v1/traces') { observed.unexpectedRequests++; continue; } + observed.traceRequests++; + try { + if (!Array.isArray(request.body?.resourceSpans)) throw new Error('Missing resourceSpans'); + let spans = 0, qualificationSpans = 0; + for (const resource of request.body.resourceSpans) { + if (!Array.isArray(resource?.scopeSpans)) throw new Error('Missing scopeSpans'); + for (const scope of resource.scopeSpans) { + if (!Array.isArray(scope?.spans)) throw new Error('Missing spans'); + for (const span of scope.spans) { + if (typeof span?.name !== 'string' || !span.name) throw new Error('Invalid span'); + spans++; if (span.name === 'sleep-qualification-stimulus') qualificationSpans++; + } + } + } + observed.exportedSpans += spans; observed.qualificationSpans += qualificationSpans; + } catch { observed.malformedTraceRequests++; } + } + return observed; +} + +/** Coverage booleans come from recorded activity; the evaluator also checks their provenance. */ +export function configuredSleepFeatures({ announcement, signalObservation, backupCheckpoint }) { + const known = knownSignalCounts(signalObservation); + return { telemetry: Boolean(known && signalObservation.telemetryRequests > 0), announcements: Boolean(observedAnnouncement(announcement)), + automaticBackups: backupCheckpoint?.owner === 'normal-idle-checkpoint' && backupCheckpoint.wakeMarkerObserved === true + && backupCheckpoint.failureObserved === false && Array.isArray(backupCheckpoint.archives) && backupCheckpoint.archives.length > 0, + otel: Boolean(known && signalObservation.traceRequests > 0 && signalObservation.exportedSpans > 0 + && signalObservation.qualificationSpans > 0 && signalObservation.qualificationSpans <= signalObservation.exportedSpans + && signalObservation.metricRequests === 0 && signalObservation.logRequests === 0 + && signalObservation.malformedTraceRequests === 0 && signalObservation.unexpectedRequests === 0), + exportedSpans: known ? signalObservation.exportedSpans : 0 }; +} +const validBackupCheckpoint = (backup, inRun, quietFinishedAt) => backup?.owner === 'normal-idle-checkpoint' && backup.wakeMarkerObserved === true + && backup.failureObserved === false && inRun(backup.startedAt) && inRun(backup.observedAt) + && Date.parse(backup.startedAt) >= Date.parse(quietFinishedAt) && Date.parse(backup.observedAt) >= Date.parse(backup.startedAt) && Array.isArray(backup.archives) && backup.archives.length > 0 + && backup.archives.every(archive => typeof archive.name === 'string' && /^[^/\\]+\.sql\.gz$/.test(archive.name) + && finite(archive.bytes) && archive.bytes > 0 && finite(archive.sqlBytes) && archive.sqlBytes > 0 + && /^[a-f0-9]{64}$/.test(archive.sha256 ?? '') && archive.gzipVerified === true && archive.fixtureIdentifierFound === true + && ['birthtime', 'mtime'].includes(archive.createdAtSource) + && inRun(archive.createdAt) && Date.parse(archive.createdAt) >= Date.parse(backup.startedAt) && Date.parse(archive.createdAt) <= Date.parse(backup.observedAt)); + +/** Canonical logical configuration: ephemeral collector addresses are bound separately. */ +export function logicalSleepConfiguration(configuration) { + if (!configuration || typeof configuration.announcements !== 'boolean' || typeof configuration.serveUi !== 'boolean' + || typeof configuration.telemetryOrigin !== 'string' || typeof configuration.otelOrigin !== 'string' + || (configuration.managedInstance !== undefined && typeof configuration.managedInstance !== 'boolean') + || (configuration.deploymentMode !== undefined && !['authenticated', 'local_trusted'].includes(configuration.deploymentMode)) + || (configuration.storageProvider !== undefined && !['s3', 'local_disk'].includes(configuration.storageProvider)) + || typeof configuration.automaticBackups?.idleCheckpoint !== 'boolean' + || !finite(configuration.automaticBackups?.intervalMinutes) || configuration.automaticBackups.intervalMinutes < 1) { + throw new Error('sleep_qualification_configuration_invalid'); + } + return { automaticBackups: { intervalMinutes: configuration.automaticBackups.intervalMinutes, + idleCheckpoint: configuration.automaticBackups.idleCheckpoint }, announcements: configuration.announcements, + telemetry: Boolean(configuration.telemetryOrigin), otel: { enabled: Boolean(configuration.otelOrigin), protocol: 'http/json' }, serveUi: configuration.serveUi, managedInstance: configuration.managedInstance ?? false, + deploymentMode: configuration.deploymentMode ?? 'local_trusted', storageProvider: configuration.storageProvider ?? 'local_disk' }; +} +export const sleepConfigurationDigest = configuration => createHash('sha256').update(JSON.stringify(logicalSleepConfiguration(configuration))).digest('hex'); + +/** Finish after observations, then validate the exact serialized local receipt. */ +export function finalizeLocalSleepReceipt(observations, expected, now = () => new Date().toISOString()) { + const receipt = { ...observations, finishedAt: now() }; + const report = evaluateSleepQualification(receipt, expected); + if (report.layers.workerQuiet.verdict !== 'VERIFIED' || report.layers.eligibility.verdict !== 'VERIFIED') { + throw new Error('sleep_qualification_local_evidence_invalid'); + } + return { receipt, report }; +} + +/** Evaluate this campaign's measured boundaries. A local run cannot certify a provider. */ +export function evaluateSleepQualification(receipt, expected) { + if (!receipt || receipt.version !== 1 || !expected || !sha(expected.appSha) + || !sha(expected.cloudSha) || typeof expected.target !== 'string' || !expected.target + || typeof expected.configDigest !== 'string' || !/^[a-f0-9]{64}$/.test(expected.configDigest) + || !stamp(receipt.startedAt) || !stamp(receipt.finishedAt) + || Date.parse(receipt.finishedAt) < Date.parse(receipt.startedAt) + || ['appSha', 'cloudSha', 'target', 'configDigest'].some(key => receipt[key] !== expected[key])) { + throw new Error('sleep_qualification_identity_or_window_invalid'); + } + if (sleepConfigurationDigest(receipt.configuration) !== expected.configDigest) throw new Error('sleep_qualification_configuration_digest_invalid'); + const origins = receipt.collectorOrigins; + if (!origins || origins.telemetry !== receipt.configuration.telemetryOrigin || origins.otel !== receipt.configuration.otelOrigin + || (expected.collectorOrigins && (origins.telemetry !== expected.collectorOrigins.telemetry || origins.otel !== expected.collectorOrigins.otel))) { + throw new Error('sleep_qualification_collector_origins_invalid'); + } + const output = Object.fromEntries(layers.map(layer => [layer, result('INCONCLUSIVE', 'No evidence for this layer')])); + const inRun = value => stamp(value) && Date.parse(value) >= Date.parse(receipt.startedAt) && Date.parse(value) <= Date.parse(receipt.finishedAt); + const quiet = receipt.quiet; + if (quiet) { + const observedFeatures = configuredSleepFeatures(receipt); + const valid = inRun(quiet.startedAt) && inRun(quiet.finishedAt) && finite(quiet.elapsedMs) + && Date.parse(quiet.finishedAt) - Date.parse(quiet.startedAt) >= 90_000 && quiet.elapsedMs >= 90_000 + && quiet.clock === 'real' && finite(quiet.sqlQueries) && finite(quiet.connectionsOpened) + && finite(quiet.connectionsRemaining) && finite(quiet.nonDatabaseRequests) && finite(quiet.nonDatabaseBytes) + && observedNoIncomingHttp(quiet.observerRequests, quiet) + && inRun(quiet.observerRequests.before.observedAt) && inRun(quiet.observerRequests.after.observedAt) + && observedFeatures.telemetry && observedFeatures.announcements && observedFeatures.otel + && Object.entries(observedFeatures).every(([key, value]) => receipt.features?.[key] === value) + && receipt.features?.telemetry === true && receipt.features?.announcements === true + && receipt.features?.automaticBackups === true && validBackupCheckpoint(receipt.backupCheckpoint, inRun, quiet.finishedAt) + && receipt.features?.otel === true && receipt.features?.exportedSpans > 0; + const idle = valid && quiet.sqlQueries === 0 && quiet.connectionsOpened === 0 && quiet.connectionsRemaining === 0 + && quiet.nonDatabaseRequests === 0 && quiet.nonDatabaseBytes === 0; + output.workerQuiet = result(idle ? 'VERIFIED' : 'NOT VERIFIED', idle ? 'Real configured quiet window, including closed DB connections' : 'Configured real quiet window missing, contaminated, or active'); + } + const eligibility = receipt.eligibility; + if (eligibility) { + const valid = inRun(eligibility.observedAt) && eligibility.backgroundWork === 'none' + && eligibility.ownedHold === true && eligibility.unchangedGeneration === true + && finite(eligibility.apiCooldownElapsedMs) && eligibility.apiCooldownElapsedMs >= 600_000; + output.eligibility = result(valid ? 'VERIFIED' : 'NOT VERIFIED', valid ? 'Owned stable safety proof after real API activity cooldown' : 'Safety proof or real API activity cooldown not established'); + } + const provider = receipt.provider; + if (provider) { + if (!provider.live || provider.target !== expected.target || provider.configDigest !== expected.configDigest + || provider.appSha !== expected.appSha || provider.cloudSha !== expected.cloudSha + || !inRun(provider.observedAt)) { + throw new Error('sleep_qualification_provider_identity_invalid'); + } + const untouched = Array.isArray(provider.observerRequests) + && provider.observerRequests.every(entry => entry.boundary === 'provider-control-plane' && (entry.method === 'GET' || (entry.provider === 'railway' && entry.method === 'POST' && entry.operation === 'query-only'))); + const endpoint = provider.neon; + if (untouched && endpoint?.identityVerified === true && endpoint.currentState === 'idle' && endpoint.pendingState === null) { + output.databaseSuspension = result('VERIFIED', 'Exact endpoint idle from live Neon control-plane metadata'); + } + if (untouched && endpoint?.identityVerified === true && (endpoint.currentState === 'active' + || (endpoint.currentState === 'idle' && endpoint.pendingState === 'active'))) { + output.databaseSuspension = result('NOT VERIFIED', 'Exact Neon endpoint is active or becoming active'); + } + const services = provider.railway?.services; + if (untouched && provider.railway?.identityVerified === true && provider.railway.state === 'OBSERVED' && Array.isArray(services) && services.length > 0 + && services.every(service => service.identityVerified === true && service.state === 'OBSERVED' && service.allDeploymentsStopped === true + && typeof service.backupsEnabled === 'boolean' && (service.backupsEnabled === false || service.backupStorageVerified === true))) { + output.hostSuspension = result('VERIFIED', 'Every exact owned service has stopped deployments'); + } + if (untouched && provider.railway?.identityVerified === true && Array.isArray(services) + && services.some(service => service.identityVerified === true && service.state === 'OBSERVED' && service.allDeploymentsStopped === false)) { + output.hostSuspension = result('NOT VERIFIED', 'An exact observed owned service still has running deployments'); + } + const storage = provider.storage; + if (output.hostSuspension.verdict === 'VERIFIED' && storage?.identityVerified === true + && storage.volumeId && storage.beforeVolumeId === storage.volumeId && storage.afterVolumeId === storage.volumeId + && storage.mountPath === '/paperclip' && storage.backupsEnabledBefore === true && storage.backupsEnabledAfter === true + && /^[a-f0-9]{64}$/.test(storage.sentinelBefore ?? '') && storage.sentinelBefore === storage.sentinelAfter + && inRun(storage.beforeAt) && inRun(storage.afterAt) && Date.parse(storage.beforeAt) < Date.parse(provider.observedAt) + && Date.parse(storage.afterAt) > Date.parse(provider.observedAt)) { + output.backupPersistence = result('VERIFIED', 'Attached volume and fsynced sentinel survive owned stop/wake with backups enabled'); + } + } + for (const [layer, kind] of [['inboundWake', 'inbound'], ['timedWake', 'deadline']]) { + const wake = receipt[layer]; + if (!wake) continue; + const sleeping = output.databaseSuspension.verdict === 'VERIFIED' && output.hostSuspension.verdict === 'VERIFIED'; + const chronology = provider && inRun(wake.stimulusAt) && inRun(wake.finishedAt) + && Date.parse(wake.stimulusAt) > Date.parse(provider.observedAt) && Date.parse(wake.finishedAt) >= Date.parse(wake.stimulusAt); + const alarm = kind !== 'deadline' || (inRun(wake.alarmPersistedAt) + && Date.parse(wake.alarmPersistedAt) < Date.parse(provider.observedAt) && wake.sameFinalGeneration === true); + if (!sleeping) { output[layer] = result('INCONCLUSIVE', 'Exact provider sleep and known backup policy have not been established'); continue; } + const verified = sleeping && chronology && alarm && wake.ordinaryProducer === true + && wake.authorizationVerified === true && wake.persistedOutcomeVerified === true && wake.duplicateEffects === 0; + output[layer] = result(verified ? 'VERIFIED' : 'NOT VERIFIED', verified ? `Useful authorized ${kind} wake after provider sleep` : `Missing provider sleep, chronology, authorization or useful ${kind} outcome`); + } + if (receipt.returnToIdle) { + const presentWakes = ['inboundWake', 'timedWake'].filter(layer => Object.hasOwn(receipt, layer)); + const everyWakeInRun = presentWakes.every(layer => inRun(receipt[layer]?.stimulusAt) && inRun(receipt[layer]?.finishedAt) + && Date.parse(receipt[layer].finishedAt) >= Date.parse(receipt[layer].stimulusAt)); + const lastWake = Math.max(...presentWakes.map(layer => Date.parse(receipt[layer]?.finishedAt))); + const idle = receipt.returnToIdle; + const usefulWake = ['inboundWake', 'timedWake'].some(layer => output[layer].verdict === 'VERIFIED'); + const valid = usefulWake && everyWakeInRun && Number.isFinite(lastWake) && inRun(idle.observedAt) && Date.parse(idle.observedAt) > lastWake + && inRun(idle.quietStartedAt) && inRun(idle.quietFinishedAt) + && Date.parse(idle.quietStartedAt) >= lastWake && Date.parse(idle.quietFinishedAt) <= Date.parse(idle.observedAt) + && Date.parse(idle.quietFinishedAt) - Date.parse(idle.quietStartedAt) >= 90_000 + && finite(idle.freshQuietWindowMs) && idle.freshQuietWindowMs >= 90_000 + && idle.freshQuietWindowMs <= Date.parse(idle.quietFinishedAt) - Date.parse(idle.quietStartedAt) && idle.backgroundWork === 'none' && idle.ownedHold === true; + output.returnToIdle = result(valid ? 'VERIFIED' : 'NOT VERIFIED', valid ? 'Fresh quiet owned proof after useful wake' : 'No fresh quiet proof after useful wake'); + } + return { version: 1, target: expected.target, appSha: expected.appSha, cloudSha: expected.cloudSha, configDigest: expected.configDigest, layers: output }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [receiptPath, expectedPath, outputPath] = process.argv.slice(2); + if (!receiptPath || !expectedPath || !outputPath) throw new Error('Usage: node scripts/sleep-qualification/evidence.mjs receipt.json expected.json output.json'); + const report = evaluateSleepQualification(JSON.parse(await readFile(receiptPath, 'utf8')), JSON.parse(await readFile(expectedPath, 'utf8'))); + await writeFile(outputPath, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 }); + process.stdout.write(JSON.stringify(report) + '\n'); + if (Object.values(report.layers).some(layer => layer.verdict === 'NOT VERIFIED')) process.exitCode = 1; +} diff --git a/scripts/sleep-qualification/setup-otel.mjs b/scripts/sleep-qualification/setup-otel.mjs new file mode 100644 index 00000000000..684d3b119fb --- /dev/null +++ b/scripts/sleep-qualification/setup-otel.mjs @@ -0,0 +1,125 @@ +import { lstat, mkdtemp, mkdir, readFile, readlink, rename, rm, symlink, unlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { randomUUID } from 'node:crypto'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +// Explicit opt-in test setup. No manifests, lockfiles or other worktrees change. +const server = fileURLToPath(new URL('../../server/', import.meta.url)); +export const otelPeerNames = ['@opentelemetry/sdk-node', '@opentelemetry/auto-instrumentations-node', '@opentelemetry/exporter-trace-otlp-http', '@opentelemetry/resources', '@opentelemetry/semantic-conventions']; +const ledgerPath = root => path.join(root, 'node_modules', '.paperclip-sleep-otel-links.json'); +async function entryAt(destination) { + try { return await lstat(destination); } catch (error) { if (error.code !== 'ENOENT') throw error; } +} +async function versionAt(root, name) { + try { return JSON.parse(await readFile(path.join(root, 'node_modules', name, 'package.json'), 'utf8')).version; } + catch (error) { if (error.code !== 'ENOENT') throw error; return undefined; } +} +async function readOwnership(root) { + const file = ledgerPath(root), entry = await entryAt(file); + if (!entry) return []; + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error('OTel ownership ledger must be a regular file'); + const record = JSON.parse(await readFile(file, 'utf8')); + if (record.version !== 1 || record.serverDirectory !== root || !Array.isArray(record.links) + || record.links.length > otelPeerNames.length || new Set(record.links.map(link => link.name)).size !== record.links.length + || record.links.some(link => !otelPeerNames.includes(link.name) || typeof link.target !== 'string' + || !path.isAbsolute(link.target) || path.resolve(link.target) !== link.target + || !link.target.endsWith(path.join('node_modules', link.name)))) throw new Error('Invalid OTel ownership ledger'); + return record.links; +} +async function writeOwnership(root, links) { + const file = ledgerPath(root); + if (!links.length) { await rm(file, { force: true }); return; } + await mkdir(path.dirname(file), { recursive: true }); + const temporary = `${file}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify({ version: 1, serverDirectory: root, links }, null, 2), { flag: 'wx' }); + await rename(temporary, file); + } finally { await rm(temporary, { force: true }); } +} +async function ownsLink(destination, record) { + const entry = await entryAt(destination); + return Boolean(record && entry?.isSymbolicLink() + && path.resolve(path.dirname(destination), await readlink(destination)) === record.target); +} +async function installWithNpm(directory) { + await new Promise((resolve, reject) => { + const child = spawn('npm', ['install', '--prefix', directory, '--ignore-scripts', '--no-audit', '--no-fund', '--prefer-offline'], { stdio: 'inherit', shell: false }); + child.once('error', reject); child.once('exit', code => code === 0 ? resolve() : reject(new Error(`OTel fixture installation failed (${code})`))); + }); +} + +/** Only the exact recorded symlink may be removed; install prefixes retain evidence. */ +export async function cleanupSleepOtelLinks({ serverDirectory = server } = {}) { + const root = path.resolve(serverDirectory), records = await readOwnership(root), remaining = [], removed = [], errors = []; + for (const record of records) { + const destination = path.join(root, 'node_modules', record.name); + if (!await entryAt(destination)) continue; + if (!await ownsLink(destination, record)) { + remaining.push(record); errors.push(new Error(`Refusing to clean unrelated ${record.name}`)); continue; + } + await unlink(destination); removed.push(record.name); + } + await writeOwnership(root, remaining); + if (errors.length) throw new AggregateError(errors, 'Some OTel destinations no longer match owned links'); + return { removed, installPrefixes: [...new Set(records.map(record => path.dirname(path.dirname(path.dirname(record.target)))))], boundary: 'Only recorded matching symlinks removed; temporary install prefixes are not deleted' }; +} + +export async function setupSleepOtelPeers({ serverDirectory = server, temporaryRoot = tmpdir(), install = installWithNpm } = {}) { + const root = path.resolve(serverDirectory), manifest = JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8')); + const owned = await readOwnership(root), missing = []; + for (const name of otelPeerNames) { + const exact = manifest.peerDependencies[name]; + if (!/^\d+\.\d+\.\d+$/.test(exact ?? '')) throw new Error(`Exact existing peer version required for ${name}`); + const installed = await versionAt(root, name); + if (installed && installed !== exact) throw new Error(`Installed ${name} does not match ${exact}; refusing to replace it`); + if (installed) continue; + const destination = path.join(root, 'node_modules', name); + if (await entryAt(destination) && !await ownsLink(destination, owned.find(link => link.name === name))) { + throw new Error(`Refusing to replace unrelated ${name}; restore its target or verify ownership before removing the unrecorded entry`); + } + missing.push(name); + } + if (!missing.length) return { linked: [], alreadyInstalled: true }; + const directory = await mkdtemp(path.join(temporaryRoot, 'paperclip-sleep-otel-')); + let linked = 0; + try { + await writeFile(path.join(directory, 'package.json'), JSON.stringify({ private: true, dependencies: Object.fromEntries(otelPeerNames.map(name => [name, manifest.peerDependencies[name]])) }, null, 2)); + await install(directory); + const resolved = Object.fromEntries(await Promise.all(otelPeerNames.map(async name => [name, await versionAt(directory, name)]))); + for (const name of otelPeerNames) if (resolved[name] !== manifest.peerDependencies[name]) throw new Error(`Resolved ${name} differs from pinned peer`); + const records = [...owned]; + for (const name of missing) { + const destination = path.join(root, 'node_modules', name), previous = owned.find(record => record.name === name); + await mkdir(path.dirname(destination), { recursive: true }); + if (await entryAt(destination)) { + if (!await ownsLink(destination, previous)) throw new Error(`OTel destination changed; refusing to replace ${name}`); + await unlink(destination); + } + const target = path.join(directory, 'node_modules', name); + await symlink(target, destination, 'dir'); linked++; + const updated = [...records.filter(record => record.name !== name), { name, target }]; + try { await writeOwnership(root, updated); } + catch (error) { + if (await ownsLink(destination, { name, target })) { await unlink(destination); linked--; } + throw error; + } + records.splice(0, records.length, ...updated); + } + const receipt = { version: 1, directory, packages: resolved, linked: missing, ownershipLedger: ledgerPath(root), cleanup: 'node scripts/sleep-qualification/setup-otel.mjs --cleanup' }; + await writeFile(path.join(directory, 'qualification-setup.json'), JSON.stringify(receipt, null, 2)); + return receipt; + } catch (error) { + // A prefix with published links stays available for explicit owned cleanup. + if (!linked) await rm(directory, { recursive: true, force: true }); + throw error; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const args = process.argv.slice(2); + if (args.length > 1 || (args.length && args[0] !== '--cleanup')) throw new Error('Usage: node scripts/sleep-qualification/setup-otel.mjs [--cleanup]'); + const receipt = args[0] === '--cleanup' ? await cleanupSleepOtelLinks() : await setupSleepOtelPeers(); + process.stdout.write(JSON.stringify(receipt) + '\n'); +} diff --git a/server/scripts/sleep-otel-setup.test.mjs b/server/scripts/sleep-otel-setup.test.mjs new file mode 100644 index 00000000000..ffbedb038f1 --- /dev/null +++ b/server/scripts/sleep-otel-setup.test.mjs @@ -0,0 +1,67 @@ +import { it } from 'vitest'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, readlink, readdir, lstat, rm, symlink, unlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { cleanupSleepOtelLinks, otelPeerNames, setupSleepOtelPeers } from '../../scripts/sleep-qualification/setup-otel.mjs'; + +async function fixture(run) { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-otel-setup-test-')); + const serverDirectory = path.join(directory, 'server'), temporaryRoot = path.join(directory, 'installs'); + await mkdir(serverDirectory); await mkdir(temporaryRoot); + await writeFile(path.join(serverDirectory, 'package.json'), JSON.stringify({ peerDependencies: Object.fromEntries(otelPeerNames.map(name => [name, '1.2.3'])) })); + let installs = 0; + const install = async prefix => { + installs++; + const manifest = JSON.parse(await readFile(path.join(prefix, 'package.json'), 'utf8')); + for (const [name, version] of Object.entries(manifest.dependencies)) { + const packageDirectory = path.join(prefix, 'node_modules', name); + await mkdir(packageDirectory, { recursive: true }); + await writeFile(path.join(packageDirectory, 'package.json'), JSON.stringify({ version })); + } + }; + const options = { serverDirectory, temporaryRoot, install }; + try { await run({ directory, options, install, installs: () => installs, destination: name => path.join(serverDirectory, 'node_modules', name) }); } + finally { await rm(directory, { recursive: true, force: true }); } +} +it('reruns after losing its temporary peers and cleans only its recorded matching links', async () => fixture(async f => { + const first = await setupSleepOtelPeers(f.options); + assert.equal(first.linked.length, 5); assert.equal(f.installs(), 1); + assert.equal((await setupSleepOtelPeers(f.options)).alreadyInstalled, true); assert.equal(f.installs(), 1); + await rm(first.directory, { recursive: true }); + assert.equal((await lstat(f.destination(otelPeerNames[0]))).isSymbolicLink(), true); + const repaired = await setupSleepOtelPeers(f.options); + assert.equal(f.installs(), 2); assert.notEqual(repaired.directory, first.directory); + for (const name of otelPeerNames) assert.equal(await readlink(f.destination(name)), path.join(repaired.directory, 'node_modules', name)); + const cleaned = await cleanupSleepOtelLinks(f.options); assert.equal(cleaned.removed.length, 5); + for (const name of otelPeerNames) await assert.rejects(lstat(f.destination(name)), { code: 'ENOENT' }); + assert.ok(await readFile(path.join(repaired.directory, 'qualification-setup.json'), 'utf8')); // Evidence retained. +})); +it('preserves unowned dangling links, incomplete directories and mismatched installed packages', async () => { + for (const kind of ['dangling', 'directory', 'mismatch']) await fixture(async f => { + const destination = f.destination(otelPeerNames[0]); await mkdir(path.dirname(destination), { recursive: true }); + if (kind === 'dangling') await symlink(path.join(f.directory, 'foreign-missing'), destination); + else { await mkdir(destination); await writeFile(path.join(destination, kind === 'mismatch' ? 'package.json' : 'keep.txt'), kind === 'mismatch' ? JSON.stringify({ version: '9.9.9' }) : 'foreign'); } + await assert.rejects(setupSleepOtelPeers(f.options), /refusing|Refusing/); assert.equal(f.installs(), 0); + if (kind === 'dangling') assert.equal(await readlink(destination), path.join(f.directory, 'foreign-missing')); + else assert.ok(await readFile(path.join(destination, kind === 'mismatch' ? 'package.json' : 'keep.txt'), 'utf8')); + }); +}); +it('keeps healthy unrelated packages and refuses cleanup after an owned link is replaced', async () => fixture(async f => { + const destination = f.destination(otelPeerNames[0]); await mkdir(destination, { recursive: true }); + await writeFile(path.join(destination, 'package.json'), JSON.stringify({ version: '1.2.3' })); + const receipt = await setupSleepOtelPeers(f.options); assert.equal(receipt.linked.length, 4); + const replaced = f.destination(otelPeerNames[1]); await unlink(replaced); + const foreign = path.join(f.directory, 'foreign'); await mkdir(foreign); await writeFile(path.join(foreign, 'keep.txt'), 'foreign'); + await symlink(foreign, replaced); + await assert.rejects(cleanupSleepOtelLinks(f.options), /no longer match/); + assert.equal(await readlink(replaced), foreign); assert.equal(await readFile(path.join(foreign, 'keep.txt'), 'utf8'), 'foreign'); + assert.equal(JSON.parse(await readFile(path.join(destination, 'package.json'), 'utf8')).version, '1.2.3'); +})); +it('preserves an entry introduced during installation and removes an unpublished failed install prefix', async () => fixture(async f => { + await assert.rejects(setupSleepOtelPeers({ ...f.options, install: async prefix => { + await f.install(prefix); const destination = f.destination(otelPeerNames[0]); await mkdir(destination, { recursive: true }); await writeFile(path.join(destination, 'keep.txt'), 'new foreign entry'); + } }), /destination changed/); + assert.equal(await readFile(path.join(f.destination(otelPeerNames[0]), 'keep.txt'), 'utf8'), 'new foreign entry'); + assert.equal((await readdir(f.options.temporaryRoot)).length, 0); +})); diff --git a/server/scripts/sleep-qualification.test.mjs b/server/scripts/sleep-qualification.test.mjs new file mode 100644 index 00000000000..c0a275152a5 --- /dev/null +++ b/server/scripts/sleep-qualification.test.mjs @@ -0,0 +1,233 @@ +import { it as test } from 'vitest'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { configuredSleepFeatures, evaluateSleepQualification, observeConfiguredSignals, observeIncomingHttpWindow, sleepConfigurationDigest, finalizeLocalSleepReceipt } from '../../scripts/sleep-qualification/evidence.mjs'; +const configuration = { automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, telemetryOrigin: 'http://127.0.0.1:4318/telemetry', otelOrigin: 'http://127.0.0.1:4318', serveUi: true }; +// Independent preflight contract, not copied from the receipt/digest implementation. +const configDigest = createHash('sha256').update(JSON.stringify({ automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, telemetry: true, otel: { enabled: true, protocol: 'http/json' }, serveUi: true, managedInstance: false, deploymentMode: 'local_trusted', storageProvider: 'local_disk' })).digest('hex'); +const expected = { appSha: 'a'.repeat(40), cloudSha: 'b'.repeat(40), target: 'owned-fixture', configDigest }; +function fixture() { + return { version: 1, ...expected, configuration: structuredClone(configuration), collectorOrigins: { telemetry: configuration.telemetryOrigin, otel: configuration.otelOrigin }, + announcement: { intercepted: true, requests: ['https://pages.paperclip.ing/announcements/v1/current.json'], unmatched: [] }, + signalObservation: { telemetryRequests: 1, traceRequests: 1, exportedSpans: 1, qualificationSpans: 1, metricRequests: 0, logRequests: 0, malformedTraceRequests: 0, unexpectedRequests: 0 }, + backupCheckpoint: { owner: 'normal-idle-checkpoint', startedAt: '2099-01-01T00:12:31Z', observedAt: '2099-01-01T00:13:00Z', wakeMarkerObserved: true, failureObserved: false, archives: [{ name: 'owned.sql.gz', bytes: 128, sqlBytes: 256, sha256: 'd'.repeat(64), gzipVerified: true, fixtureIdentifierFound: true, createdAt: '2099-01-01T00:12:45Z', createdAtSource: 'birthtime' }] }, startedAt: '2099-01-01T00:00:00Z', finishedAt: '2099-01-01T01:00:00Z', + features: { telemetry: true, announcements: true, automaticBackups: true, otel: true, exportedSpans: 1 }, + quiet: { startedAt: '2099-01-01T00:11:00Z', finishedAt: '2099-01-01T00:12:30Z', elapsedMs: 90_000, clock: 'real', sqlQueries: 0, connectionsOpened: 0, connectionsRemaining: 0, nonDatabaseRequests: 0, nonDatabaseBytes: 0, + observerRequests: observeIncomingHttpWindow( + { source: 'node-http-server-events', captureId: '11111111-1111-4111-8111-111111111111', installed: true, valid: true, requests: 12, observedAt: '2099-01-01T00:11:00Z' }, + { source: 'node-http-server-events', captureId: '11111111-1111-4111-8111-111111111111', installed: true, valid: true, requests: 12, observedAt: '2099-01-01T00:12:30Z' }) }, + eligibility: { observedAt: '2099-01-01T00:13:00Z', backgroundWork: 'none', ownedHold: true, unchangedGeneration: true, apiCooldownElapsedMs: 600_000 } }; +} +function deployed() { + const f = fixture(); + f.provider = { ...expected, live: true, observedAt: '2099-01-01T00:14:00Z', observerRequests: [{ method: 'GET', boundary: 'provider-control-plane' }], + neon: { identityVerified: true, currentState: 'idle', pendingState: null }, + railway: { identityVerified: true, state: 'OBSERVED', services: [{ identityVerified: true, state: 'OBSERVED', backupsEnabled: false, backupStorageVerified: false, allDeploymentsStopped: true }] } }; + return f; +} +test('local configured quiet leaves every provider and wake claim inconclusive', () => { + const r = evaluateSleepQualification(fixture(), expected).layers; + assert.equal(r.workerQuiet.verdict, 'VERIFIED'); assert.equal(r.eligibility.verdict, 'VERIFIED'); + for (const key of ['hostSuspension', 'databaseSuspension', 'backupPersistence', 'inboundWake', 'timedWake']) assert.equal(r[key].verdict, 'INCONCLUSIVE'); +}); +test('wrong source, target and configuration evidence cannot be combined', () => { + for (const key of Object.keys(expected)) { const f = fixture(); f[key] = 'wrong'; assert.throws(() => evaluateSleepQualification(f, expected), /identity/); } + const f = deployed(); f.provider.target = 'foreign'; assert.throws(() => evaluateSleepQualification(f, expected), /provider_identity/); +}); +test('fake time, a no-op exporter, open connections, observer health requests and a shortened cooldown fail', () => { + for (const change of [f => f.quiet.clock = 'qualified', f => f.features.exportedSpans = 0, f => f.quiet.connectionsRemaining = 1, + f => { f.quiet.observerRequests.after.requests++; f.quiet.observerRequests.requests++; }, f => f.quiet.nonDatabaseBytes = 1, f => f.quiet.elapsedMs = 89_999]) { + const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } + const f = fixture(); f.eligibility.apiCooldownElapsedMs = 599_999; assert.equal(evaluateSleepQualification(f, expected).layers.eligibility.verdict, 'NOT VERIFIED'); +}); +test('incoming HTTP observation must be installed, intact and cover the whole window; legacy empty lists are not evidence', () => { + const changes = [f => f.quiet.observerRequests = [], f => delete f.quiet.observerRequests, + f => f.quiet.observerRequests.before.installed = false, f => f.quiet.observerRequests.after.installed = false, + f => f.quiet.observerRequests.before.valid = false, f => f.quiet.observerRequests.after.valid = false, + f => delete f.quiet.observerRequests.before, f => delete f.quiet.observerRequests.after, + f => f.quiet.observerRequests.before.source = 'harness-api-only', f => f.quiet.observerRequests.after.source = 'unknown', + f => f.quiet.observerRequests.after.captureId = '22222222-2222-4222-8222-222222222222', + f => f.quiet.observerRequests.before.captureId = '', + f => f.quiet.observerRequests.before.observedAt = '2099-01-01T00:11:00.001Z', + f => f.quiet.observerRequests.after.observedAt = '2099-01-01T00:12:29.999Z', + f => f.quiet.observerRequests.before.observedAt = '2098-12-31T23:59:59Z', + f => f.quiet.observerRequests.after.observedAt = '2099-01-01T01:00:01Z', + f => f.quiet.observerRequests.after.observedAt = 'invalid', + f => f.quiet.observerRequests.after.requests--, + // A forged zero cannot hide an actual request while SQL and outbound counters remain zero. + f => f.quiet.observerRequests.after.requests++, + f => f.quiet.observerRequests.requests = 1]; + for (const point of ['before', 'after']) { + for (const value of [-1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '12']) { + changes.push(f => f.quiet.observerRequests[point].requests = value); + } + } + for (const value of [-1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '0']) changes.push(f => f.quiet.observerRequests.requests = value); + for (const change of changes) { + const f = fixture(); change(f); + assert.equal(f.quiet.sqlQueries, 0); assert.equal(f.quiet.nonDatabaseRequests, 0); + assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } +}); +test('configured feature claims require complete announcement and integer collector observations', () => { + const changes = [f => delete f.announcement, f => f.announcement.intercepted = false, + f => f.announcement.requests = [], f => f.announcement.requests = ['https://foreign.example/feed'], + f => f.announcement.unmatched.push('https://foreign.example/feed'), f => delete f.announcement.unmatched, + f => delete f.signalObservation, f => f.signalObservation.telemetryRequests = 0, + f => f.signalObservation.traceRequests = 0, f => f.signalObservation.exportedSpans = 0, + f => f.signalObservation.qualificationSpans = 0, f => f.signalObservation.qualificationSpans = 2, + f => f.signalObservation.metricRequests = 1, f => f.signalObservation.logRequests = 1, + f => f.signalObservation.malformedTraceRequests = 1, f => f.signalObservation.unexpectedRequests = 1, + f => f.features.telemetry = false, f => f.features.announcements = false, f => f.features.otel = false]; + for (const key of Object.keys(fixture().signalObservation)) { + changes.push(f => delete f.signalObservation[key]); + for (const value of [-1, 0.5, NaN, Infinity, '1']) changes.push(f => f.signalObservation[key] = value); + } + for (const change of changes) { const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); } +}); +test('assembled receipts derive coverage from actual named spans and requests, including unwanted exporters', () => { + const requests = [{ path: '/telemetry', body: {} }, { path: '/v1/traces', body: { resourceSpans: [{ scopeSpans: [{ spans: [ + { name: 'ordinary-http' }, { name: 'sleep-qualification-stimulus' }, + ] }] }] } }]; + const f = fixture(); f.signalObservation = observeConfiguredSignals(requests); f.features = configuredSleepFeatures(f); + assert.deepEqual(f.signalObservation, { telemetryRequests: 1, traceRequests: 1, exportedSpans: 2, qualificationSpans: 1, + metricRequests: 0, logRequests: 0, malformedTraceRequests: 0, unexpectedRequests: 0 }); + assert.equal(f.features.exportedSpans, 2); // One trace request contained two real span records. + assert.equal(finalizeLocalSleepReceipt(f, expected, () => f.finishedAt).report.layers.workerQuiet.verdict, 'VERIFIED'); + for (const extra of [{ path: '/v1/metrics?batch=1', body: {} }, { path: '/v1/logs', body: {} }, + { path: '/unmatched', body: {} }, { path: '/v1/traces', body: { resourceSpans: [{}] } }]) { + const changed = fixture(); changed.signalObservation = observeConfiguredSignals([...requests, extra]); changed.features = configuredSleepFeatures(changed); + assert.equal(evaluateSleepQualification(changed, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } + const noop = fixture(); noop.signalObservation = observeConfiguredSignals([{ path: '/telemetry', body: {} }, + { path: '/v1/traces', body: { resourceSpans: [{ scopeSpans: [{ spans: [{ name: 'other', attributes: [{ value: 'sleep-qualification-stimulus' }] }] }] }] } }]); + noop.features = configuredSleepFeatures(noop); assert.equal(noop.signalObservation.qualificationSpans, 0); + assert.equal(evaluateSleepQualification(noop, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); +}); +test('registry sleep, incomplete deployment inventory and a failed metadata read cannot prove providers stopped', () => { + const f = deployed(); f.provider.neon = { currentState: 'sleeping', identityVerified: true }; f.provider.railway.services = []; + let r = evaluateSleepQualification(f, expected).layers; assert.equal(r.databaseSuspension.verdict, 'INCONCLUSIVE'); assert.equal(r.hostSuspension.verdict, 'INCONCLUSIVE'); + f.provider.neon = { currentState: 'idle', pendingState: 'active', identityVerified: true }; r = evaluateSleepQualification(f, expected).layers; assert.equal(r.databaseSuspension.verdict, 'NOT VERIFIED'); +}); +test('useful authorized deadline outcome requires a durable alarm before the exact stop proof', () => { + const f = deployed(); f.timedWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0, sameFinalGeneration: true, alarmPersistedAt: '2099-01-01T00:13:30Z' }; + assert.equal(evaluateSleepQualification(f, expected).layers.timedWake.verdict, 'VERIFIED'); + for (const change of [x => x.timedWake.alarmPersistedAt = '2099-01-01T00:15:00Z', x => x.timedWake.sameFinalGeneration = false, x => x.timedWake.persistedOutcomeVerified = false, x => x.timedWake.duplicateEffects = 1]) { + const x = structuredClone(f); change(x); assert.equal(evaluateSleepQualification(x, expected).layers.timedWake.verdict, 'NOT VERIFIED'); + } +}); +test('writable path alone cannot establish backup persistence across stop and wake', () => { + const f = deployed(); f.provider.storage = { identityVerified: true, volumeId: 'volume', beforeVolumeId: 'volume', afterVolumeId: 'volume', mountPath: '/paperclip', backupsEnabledBefore: true, backupsEnabledAfter: true, sentinelBefore: 'd'.repeat(64), sentinelAfter: 'd'.repeat(64), beforeAt: '2099-01-01T00:13:00Z', afterAt: '2099-01-01T00:25:00Z' }; + assert.equal(evaluateSleepQualification(f, expected).layers.backupPersistence.verdict, 'VERIFIED'); + for (const change of [x => delete x.provider.storage.volumeId, x => x.provider.storage.afterVolumeId = 'replacement', x => x.provider.storage.backupsEnabledAfter = false, x => x.provider.storage.sentinelAfter = 'e'.repeat(64)]) { + const x = structuredClone(f); change(x); assert.equal(evaluateSleepQualification(x, expected).layers.backupPersistence.verdict, 'INCONCLUSIVE'); + } +}); + +test('unknown backup policy and unverified enabled storage cannot certify a stopped host or wake', () => { + for (const change of [x => delete x.provider.railway.state, x => x.provider.railway.state = 'INCONCLUSIVE', + x => delete x.provider.railway.services[0].backupsEnabled, x => x.provider.railway.services[0].backupsEnabled = null, + x => x.provider.railway.services[0].state = 'INCONCLUSIVE', x => x.provider.railway.services[0].backupsEnabled = true]) { + const f = deployed(); change(f); + f.inboundWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0 }; + f.timedWake = { ...f.inboundWake, alarmPersistedAt: '2099-01-01T00:13:30Z', sameFinalGeneration: true }; + const r = evaluateSleepQualification(f, expected).layers; + assert.equal(r.hostSuspension.verdict, 'INCONCLUSIVE'); assert.equal(r.inboundWake.verdict, 'INCONCLUSIVE'); assert.equal(r.timedWake.verdict, 'INCONCLUSIVE'); + } + const f = deployed(); assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'VERIFIED'); + f.provider.railway.services[0].backupsEnabled = true; f.provider.railway.services[0].backupStorageVerified = true; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'VERIFIED'); +}); + +function woken() { + const f = deployed(); + f.inboundWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0 }; + f.returnToIdle = { quietStartedAt: '2099-01-01T00:31:00Z', quietFinishedAt: '2099-01-01T00:32:30Z', observedAt: '2099-01-01T00:33:00Z', freshQuietWindowMs: 90_000, backgroundWork: 'none', ownedHold: true }; + return f; +} +test('return to idle requires a real 90-second window after the latest wake', () => { + assert.equal(evaluateSleepQualification(woken(), expected).layers.returnToIdle.verdict, 'VERIFIED'); + for (const finishedAt of ['invalid', '2098-12-31T23:59:59Z', '2099-01-01T01:00:01Z']) { + const f = woken(); f.timedWake = { ...f.inboundWake, finishedAt, alarmPersistedAt: '2099-01-01T00:13:30Z', sameFinalGeneration: true }; + assert.equal(evaluateSleepQualification(f, expected).layers.inboundWake.verdict, 'VERIFIED'); + assert.equal(evaluateSleepQualification(f, expected).layers.returnToIdle.verdict, 'NOT VERIFIED'); + } + + for (const change of [ + f => f.returnToIdle.quietFinishedAt = '2099-01-01T00:31:00.001Z', + f => { f.returnToIdle.quietStartedAt = '2099-01-01T00:19:00Z'; f.returnToIdle.quietFinishedAt = '2099-01-01T00:20:30Z'; }, + f => f.returnToIdle.quietStartedAt = '2098-12-31T23:00:00Z', + f => f.returnToIdle.quietFinishedAt = '2099-01-01T01:00:01Z', + f => f.returnToIdle.quietFinishedAt = '2099-01-01T00:30:00Z', + f => f.returnToIdle.observedAt = '2099-01-01T00:32:00Z', + f => f.returnToIdle.freshQuietWindowMs = Infinity, + f => f.returnToIdle.freshQuietWindowMs = 90_001, + f => delete f.returnToIdle.quietStartedAt, + ]) { + const f = woken(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.returnToIdle.verdict, 'NOT VERIFIED'); + } +}); +test('exact observed active providers are negative evidence while unknown identity stays inconclusive', () => { + for (const state of [{ currentState: 'active', pendingState: null }, { currentState: 'idle', pendingState: 'active' }]) { + const f = deployed(); Object.assign(f.provider.neon, state); + assert.equal(evaluateSleepQualification(f, expected).layers.databaseSuspension.verdict, 'NOT VERIFIED'); + f.provider.neon.identityVerified = false; + assert.equal(evaluateSleepQualification(f, expected).layers.databaseSuspension.verdict, 'INCONCLUSIVE'); + } + const f = deployed(); f.provider.railway.services[0].allDeploymentsStopped = false; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'NOT VERIFIED'); + f.provider.railway.services[0].identityVerified = false; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'INCONCLUSIVE'); +}); + +test('logical digest is stable across origins and detects changed enabled configuration', () => { + assert.equal(sleepConfigurationDigest(configuration), configDigest); + const f = fixture(); f.configuration.telemetryOrigin = 'http://127.0.0.1:5555/telemetry'; f.configuration.otelOrigin = 'http://127.0.0.1:5555'; + f.collectorOrigins = { telemetry: f.configuration.telemetryOrigin, otel: f.configuration.otelOrigin }; + assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'VERIFIED'); + assert.throws(() => evaluateSleepQualification(f, { ...expected, collectorOrigins: fixture().collectorOrigins }), /collector_origins/); + for (const change of [x => x.configuration.announcements = false, x => x.configuration.serveUi = false, + x => x.configuration.automaticBackups.intervalMinutes = 5, x => x.configuration.automaticBackups.idleCheckpoint = false, + x => x.configuration.managedInstance = true, x => x.configuration.deploymentMode = 'authenticated', x => x.configuration.storageProvider = 's3', + x => x.configuration.otelOrigin = '', x => x.configuration.telemetryOrigin = '']) { + const changed = fixture(); change(changed); assert.throws(() => evaluateSleepQualification(changed, expected), /configuration_digest/); + } +}); +test('configured backup feature requires observed verified owned archives and a wake marker', () => { + for (const change of [f => delete f.backupCheckpoint, f => f.backupCheckpoint.archives = [], + f => f.backupCheckpoint.wakeMarkerObserved = false, f => f.backupCheckpoint.failureObserved = true, + f => f.backupCheckpoint.archives[0].gzipVerified = false, f => f.backupCheckpoint.archives[0].fixtureIdentifierFound = false, + f => f.backupCheckpoint.archives[0].bytes = 0, f => f.backupCheckpoint.archives[0].sha256 = 'invalid', + f => f.backupCheckpoint.archives[0].createdAt = '2098-01-01T00:00:00Z', + f => f.backupCheckpoint.archives[0].createdAtSource = 'unknown', + f => f.backupCheckpoint.startedAt = '2099-01-01T00:11:00Z']) { + const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } +}); +test('receipt finalization follows observations and validates the actual resulting receipt', () => { + const f = fixture(); delete f.finishedAt; + let tick = Date.parse(f.eligibility.observedAt); + const { receipt, report } = finalizeLocalSleepReceipt(f, expected, () => new Date(++tick).toISOString()); + assert.ok(Date.parse(receipt.finishedAt) > Date.parse(receipt.eligibility.observedAt)); + assert.equal(report.layers.eligibility.verdict, 'VERIFIED'); + assert.throws(() => finalizeLocalSleepReceipt(f, expected, () => f.startedAt), /local_evidence/); +}); +test('CLI exits one for known active providers and zero for inconclusive identity', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-evidence-cli-')); + try { + const input = path.join(directory, 'receipt.json'), expectedPath = path.join(directory, 'expected.json'), output = path.join(directory, 'report.json'); + await writeFile(expectedPath, JSON.stringify(expected)); + const f = deployed(); f.provider.neon.currentState = 'active'; + for (const [identityVerified, status] of [[true, 1], [false, 0]]) { + f.provider.neon.identityVerified = identityVerified; await writeFile(input, JSON.stringify(f)); + const child = spawnSync(process.execPath, [fileURLToPath(new URL('../../scripts/sleep-qualification/evidence.mjs', import.meta.url)), input, expectedPath, output], { encoding: 'utf8' }); + assert.equal(child.status, status, child.stderr); + } + } finally { await rm(directory, { recursive: true, force: true }); } +}); diff --git a/server/src/__tests__/helpers/maintenance/announcement-boundary.ts b/server/src/__tests__/helpers/maintenance/announcement-boundary.ts new file mode 100644 index 00000000000..44d642fe9c0 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/announcement-boundary.ts @@ -0,0 +1,36 @@ +import { mock } from 'node:test'; +import { DEFAULT_ANNOUNCEMENT_FEED_URL, announcementManifestSchema } from '@paperclipai/shared'; + +/** Test-child transport seam: actual route/feed/cache, no DNS or HTTPS claim. */ +export async function installMaintenanceAnnouncementBoundary() { + const actual = await import('../../../app.js'); + const requests: string[] = [], unmatched: string[] = []; + let intercepted = false; + const manifest = announcementManifestSchema.parse({ schemaVersion: 1, announcement: { + id: 'new-projects', eyebrow: 'New', title: 'Projects', description: 'Organize your work.', + primaryAction: { kind: 'route', label: 'Open', path: '/projects' }, + } }); + const wrapped: typeof actual.createApp = (db, options) => { + const announcements = options.announcements; + if (!announcements?.enabled || announcements.feedUrl !== DEFAULT_ANNOUNCEMENT_FEED_URL) { + throw new Error('Configured announcement boundary requires the exact enabled default feed'); + } + intercepted = true; + const fetch = async (url: URL, init: RequestInit) => { + if (url.href !== DEFAULT_ANNOUNCEMENT_FEED_URL || init.method !== 'GET') { + unmatched.push(url.href); + throw new Error('Unmatched announcement transport is forbidden'); + } + requests.push(url.href); + return Response.json(manifest, { headers: { ETag: '"qualification-v1"' } }); + }; + const withTransport = { ...options, announcements: { ...announcements, fetch } }; + return actual.createApp(db, withTransport); + }; + mock.module(new URL('../../../app.ts', import.meta.url).href, { namedExports: { ...actual, createApp: wrapped } }); + // Fail before server startup if the TS loader/module cache bypasses the mock. + if ((await import('../../../app.js')).createApp !== wrapped) throw new Error('Announcement module boundary did not intercept the startup import'); + return { assertIntercepted() { if (!intercepted) throw new Error('Actual server bypassed the announcement boundary'); }, + snapshot: () => ({ intercepted, requests: [...requests], unmatched: [...unmatched], + boundary: 'Feed transport injected through existing fetch seam; actual route/feed validation/cache/UI. DNS and HTTPS guard are not exercised in this combined case.' }) }; +} diff --git a/server/src/__tests__/helpers/maintenance/backup-observation.ts b/server/src/__tests__/helpers/maintenance/backup-observation.ts new file mode 100644 index 00000000000..c3a273c1346 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/backup-observation.ts @@ -0,0 +1,54 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { lstat, open, readdir } from 'node:fs/promises'; +import path from 'node:path'; +import { Transform, Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { createGunzip } from 'node:zlib'; + +/** Inspect only archives created by the normal idle owner after the quiet counters freeze. + * Stream through EOF to validate the gzip trailer; retain no SQL/credential contents. */ +export async function observeIdleBackupCheckpoint(directory: string, beforeNames: string[], startedAt: string, fixtureId: string) { + if (!/^[a-f0-9-]{36}$/.test(fixtureId)) throw new Error('Invalid owned backup fixture identifier'); + const checkpointMs = Date.parse(startedAt); + if (!Number.isFinite(checkpointMs) || checkpointMs <= 0) throw new Error('Invalid owned checkpoint timestamp'); + const backupDir = path.join(directory, 'backups'); + const marker = await lstat(path.join(backupDir, '.idle-backup-wake-required')); + if (!marker.isFile() || marker.isSymbolicLink() || marker.size === 0) throw new Error('Missing normal idle backup wake marker'); + for (const failure of [path.join(directory, 'health/db-backup-to-s3.failure'), path.join(directory, 'db-backup-to-s3.failure'), path.join(backupDir, 'db-backup-to-s3.failure')]) { + try { await lstat(failure); throw new Error('Observed database backup failure marker'); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + } + const names = (await readdir(backupDir)).filter(name => name.endsWith('.sql.gz') && !beforeNames.includes(name)).sort(); + if (!names.length) throw new Error('Normal idle owner produced no new archive'); + const archives = []; + for (const name of names) { + const handle = await open(path.join(backupDir, name), constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await handle.stat(); + // Some filesystems report birthtime=0. The new-name filter and opened + // non-symlink file still apply; use its content mtime only in that case. + // An old positive birthtime must not be rescued by a recent modification. + const createdAtSource = stat.birthtimeMs === 0 ? 'mtime' : 'birthtime'; + const createdMs = stat.birthtimeMs === 0 ? stat.mtimeMs : stat.birthtimeMs; + if (!stat.isFile() || stat.size <= 0 || !Number.isFinite(createdMs) || createdMs <= 0 + || createdMs < checkpointMs || Math.trunc(createdMs) > Date.now()) throw new Error('Archive predates owned checkpoint or is invalid'); + const hash = createHash('sha256'); let sqlBytes = 0, tail = '', fixtureIdentifierFound = false; + await pipeline(handle.createReadStream({ autoClose: false }), + new Transform({ transform(chunk, _encoding, done) { hash.update(chunk); done(null, chunk); } }), createGunzip(), + new Writable({ write(chunk, _encoding, done) { + sqlBytes += chunk.length; + if (sqlBytes > 512 * 1024 * 1024) { done(new Error('Owned fixture archive exceeded bounded observation size')); return; } + const text = tail + chunk.toString('utf8'); fixtureIdentifierFound ||= text.includes(fixtureId); + tail = text.slice(-fixtureId.length); done(); + } })); + if (!sqlBytes || !fixtureIdentifierFound) throw new Error('Archive does not contain the owned fixture'); + archives.push({ name, bytes: stat.size, sqlBytes, sha256: hash.digest('hex'), createdAt: new Date(createdMs).toISOString(), createdAtSource, gzipVerified: true, fixtureIdentifierFound }); + } finally { await handle.close(); } + } + return { owner: 'normal-idle-checkpoint', startedAt, observedAt: new Date().toISOString(), archives, wakeMarkerObserved: true, failureObserved: false }; +} +export async function idleBackupArchiveNames(directory: string) { + try { return (await readdir(path.join(directory, 'backups'))).filter(name => name.endsWith('.sql.gz')); } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error; } +} diff --git a/server/src/__tests__/helpers/maintenance/browser-observation.ts b/server/src/__tests__/helpers/maintenance/browser-observation.ts new file mode 100644 index 00000000000..76b4488d434 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/browser-observation.ts @@ -0,0 +1,82 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { withNativeMaintenanceBrowser } from './native-browser.js'; + +/** Actual Chromium visibility; never substitutes a fake document or frozen lifecycle. */ +export async function observeMaintenanceBrowser(origin: string, announcementDescription?: string, tenantHeaders?: Record) { + return withNativeMaintenanceBrowser(async (browser, context) => { + const unmatchedOrigins: string[] = []; + if (tenantHeaders) { + // Both HTTP and native WS need the simulated Cloud front door identity. + // Gate every possible foreign transport BEFORE networking; no fixture + // header can reach another origin. about:blank does not use networking. + await context.route('**/*', async route => { + if (new URL(route.request().url()).origin !== origin) { unmatchedOrigins.push(route.request().url()); await route.abort('blockedbyclient'); return; } + await route.continue(); + }); + await context.routeWebSocket(url => url.origin.replace(/^ws/, 'http') !== origin, route => { + unmatchedOrigins.push(route.url()); route.close({ code: 1008, reason: 'Outside owned app origin' }); + }); + await context.setExtraHTTPHeaders(tenantHeaders); + } + const page = await context.newPage(); + const requests: Array<{ at: number; path: string }> = []; + page.on('request', request => { const url = new URL(request.url()); if (url.origin === origin) requests.push({ at: performance.now(), path: url.pathname }); }); + await page.goto(origin, { waitUntil: 'networkidle' }); + if (announcementDescription) await page.getByText(announcementDescription, { exact: true }).waitFor({ state: 'visible' }); + await page.bringToFront(); + if (await page.evaluate(() => document.visibilityState) !== 'visible') throw new Error('Browser active-use observation requires a visible page'); + const beforeActive = requests.length; + await page.reload({ waitUntil: 'networkidle' }); + if (!requests.slice(beforeActive).some(request => request.path.startsWith('/api/'))) throw new Error('Active browser did not make ordinary app requests'); + const visibilityEvents: Array<{ state: string; rendererAt: number }> = []; + const prefix = 'paperclip-sleep-visibility:'; + page.on('console', message => { if (message.text().startsWith(prefix)) visibilityEvents.push(JSON.parse(message.text().slice(prefix.length))); }); + // Passive native events only; never replace document visibility properties. + await page.evaluate(prefix => document.addEventListener('visibilitychange', () => console.log(prefix + JSON.stringify({ state: document.visibilityState, rendererAt: performance.now() }))), prefix); + const visibility = () => page.evaluate(() => ({ state: document.visibilityState, focused: document.hasFocus(), rendererAt: performance.now(), + ownVisibilityOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'visibilityState')), ownHiddenOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hidden')), + ownFocusOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hasFocus')) })); + const initialVisibility = await visibility(); + if (initialVisibility.state !== 'visible') throw new Error('Native browser is not visible before the actual tab switch'); + const foreground = await context.newPage(); + await foreground.goto('about:blank'); await foreground.bringToFront(); + const native = await browser.newBrowserCDPSession(); + const windowIdentity = async (target: typeof page) => { + const session = await context.newCDPSession(target); + try { + const { targetInfo } = await session.send('Target.getTargetInfo'); + const { windowId } = await native.send('Browser.getWindowForTarget', { targetId: targetInfo.targetId }); + return { targetId: targetInfo.targetId, windowId }; + } finally { await session.detach(); } + }; + const appWindow = await windowIdentity(page), foregroundWindow = await windowIdentity(foreground); + if (appWindow.windowId !== foregroundWindow.windowId) throw new Error('Background browser qualification requires an actual same-window tab switch'); + const hiddenDeadline = performance.now() + 5_000; + while (await page.evaluate(() => document.visibilityState) !== 'hidden') { + if (performance.now() > hiddenDeadline) throw new Error('Browser could not establish a real background tab; visibility simulation is not qualification'); + await delay(50); + } + // Let requests already admitted by the transition settle before the untouched window. + await delay(2_000); + const initialHidden = await visibility(); + if (initialHidden.state !== 'hidden') throw new Error('Browser lost native hidden state during settling'); + if (!visibilityEvents.some(event => event.state === 'hidden' && event.rendererAt <= initialHidden.rendererAt)) throw new Error('Native hidden event was not observed'); + const hiddenRequests = requests.length, started = performance.now(); + await delay(90_000); + const finalHidden = await visibility(); + const hiddenRealElapsedMs = performance.now() - started; + if (hiddenRealElapsedMs < 90_000) throw new Error('Native hidden observation was shorter than 90 real seconds'); + const intervalVisibilityEvents = visibilityEvents.filter(event => event.rendererAt >= initialHidden.rendererAt && event.rendererAt <= finalHidden.rendererAt); + if (finalHidden.state !== 'hidden' || intervalVisibilityEvents.some(event => event.state !== 'hidden')) throw new Error(`Browser left native hidden state: ${JSON.stringify({ initialHidden, finalHidden, intervalVisibilityEvents })}`); + if ([initialVisibility, initialHidden, finalHidden].some(state => state.ownVisibilityOverride || state.ownHiddenOverride || state.ownFocusOverride)) throw new Error('Browser visibility or focus properties were overridden'); + const hidden = requests.slice(hiddenRequests); + if (hidden.some(request => request.path.startsWith('/api/'))) throw new Error(`Background tab produced tenant traffic: ${JSON.stringify(hidden)}`); + if (unmatchedOrigins.length) throw new Error(`Browser attempted non-owned transport: ${JSON.stringify(unmatchedOrigins)}`); + await page.bringToFront(); const restoredVisibility = await visibility(); + if (restoredVisibility.state !== 'visible') throw new Error('Native app tab did not become visible after activation'); + if (!visibilityEvents.some(event => event.state === 'visible' && event.rendererAt >= finalHidden.rendererAt)) throw new Error('Native visible restoration event was not observed'); + return { activeRequests: requests.slice(beforeActive, hiddenRequests).length, hiddenRealElapsedMs, + hiddenRequests: hidden, unmatchedOrigins, visibility: 'Native Chromium default context with no Playwright focus override', initialVisibility, initialHidden, finalHidden, + intervalVisibilityEvents, visibilityEvents, restoredVisibility, appWindow, foregroundWindow, announcementRendered: Boolean(announcementDescription), authBoundary: tenantHeaders ? 'Owned Cloud front-door headers; actual tenant resolver/membership' : 'Legacy local board', closed: true }; + }); +} diff --git a/server/src/__tests__/helpers/maintenance/chat-history-producer.ts b/server/src/__tests__/helpers/maintenance/chat-history-producer.ts new file mode 100644 index 00000000000..d2050cc5ba9 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/chat-history-producer.ts @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import type { Db } from '@paperclipai/db'; +import { githubIngressFixture } from '../github-webhook-ingress.js'; +import { chatChannelService } from '../../../services/chat-channels.js'; + +/** The fixture supplies the named external signed-webhook boundary; domain owners write receipts. */ +export async function produceChatHistory(db: Db) { + const fixture = await githubIngressFixture(db); + const action = await fixture.insertAction(); + await fixture.service.processGitHubWebhookIngress(action.id); + const finished = await fixture.readAction(action.id); + assert.equal(fixture.calls.length, 1); + assert.equal(finished.status, 'processed'); + const service = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error('Unexpected chat turn'); } }, + fetch: async () => { throw new Error('Unexpected provider HTTP outside signed-webhook fixture'); } }); + try { + await service.configure(fixture.endpoint.id, { action: 'remove' }, 'local-board'); + } finally { await service.shutdown(); } + return { companyId: fixture.companyId, actionId: action.id, endpointId: fixture.endpoint.id, + boundary: 'Existing signed webhook fixture input; actual chat ingress completion and ordinary endpoint removal preserve history' }; +} diff --git a/server/src/__tests__/helpers/maintenance/configuration-producers.ts b/server/src/__tests__/helpers/maintenance/configuration-producers.ts new file mode 100644 index 00000000000..a67d82912ec --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/configuration-producers.ts @@ -0,0 +1,48 @@ +import type { createDb } from '@paperclipai/db'; +import { expect } from 'vitest'; +import { randomUUID } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { startMaintenanceServer } from './server-harness.js'; + +/** Real routes own request creation, human rejection, run cancellation and outbox retirement. */ +export async function produceCompletedToolReview(server: Awaited>, db: ReturnType, companyId: string) { + await server.api('POST', `/companies/${companyId}/tools/policies`, { name: 'Review harmless qualification writes', policyType: 'require_approval', selectors: { toolName: 'mcp-remote-fixture:update_note' } }); + const readyPath = path.join(server.directory, `tool-review-ready-${randomUUID()}`); + const readyProgram = `const fs=require('node:fs');const marker=${JSON.stringify(readyPath)};fs.writeFileSync(marker+'.pending',process.env.PAPERCLIP_RUN_ID,{flag:'wx'});fs.renameSync(marker+'.pending',marker);setTimeout(()=>process.exit(0),60000)`; + const agent = await server.api<{ id: string }>('POST', `/companies/${companyId}/agents`, { name: 'Owned tool review process', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', readyProgram], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const issue = await server.api<{ id: string }>('POST', `/companies/${companyId}/issues`, { title: 'Harmless tool review history', status: 'todo', assigneeAgentId: agent.id }); + let runId: string | undefined; + await expect.poll(async () => { + const [run] = await db.$client`select id,status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; + runId = run?.id; + if (run?.status !== 'running') return false; + // The process runs only after its awaited adapter.invoke write can commit + // behind the final dispatch gate's run lock. Status alone precedes setup. + try { return await readFile(readyPath, 'utf8') === run.id; } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; } + }, { timeout: 30_000 }).toBe(true); + const readiness = { runId, marker: path.basename(readyPath), observedAt: new Date().toISOString(), source: 'actual-process-run-id-marker' }; + await writeFile(path.join(server.directory, 'sleep-tool-review-readiness.json'), JSON.stringify(readiness, null, 2)); + const session = await server.api<{ sessionId: string; token: string }>('POST', '/tool-gateway/sessions', { companyId, agentId: agent.id, runId, issueId: issue.id }); + const call = await fetch(`http://127.0.0.1:${server.port}/api/tool-gateway/tools/call`, { method: 'POST', signal: AbortSignal.timeout(15_000), + headers: { 'Content-Type': 'application/json', 'x-paperclip-tool-gateway-token': session.token }, + body: JSON.stringify({ tool: 'mcp-remote-fixture:update_note', parameters: { noteId: 'qualification', body: 'Harmless denied operation' } }) }); + expect(call.ok).toBe(false); + expect(JSON.stringify(await call.json())).toContain('approval_required'); + const [request] = await db.$client`select id,status from tool_action_requests where issue_id=${issue.id}`; + expect(request.status).toBe('pending'); + await server.api('POST', `/tool-gateway/action-requests/${request.id}/decline`, { companyId, reason: 'Qualification declines the harmless write' }); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + await expect.poll(async () => { + const [run] = await db.$client`select status from heartbeat_runs where id=${runId!}`; + return ['succeeded', 'failed', 'cancelled', 'timed_out'].includes(run?.status); + }, { timeout: 75_000 }).toBe(true); + await expect.poll(async () => { + const [delivery] = await db.$client`select delivered_at is not null as delivered from tool_action_deliveries where action_request_id=${request.id}`; + return delivery?.delivered; + }, { timeout: 30_000 }).toBe(true); + await server.api('POST', `/tool-gateway/sessions/${session.sessionId}/revoke`, { companyId }); + return { requestId: request.id, issueId: issue.id, runId, readiness, boundary: 'Actual request/rejection/session revocation and closed-task retirement; no provider write executed' }; +} diff --git a/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts b/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts new file mode 100644 index 00000000000..c51c4e337c1 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { assertEarlySleepDiagnostics } from './early-sleep-diagnostics.js'; +import type { startMaintenanceServer } from './server-harness.js'; + +type Report = { ownerId: string; draining: boolean; quiescent: boolean; expiresAt: string; + idleSleepSafety: { diagnostics?: { complete?: boolean; checks?: Array<{ reason: string; state: string }> } } }; +const transientStates = new Map([['inspection.generation', 'unknown'], ['local.tracked_work', 'present'], ['plugins.runtime_drain', 'not_checked']]); + +export function classifyEarlyDiagnostic(report: Report, elapsedSinceApiMs: number): 'strict' | 'transient' { + const checks = report.idleSleepSafety.diagnostics?.checks; + if (!Array.isArray(checks)) throw new Error('Missing early diagnostic checks'); + // A real durable blocker fails now, even if unrelated inspection work is in flight. + const expectedStates = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]); + for (const check of checks) { + if (check.state === (expectedStates.get(check.reason) ?? 'none')) continue; + if (transientStates.get(check.reason) !== check.state) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`); + } + if (checks.some(check => transientStates.get(check.reason) === check.state)) return 'transient'; + assertEarlySleepDiagnostics(report, elapsedSinceApiMs); + return 'strict'; +} + +/** Bounded setup-only observation, never a quiet-window poll or an uncertainty exemption. */ +export async function stabilizeEarlyDiagnostics(server: Awaited>, options: { + ownerId: string; apiSucceededAt: number; read: () => Promise; persist: (observations: unknown[]) => Promise; +}) { + const observations: unknown[] = []; + for (let round = 1; round <= 3; round++) { + const before = await server.settled(); + observations.push({ round, kind: 'settled-before', at: new Date().toISOString(), snapshot: before }); await options.persist(observations); + assert(before.managed?.apiActivityActive, 'Actual successful managed API key activity is not held'); + const reports: Report[] = []; let transient = false; + for (const position of ['initial', 'final']) { + const report = await options.read(); reports.push(report); + observations.push({ round, kind: position, at: new Date().toISOString(), report }); await options.persist(observations); + assert(report.ownerId === options.ownerId && report.draining && report.quiescent && Date.parse(report.expiresAt) > Date.now(), 'Early diagnostic lost exact live hold'); + const classification = classifyEarlyDiagnostic(report, performance.now() - options.apiSucceededAt); + transient = classification === 'transient' || transient; + } + const after = await server.snapshot(); + observations.push({ round, kind: 'snapshot-after', at: new Date().toISOString(), snapshot: after }); await options.persist(observations); + if (transient) continue; + assert(before.idle.active === 0 && after.idle.active === 0, 'Accepted early diagnostic has active owned work'); + // Each real HTTP authentication is tracked work. Its generation changes are + // observer stimuli; each report separately proves its own scan stayed stable. + assert.deepEqual(reports[0]!.idleSleepSafety, reports[1]!.idleSleepSafety, 'Accepted early diagnostic pair differs'); + assert(after.managed?.apiActivityActive, 'Real managed API activity expired during setup'); + return { rounds: round, initial: reports[0]!, final: reports[1]!, observations, + boundary: 'Strict complete stable setup inventory with real API cooldown active; not sleep eligibility' }; + } + throw new Error('Early inspection did not stabilize in three bounded owner-settled rounds'); +} diff --git a/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts b/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts new file mode 100644 index 00000000000..f7bda4c2369 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts @@ -0,0 +1,18 @@ +type Check = { reason: string; state: string }; +/** Early diagnostics are only a fail-fast blocker inventory, never sleep proof. + * The registered-probe aggregate includes the real 10-minute API activity hold; + * it cannot distinguish other probes, so the later full cooldown/proof is mandatory. */ +export function assertEarlySleepDiagnostics(report: { idleSleepSafety?: { diagnostics?: { complete?: boolean; checks?: Check[] } } }, elapsedSinceApiMs: number) { + const diagnostics = report.idleSleepSafety?.diagnostics; + if (!Number.isFinite(elapsedSinceApiMs) || elapsedSinceApiMs < 0 || elapsedSinceApiMs >= 600_000) throw new Error('Early diagnostic requires the actual API cooldown still active'); + if (diagnostics?.complete !== true || !Array.isArray(diagnostics.checks)) throw new Error('Early diagnostic coverage incomplete'); + const expected = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]); + const seen = new Set(); + for (const check of diagnostics.checks) { + if (seen.has(check.reason) || check.state !== (expected.get(check.reason) ?? 'none')) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`); + seen.add(check.reason); + } + for (const required of [...expected.keys(), 'inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog']) { + if (!seen.has(required)) throw new Error(`Missing early sleep diagnostic: ${required}`); + } +} diff --git a/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts b/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts new file mode 100644 index 00000000000..8431d6dbbc4 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts @@ -0,0 +1,47 @@ +import { eq } from 'drizzle-orm'; +import { randomUUID } from 'node:crypto'; +import { toolApplications, toolConnections, connectionGrants, type Db } from '@paperclipai/db'; +import { writeConnectionCredential } from '../../../services/connection-credentials.js'; +import { secretService } from '../../../services/secrets.js'; +import { createOAuthCredentialStore } from '../../../services/tool-access/oauth/credential-store.js'; +import { createOAuthRefreshLeases } from '../../../services/tool-access/oauth/refresh-leases.js'; +import { createOAuthGrantRefresh } from '../../../services/tool-access/oauth/grant-refresh.js'; +import { createGitHubGrantContinuity } from '../../../services/tool-access/github-grant-continuity.js'; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from '../../../services/paperclip-cloud-connector.js'; + +/** Bootstrap the named OAuth consent boundary, then use real vault, binding and continuity owners. */ +export async function prepareGitHubConfiguration(db: Db, companyId: string) { + const owner = 'local-board', now = Date.now(); + const [application] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: 'GitHub qualification', type: 'mcp_http', status: 'active' }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: application.id, uid: randomUUID(), name: 'Retained GitHub', status: 'active', enabled: false, + transport: 'mcp_remote', authKind: 'oauth', credentialPolicy: 'per_user', credentialSource: 'paperclip_vault', transportConfig: {}, + config: { url: 'https://api.github.com/mcp', sourceTemplateKey: 'github', oauth: { strategy: 'paperclip_cloud_connector', connectorProfile: 'github.code' } } }).returning(); + const refs = []; + for (const [configPath, value] of [['oauth.access_token', 'qualification-access'], ['oauth.refresh_token', 'qualification-refresh']] as const) { + const saved = await writeConnectionCredential(db, { companyId, connectionName: 'GitHub', configPath, label: configPath, value, ownerUserId: owner }); + refs.push({ configPath, secretId: saved.secret.id, versionSelector: 'latest' as const, required: true }); + } + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection.id, kind: 'user', subjectUserId: owner, status: 'active', credentialSecretRefs: refs, + providerTenant: { oauth: { strategy: 'paperclip_cloud_connector', accessTokenExpiresAt: new Date(now + 8 * 3_600_000).toISOString(), refreshedAt: new Date(now).toISOString() } } }).returning(); + await createOAuthCredentialStore(db, secretService(db)).syncCredentialBindings(connection, refs); + return { connectionId: connection.id, grantId: grant.id }; +} + +export async function settleGitHubConfiguration(db: Db, input: { connectionId: string; grantId: string }) { + const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, input.connectionId)))[0]; + const connection = await getConnection(); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, input.grantId)); + const config = paperclipCloudConnectorConfigFromEnv(); + if (!config) throw new Error('Owned qualification connector is not configured'); + const connector = createPaperclipCloudConnector({ config }); + const credentials = createOAuthCredentialStore(db, secretService(db)); + const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, credentials.getOAuthGrantRow); + const refresh = createOAuthGrantRefresh(db, { now: () => new Date(), store: credentials, leases, getConnectionRow: getConnection, + currentCloudConnector: () => connector, exchangeOAuthToken: async () => { throw new Error('Unexpected direct OAuth outside managed GitHub fixture'); } }); + const continuity = createGitHubGrantContinuity(db, { now: () => new Date(), currentCloudConnector: () => connector, + refreshOAuthGrantCredentials: refresh, resolveOAuthGrantSecret: credentials.resolveOAuthGrantSecret }); + const settled = await continuity.refreshManagedGitHubGrantAccess(connection, grant, { actorType: 'system', actorId: 'qualification' }); + if (settled.providerTenant?.github?.bindingSync || settled.providerTenant?.github?.continuityRetry) throw new Error('Real GitHub binding debt did not settle'); + return { connectionId: connection.id, grantId: grant.id, credentialRefs: settled.credentialSecretRefs.map(ref => ref.configPath), + boundary: 'Actual encrypted user credential/version/binding owners and GitHub authority refresh; real signed broker binding; named external GitHub HTTP only' }; +} diff --git a/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts b/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts new file mode 100644 index 00000000000..99218823a88 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict'; +import { installMaintenanceIdleWorkCapture } from './idle-work-capture.js'; + +// Isolated Node/TS-loader identity probe. Never imports app/index or opens a DB. +const original = await import('../../../services/task-admission.js'); +const before = process.argv[2] === 'unhandled' ? undefined : original.beginIdleTrackedWork(); +const capture = await installMaintenanceIdleWorkCapture(); +try { + const observed = await import('../../../services/task-admission.js'); + assert.equal(observed.idleWorkSnapshot, original.idleWorkSnapshot); + if (process.argv[2] === 'unhandled') { + const reason = new Error('original unhandled tracked rejection'); + let listener!: (reason: unknown, promise: Promise) => void; + let timer: ReturnType | undefined; + const event = new Promise<{ reason: unknown; promise: Promise }>((resolve, reject) => { + listener = (reason, promise) => resolve({ reason, promise }); + process.once('unhandledRejection', listener); + timer = setTimeout(() => reject(new Error('Original tracked rejection did not reach unhandledRejection')), 1500); + }); + try { + // Deliberately never await or attach a handler to this original promise. + const tracked = observed.trackIdleWork(Promise.reject(reason)); + const unhandled = await event; + assert.equal(unhandled.promise, tracked); + assert.equal(unhandled.reason, reason); + assert.equal(original.idleWorkSnapshot().active, 0); + assert.equal(capture.snapshot().observedActive, 0); + console.log(JSON.stringify({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 })); + } finally { + if (timer) clearTimeout(timer); + process.removeListener('unhandledRejection', listener); + } + } else { + assert.equal(observed.idleWorkSnapshot().active, 1); // Existing state was not reset. + assert.equal(capture.snapshot().observedActive, 0); // The pre-install path is explicitly missing. + before!(); + const finish = observed.beginIdleTrackedWork(); + assert.equal(original.idleWorkSnapshot().active, 1); + finish(); finish(); + assert.equal(original.idleWorkSnapshot().active, 0); + assert.equal(await observed.trackIdleWork(Promise.resolve('original-result')), 'original-result'); + assert.equal(original.idleWorkSnapshot().active, 0); + const result = capture.summary(); + assert(result.snapshot.missingPaths > 0); + console.log(JSON.stringify({ identity: true, originalStatePreserved: true, finalActive: original.idleWorkSnapshot().active, missingPathsExplicit: result.snapshot.missingPaths })); + } +} finally { + before?.(); + capture.dispose(); +} diff --git a/server/src/__tests__/helpers/maintenance/idle-work-capture.ts b/server/src/__tests__/helpers/maintenance/idle-work-capture.ts new file mode 100644 index 00000000000..732d1da4f27 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/idle-work-capture.ts @@ -0,0 +1,137 @@ +import { mock } from 'node:test'; +import { promiseHooks } from 'node:v8'; + +type Admission = typeof import('../../../services/task-admission.js'); +type Entry = { id: number; kind: 'begin' | 'track'; sourceId: number; startedAt: number; generation: number }; +type Completion = Entry & { finishedAt: number; finishedGeneration: number }; +export type IdleWorkCaptureSnapshot = { + actual: { active: number; generation: number }; observedActive: number; active: Entry[]; completionTail: Completion[]; + newSources: Array<{ id: number; callers: string[] }>; overflow: number; sourceOverflow: number; + missingCallsites: number; missingPaths: number; observerErrors: number; disposed: boolean; +}; + +/** Test-only metadata. No original work, promise, drain or counter is changed. */ +export function createMaintenanceIdleWorkCapture(actual: Pick, + controls: { now?: () => number; stack?: () => string | undefined } = {}) { + const active = new Map(), sources = new Map(); + const tail: Completion[] = []; + let trackedPromises = new WeakMap, number>(); + let disposed = false; + let serial = 0, emittedSourceId = 0, overflow = 0, sourceOverflow = 0, missingCallsites = 0, missingPaths = 0, observerErrors = 0; + const now = controls.now ?? Date.now; + function start(kind: Entry['kind']): number | undefined { + try { + if (disposed) return; + if (active.size >= 512) { overflow++; return; } + const callers: string[] = []; + for (const line of (controls.stack?.() ?? new Error().stack ?? '').split('\n')) { + const match = line.match(/[/\\]server[/\\](?:src|dist)[/\\]((?:(?:services|middleware|routes|modules)[/\\][A-Za-z0-9_./\\-]+|(?:app|index))\.[cm]?[jt]s):(\d+):(\d+)/); + if (!match || match[1]!.split(/[/\\]/).includes('..')) continue; + const caller = `${match[1]!.replaceAll('\\', '/')}:${match[2]}:${match[3]}`; + if (!callers.includes(caller)) callers.push(caller); + if (callers.length === 4) break; + } + if (!callers.length) missingCallsites++; + const key = JSON.stringify(callers); let source = sources.get(key); + if (!source) { + if (sources.size >= 512) { sourceOverflow++; return; } + source = { id: sources.size + 1, callers }; sources.set(key, source); + } + const id = ++serial; + active.set(id, { id, kind, sourceId: source.id, startedAt: now(), generation: actual.idleWorkSnapshot().generation }); + return id; + } catch { observerErrors++; return; } + } + // V8 observes settlement without attaching a continuation or marking a rejected + // original promise handled. Unrelated promises only perform a WeakMap lookup. + const stopSettlementHook = promiseHooks.onSettled(promise => { + try { + const id = trackedPromises.get(promise); + if (id === undefined) return; + trackedPromises.delete(promise); + finish(id); + } catch { observerErrors++; } + }); + function finish(id: number | undefined) { + try { + if (id === undefined) return; + const entry = active.get(id); if (!entry) return; + active.delete(id); + tail.push({ ...entry, finishedAt: now(), finishedGeneration: actual.idleWorkSnapshot().generation }); + if (tail.length > 64) tail.shift(); + } catch { observerErrors++; } + } + const beginIdleTrackedWork: Admission['beginIdleTrackedWork'] = () => { + const originalFinish = actual.beginIdleTrackedWork(); const id = start('begin'); + return () => { try { return originalFinish(); } finally { finish(id); } }; + }; + const trackIdleWork: Admission['trackIdleWork'] = (work: Promise) => { + const original = actual.trackIdleWork(work); const id = start('track'); + try { if (id !== undefined) trackedPromises.set(original, id); } catch { observerErrors++; } + return original; + }; + return { beginIdleTrackedWork, trackIdleWork, + dispose() { + if (disposed) return; + disposed = true; + try { stopSettlementHook(); } finally { trackedPromises = new WeakMap(); } + }, + snapshot(): IdleWorkCaptureSnapshot { + const state = actual.idleWorkSnapshot(); + if (state.active !== active.size) missingPaths++; + const newSources = [...sources.values()].filter(source => source.id > emittedSourceId).map(source => ({ ...source, callers: [...source.callers] })); + emittedSourceId = sources.size; + return { actual: state, observedActive: active.size, active: [...active.values()].map(entry => ({ ...entry })), completionTail: tail.map(entry => ({ ...entry })), + newSources, overflow, sourceOverflow, missingCallsites, missingPaths, observerErrors, disposed }; + }, + summary() { return { sourceCatalog: [...sources.values()], snapshot: this.snapshot() }; }, + }; +} + +export async function installMaintenanceIdleWorkCapture() { + const actual = await import('../../../services/task-admission.js'); + const capture = createMaintenanceIdleWorkCapture(actual); + let moduleCapture: ReturnType | undefined; + const disposeCapture = capture.dispose; + capture.dispose = () => { try { disposeCapture(); } finally { const installed = moduleCapture; moduleCapture = undefined; installed?.restore(); } }; + try { + moduleCapture = mock.module(new URL('../../../services/task-admission.ts', import.meta.url).href, { + namedExports: { ...actual, beginIdleTrackedWork: capture.beginIdleTrackedWork, trackIdleWork: capture.trackIdleWork }, + }); + const observed = await import('../../../services/task-admission.js'); + if (observed.beginIdleTrackedWork !== capture.beginIdleTrackedWork || observed.trackIdleWork !== capture.trackIdleWork + || Object.keys(observed).sort().join(',') !== Object.keys(actual).sort().join(',') + || Object.keys(actual).some(key => !['beginIdleTrackedWork', 'trackIdleWork'].includes(key) + && observed[key as keyof Admission] !== actual[key as keyof Admission])) throw new Error('Idle work module capture did not preserve original exports/identity'); + return capture; + } catch (error) { + try { capture.dispose(); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Idle work capture installation and cleanup failed'); } + throw error; + } +} + +/** Reuse existing settle-loop observations. No extra poll, timer, SQL or raw stack per sample. */ +export function createSettledWorkSamples(sourceCatalog: () => Array<{ id: number; callers: string[] }>) { + // Finite longest setup observation at the existing 25ms cadence, plus its + // terminal sample. Overflow still makes diagnostic completion fail closed. + const sampleLimit = Math.ceil(61_000 / 25) + 1; + const samples: Array<{ id: number; at: number; queryTotal: number; queryDelta: number; pending: number; + idle: { active: number; generation: number }; activeIds: number[] }> = []; + let previous: number | undefined, overflowSamples = 0, lastCapture: IdleWorkCaptureSnapshot | undefined; + const startedAt = Date.now(); + return { + sample(snapshot: { now: number; idle: { active: number; generation: number }; workCapture?: IdleWorkCaptureSnapshot }, queryTotal: number, pending: number) { + lastCapture = snapshot.workCapture; + if (samples.length < sampleLimit) samples.push({ id: samples.length + 1, at: snapshot.now, queryTotal, + queryDelta: previous === undefined ? 0 : queryTotal - previous, pending, idle: { ...snapshot.idle }, activeIds: lastCapture?.active.map(entry => entry.id) ?? [] }); + else overflowSamples++; + previous = queryTotal; + }, + report() { return { startedAt, finishedAt: Date.now(), settleDeadlineMs: 30_000, sampleLimit, overflowSamples, + samples, sourceCatalog: sourceCatalog(), lastCapture }; }, + assertComplete() { + if (!lastCapture || overflowSamples || lastCapture.overflow || lastCapture.sourceOverflow || lastCapture.missingCallsites + || lastCapture.missingPaths || lastCapture.observerErrors || lastCapture.disposed) throw new Error('Passive idle work diagnostic capture incomplete'); + }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts b/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts new file mode 100644 index 00000000000..e317165325f --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts @@ -0,0 +1,40 @@ +import { randomUUID } from "node:crypto"; +import type { Server } from "node:http"; +import type { IncomingHttpSnapshot } from "../../../../../scripts/sleep-qualification/evidence.mjs"; + +/** Passive test-child observation of the actual app listener, including direct clients. */ +export function observeMaintenanceIncomingHttp(server: Server) { + // A sole upgrade listener suppresses Node's default unsupported-upgrade close. + // The real app installs its WebSocket handlers before startServer returns. + if (server.listenerCount("request") === 0 || server.listenerCount("upgrade") === 0) { + throw new Error("Maintenance HTTP observation requires existing app request and upgrade handlers"); + } + const captureId = randomUUID(); + let requests = 0, installed = true, valid = true; + const count = () => { + if (requests === Number.MAX_SAFE_INTEGER) valid = false; + else requests++; + }; + const detached = (event: string | symbol, listener: unknown) => { + if ((event === "request" || event === "upgrade") && listener === count) valid = false; + }; + // Count before application handlers without inspecting or retaining request data. + server.prependListener("request", count); + server.prependListener("upgrade", count); + server.on("removeListener", detached); + return { + snapshot(): IncomingHttpSnapshot { + return { source: "node-http-server-events", captureId, installed, + valid: valid && installed && server.listening + && server.listeners("request").includes(count) && server.listeners("upgrade").includes(count), + requests, observedAt: new Date().toISOString() }; + }, + dispose() { + if (!installed) return; + installed = false; valid = false; + server.off("removeListener", detached); + server.off("request", count); + server.off("upgrade", count); + }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/managed-profile.ts b/server/src/__tests__/helpers/maintenance/managed-profile.ts new file mode 100644 index 00000000000..ca8ff553f84 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/managed-profile.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { authUsers, companyMemberships, type Db } from '@paperclipai/db'; +import { and, eq } from 'drizzle-orm'; +import { cloudTenantPrimaryCompanyId } from '../../../services/cloud-instance.js'; +import type { MaintenanceConfiguration, MaintenanceServerOptions, startMaintenanceServer } from './server-harness.js'; + +export const managedTenantActor = { userId: 'local-board', email: 'sleep-owner@example.test', name: 'Owned sleep operator' }; +export function managedSleepConfiguration(collectorOrigin: string, serveUi: boolean): MaintenanceConfiguration { + return { automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, + telemetryOrigin: `${collectorOrigin}/telemetry`, otelOrigin: collectorOrigin, serveUi, + managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' }; +} +export function managedSleepServerOptions(configuration: MaintenanceConfiguration, jwks: string, s3Origin: string, + broker: { origin: string; variables: Record }): MaintenanceServerOptions { + return { configuration, controlJwks: jwks, cloud: { jwks, s3Origin, standby: false }, tenantActor: managedTenantActor, + githubOrigin: broker.origin, connector: { origin: broker.origin, + instanceId: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID, + signPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY, + sealPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY } }; +} +export async function provisionManagedSleepCompany(server: Awaited>, db: Db) { + // This real request invokes resolveCloudTenantActor, never a table seed or a bypassed company-create route. + const companies = await server.api>('GET', '/companies'); + const id = cloudTenantPrimaryCompanyId('maintenance-stack'); + assert(companies.some(company => company.id === id), 'Actual managed primary company was not provisioned'); + const [authority] = await db.select({ userId: authUsers.id, status: companyMemberships.status, membershipRole: companyMemberships.membershipRole }) + .from(authUsers).innerJoin(companyMemberships, and(eq(companyMemberships.principalId, authUsers.id), eq(companyMemberships.principalType, 'user'))) + .where(and(eq(authUsers.id, managedTenantActor.userId), eq(companyMemberships.companyId, id))); + assert.deepEqual(authority, { userId: managedTenantActor.userId, status: 'active', membershipRole: 'owner' }); + await server.api('PATCH', `/companies/${id}`, { defaultResponsibleUserId: managedTenantActor.userId }); + const observation = (await server.snapshot()).managed; + assert(observation?.isCloudManaged && observation.deploymentMode === 'authenticated' && observation.storageProvider === 's3', 'Actual supported managed profile not active'); + return { id, authority, observation, boundary: 'Fixture Cloud front-door headers; actual resolver creates canonical company/user/member/default grants' }; +} diff --git a/server/src/__tests__/helpers/maintenance/native-browser.ts b/server/src/__tests__/helpers/maintenance/native-browser.ts new file mode 100644 index 00000000000..189e7993509 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/native-browser.ts @@ -0,0 +1,65 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { Browser, BrowserContext } from '@playwright/test'; + +/** A fresh owned browser, not a user's profile. Never terminate by process name. */ +export async function withNativeMaintenanceBrowser(run: (browser: Browser, context: BrowserContext) => Promise): Promise { + const { chromium } = await import('@playwright/test'); + const profile = await mkdtemp(path.join(tmpdir(), 'paperclip-sleep-browser-')); + let child: ChildProcess | undefined, browser: Browser | undefined, exited: Promise | undefined; + let exitObserved = false, launchError: Error | undefined, stderr = '', failure: { error: unknown } | undefined; + try { + child = spawn(chromium.executablePath(), [`--user-data-dir=${profile}`, '--remote-debugging-address=127.0.0.1', '--remote-debugging-port=0', + '--no-first-run', '--no-default-browser-check', '--disable-background-networking', '--disable-component-update', '--disable-sync', 'about:blank'], { stdio: ['ignore', 'ignore', 'pipe'] }); + const ownedChild = child; + exited = new Promise(resolve => { + ownedChild.once('exit', () => { exitObserved = true; resolve(); }); + ownedChild.once('error', error => { launchError = error; if (!ownedChild.pid) resolve(); }); + }); + child.stderr?.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-16_384); }); + const deadline = performance.now() + 10_000; + let port: number | undefined; + while (!port) { + if (launchError) throw launchError; + if (child.exitCode !== null || child.signalCode !== null || performance.now() > deadline) throw new Error(`Owned Chromium startup failed: ${stderr}`); + try { + const candidate = Number((await readFile(path.join(profile, 'DevToolsActivePort'), 'utf8')).split('\n')[0]); + if (Number.isInteger(candidate) && candidate > 0 && candidate < 65536) port = candidate; + } catch { /* The owned startup file is not ready yet. */ } + if (!port) await delay(50); + } + // Playwright1.62 installs focus emulation on normal pages. Measured native + // tab switches stayed visible there. The supported noDefaults/default-context + // path avoids installing that override (types.d.ts ConnectOverCDPOptions). + browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`, { noDefaults: true, timeout: 10_000 }); + const context = browser.contexts()[0]; + if (!context) throw new Error('Owned Chromium has no native default context'); + return await run(browser, context); + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + let disconnectTimer: ReturnType | undefined; + try { + await Promise.race([browser?.close(), new Promise((_, reject) => { disconnectTimer = setTimeout(() => reject(new Error('Owned browser disconnect timed out')), 5_000); })]); + } catch (error) { errors.push(error); } + finally { if (disconnectTimer) clearTimeout(disconnectTimer); } + if (child && exited) { + const waitExit = async () => { + let timer: ReturnType | undefined; + try { return await Promise.race([exited!.then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), 5_000); })]); } + finally { if (timer) clearTimeout(timer); } + }; + try { + if (child.exitCode === null && child.signalCode === null && !launchError) child.kill('SIGTERM'); + if (!(await waitExit())) { child.kill('SIGKILL'); if (!(await waitExit())) throw new Error('Owned Chromium did not exit after termination'); } + } catch (error) { errors.push(error); } + } + if (!child?.pid || exitObserved) { + try { await rm(profile, { recursive: true, force: true }); } catch (error) { errors.push(error); } + } else errors.push(new Error(`Owned Chromium exit was not confirmed; profile preserved at ${profile}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Native browser work or cleanup failed'); + } +} diff --git a/server/src/__tests__/helpers/maintenance/otel-child.ts b/server/src/__tests__/helpers/maintenance/otel-child.ts new file mode 100644 index 00000000000..507fa5e529a --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/otel-child.ts @@ -0,0 +1,10 @@ +import { getStartupTracer, instrumentationReady, shutdownInstrumentation } from '../../../instrumentation.js'; +await instrumentationReady; +const span = getStartupTracer('paperclip.sleep-qualification').startSpan('sleep-qualification-stimulus'); +span.end(); +// Keep the real SDK alive beyond its 60s default metric-reader interval. This +// observes unwanted exporter traffic without disabling any exporter through env. +if (process.argv.includes('--observe-exporters')) { + await new Promise(resolve => setTimeout(resolve, 90_000)); +} +await shutdownInstrumentation(); diff --git a/server/src/__tests__/helpers/maintenance/s3-boundary.ts b/server/src/__tests__/helpers/maintenance/s3-boundary.ts new file mode 100644 index 00000000000..dc857f5d187 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/s3-boundary.ts @@ -0,0 +1,79 @@ +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; + +/** Adapted from the real-SDK loopback fixture in managed-cloud-prerequisites.test.ts. + * Small owned objects only; no provider credentials/IAM/durability qualification. */ +export function createMaintenanceS3Boundary() { + const objects = new Map(); + const requests: Array<{ method: string; path: string; status: number }> = []; + const errors: string[] = []; + const xml = (text: string) => text.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>'); + const server = createServer(async (req, res) => { + const url = new URL(req.url!, 'http://fixture.invalid'); + const operation = { method: req.method!, path: url.pathname + url.search, status: 0 }; requests.push(operation); + const reply = (status: number, body?: string | Buffer) => { operation.status = status; res.statusCode = status; res.end(body); }; + try { + if (!url.pathname.startsWith('/maintenance-test/') && url.pathname !== '/maintenance-test') throw new Error('Unknown S3 fixture bucket'); + const key = decodeURIComponent(url.pathname.slice('/maintenance-test/'.length)); + if (req.method === 'GET' && ['/maintenance-test', '/maintenance-test/'].includes(url.pathname) && url.searchParams.get('list-type') === '2' + && [...url.searchParams.keys()].every(key => ['list-type', 'prefix', 'max-keys', 'x-id'].includes(key))) { + const keys = [...objects.keys()].filter(key => key.startsWith(url.searchParams.get('prefix') ?? '')).sort(); + if (keys.length > Math.min(1000, Number(url.searchParams.get('max-keys') ?? 1000))) throw new Error('Listing exceeds bounded single-page fixture'); + res.setHeader('content-type', 'application/xml'); + reply(200, `false${keys.map(key => `${xml(key)}${objects.get(key)!.bytes.length}`).join('')}`); return; + } + if (!key || [...url.searchParams.keys()].some(key => key !== 'x-id')) throw new Error('Unknown S3 fixture operation/query'); + if (req.method === 'PUT') { + const chunks: Buffer[] = []; let length = 0; + for await (const chunk of req) { length += chunk.length; if (length > 17 * 1024 * 1024) throw new Error('S3 fixture object too large'); chunks.push(Buffer.from(chunk)); } + let bytes = Buffer.concat(chunks); + if (req.headers['content-encoding'] === 'aws-chunked') bytes = decodeMaintenanceAwsChunks(bytes); + else if (req.headers['content-encoding']) throw new Error('Unknown S3 fixture content framing'); + const decodedLength = req.headers['x-amz-decoded-content-length']; + if (decodedLength !== undefined && Number(decodedLength) !== bytes.length) throw new Error('S3 decoded length mismatch'); + const previous = objects.get(key); + if ((req.headers['if-none-match'] !== undefined && req.headers['if-none-match'] !== '*') + || (req.headers['if-match'] !== undefined && typeof req.headers['if-match'] !== 'string')) throw new Error('Unknown conditional S3 request'); + if ((req.headers['if-none-match'] === '*' && previous) || (req.headers['if-match'] && previous?.etag !== req.headers['if-match'])) { + res.setHeader('content-type', 'application/xml'); reply(412, 'PreconditionFailed'); return; + } + const total = [...objects.values()].reduce((sum, object) => sum + object.bytes.length, 0) - (previous?.bytes.length ?? 0) + bytes.length; + if (bytes.length > 16 * 1024 * 1024 || total > 256 * 1024 * 1024 || objects.size >= 2048) throw new Error('S3 fixture storage bound exceeded'); + const object = { bytes, etag: `"${createHash('md5').update(bytes).digest('hex')}"`, contentType: String(req.headers['content-type'] ?? 'application/octet-stream') }; + objects.set(key, object); res.setHeader('ETag', object.etag); reply(200); return; + } + if (req.method === 'DELETE') { objects.delete(key); reply(204); return; } + if (req.method !== 'GET' && req.method !== 'HEAD') throw new Error('Unknown S3 fixture method'); + const object = objects.get(key); + if (!object) { res.setHeader('content-type', 'application/xml'); reply(404, req.method === 'GET' ? 'NoSuchKey' : undefined); return; } + if (req.headers.range) throw new Error('Range is outside this small-object fixture contract'); + res.setHeader('ETag', object.etag); res.setHeader('content-type', object.contentType); res.setHeader('content-length', object.bytes.length); + reply(200, req.method === 'GET' ? object.bytes : undefined); + } catch (error) { + errors.push(`${req.method} ${url.pathname + url.search}: ${error instanceof Error ? error.message : String(error)}`); + if (!res.headersSent) { res.setHeader('content-type', 'application/xml'); reply(501, 'NotImplemented'); } else res.destroy(); + } + }); + return { server, snapshot: () => ({ boundary: 'Actual AWS SDK HTTP; bounded in-memory S3 protocol fixture, no IAM/SigV4/provider durability proof', requests: requests.map(request => ({ ...request })), errors: [...errors], + objects: [...objects].map(([key, object]) => ({ key, bytes: object.bytes.length, sha256: createHash('sha256').update(object.bytes).digest('hex') })) }) }; +} + +/** Decode SDK checksum framing without pretending to validate provider signatures. */ +export function decodeMaintenanceAwsChunks(wire: Buffer) { + const chunks: Buffer[] = []; let offset = 0; + while (offset < wire.length) { + const end = wire.indexOf('\r\n', offset); if (end < 0) throw new Error('Malformed AWS chunk header'); + const header = wire.subarray(offset, end).toString('ascii'); + if (!/^[0-9a-f]+$/i.test(header)) throw new Error('Unknown AWS chunk extension'); + const length = Number.parseInt(header, 16); offset = end + 2; + if (!Number.isSafeInteger(length) || length < 0 || offset + length + 2 > wire.length) throw new Error('Truncated AWS chunk'); + if (length === 0) { + const trailers = wire.subarray(offset).toString('ascii'); + if (!/^(?:x-amz-checksum-[a-z0-9]+:[A-Za-z0-9+/=]+\r\n)*\r\n$/.test(trailers)) throw new Error('Unknown AWS checksum trailer'); + return Buffer.concat(chunks); + } + chunks.push(wire.subarray(offset, offset + length)); offset += length; + if (wire.subarray(offset, offset + 2).toString('ascii') !== '\r\n') throw new Error('Malformed AWS chunk terminator'); offset += 2; + } + throw new Error('Missing AWS final chunk'); +} diff --git a/server/src/__tests__/helpers/maintenance/server-child.ts b/server/src/__tests__/helpers/maintenance/server-child.ts index 5569b29d2e9..833726d930e 100644 --- a/server/src/__tests__/helpers/maintenance/server-child.ts +++ b/server/src/__tests__/helpers/maintenance/server-child.ts @@ -2,6 +2,7 @@ import { channel } from "node:diagnostics_channel"; import { mock } from "node:test"; import type { Db } from "@paperclipai/db"; import type { Socket } from "node:net"; +import { observeMaintenanceIncomingHttp } from "./incoming-http-observation.js"; const realTimeout = globalThis.setTimeout, realInterval = globalThis.setInterval, clearRealInterval = globalThis.clearInterval, realNow = Date.now; const fakeClock = process.env.PAPERCLIP_MAINTENANCE_FAKE_CLOCK === "true"; @@ -40,14 +41,42 @@ const observeRequest = () => { outgoingHttpRequests++; }; channel("undici:request:create").subscribe(observeRequest); channel("http.client.request.start").subscribe(observeRequest); +let workCapture: Awaited> | undefined; +let databaseCapture: ReturnType; +let incomingHttp: ReturnType | undefined; try { + // Must precede the announcement wrapper's first import of the real app graph. + if (process.env.PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE === '1') { + workCapture = await (await import('./idle-work-capture.js')).installMaintenanceIdleWorkCapture(); + } + const announcement = process.env.PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE === '1' + ? await (await import('./announcement-boundary.js')).installMaintenanceAnnouncementBoundary() : undefined; const { startServer } = await import("../../../index.js"); const { idleWorkSnapshot, startTaskDrain, stopTaskDrain } = await import("../../../services/task-admission.js"); const { getStartupRecoveryState } = await import("../../../startup-recovery-state.js"); const server = await startServer(); + if (workCapture) databaseCapture = (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv(); + const managedObservation = process.env.PAPERCLIP_MAINTENANCE_MANAGED_TENANT === '1' ? await (async () => { + const { loadConfig } = await import('../../../config.js'); + const { isCloudManagedInstance } = await import('../../../services/cloud-instance.js'); + const { idleApiActivity } = await import('../../../services/idle-api-activity.js'); + const config = loadConfig(); + return () => ({ isCloudManaged: isCloudManagedInstance(), deploymentMode: config.deploymentMode, + storageProvider: config.storageProvider, apiActivityActive: idleApiActivity.active() }); + })() : undefined; const app = server.server.listeners("request")[0] as unknown as { locals: { paperclipDb: Db } }; + try { incomingHttp = observeMaintenanceIncomingHttp(server.server); } + catch (error) { + try { await server.shutdown(); } + catch (cleanupError) { throw new AggregateError([error, cleanupError], "Incoming HTTP observation and owned startup cleanup failed"); } + throw error; + } + async function shutdownServer() { + try { await server.shutdown(); } finally { incomingHttp?.dispose(); } + } + try { announcement?.assertIntercepted(); } catch (error) { await shutdownServer(); throw error; } process.send?.({ kind: "ready", port: server.listenPort }); - process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } }) => { + process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } }) => { try { if (message.command === "snapshot" || message.command === "advance" || message.command === "drain" || message.command === "resume" || message.command === "freeze" || message.command === "realtime") { if (message.command === "freeze" && bootClock) { clearRealInterval(bootClock); bootClock = realInterval(() => mock.timers.tick(0), 10); } @@ -59,7 +88,23 @@ try { if (message.command === "drain") startTaskDrain({ purpose: "idle" }); if (message.command === "resume") stopTaskDrain(); if (fakeClock) mock.timers.tick(0); - process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), now: Date.now() }); + process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), incomingHttp: incomingHttp?.snapshot(), now: Date.now(), announcement: announcement?.snapshot(), managed: managedObservation?.(), workCapture: workCapture?.snapshot() }); + } else if (message.command === "prepareGitHub" || message.command === "settleGitHub") { + const { prepareGitHubConfiguration, settleGitHubConfiguration } = await import("./github-configuration-producer.js"); + const input = message.github; + if (!input) throw new Error("Missing GitHub fixture identity"); + const result = message.command === "prepareGitHub" + ? await prepareGitHubConfiguration(app.locals.paperclipDb, input.companyId!) + : await settleGitHubConfiguration(app.locals.paperclipDb, { connectionId: input.connectionId!, grantId: input.grantId! }); + process.send?.({ id: message.id, kind: "githubConfiguration", result }); + } else if (message.command === "produceChatHistory") { + const { produceChatHistory } = await import("./chat-history-producer.js"); + process.send?.({ id: message.id, kind: "chatHistory", result: await produceChatHistory(app.locals.paperclipDb) }); + } else if (message.command === "trace") { + const { getStartupTracer } = await import("../../../instrumentation.js"); + const span = getStartupTracer("paperclip.sleep-qualification").startSpan("sleep-qualification-stimulus"); + span.end(); + process.send?.({ id: message.id, kind: "traceEnded" }); } else if (message.command === "createOrphanLease") { if (!message.orphanLease) throw new Error("Missing orphan lease fixture"); const { environmentLeases } = await import("@paperclipai/db"); @@ -83,7 +128,20 @@ try { process.send?.({ id: message.id, kind: "published" }); } else if (message.command === "shutdown") { if (fakeClock) runClock(); - await server.shutdown(); + if (workCapture) { + const failures: unknown[] = []; + let before: ReturnType | undefined; + try { before = workCapture.summary(); } catch (error) { failures.push(error); } + try { await shutdownServer(); } catch (error) { failures.unshift(error); } + // A passive flush failure must not replace the original shutdown error. + try { databaseCapture?.flush(); } catch (error) { failures.push(error); } + try { + await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-cleanup.json'), JSON.stringify({ before, after: workCapture.summary() }, null, 2)); + } catch (error) { failures.push(error); } + finally { try { workCapture.dispose(); } catch (error) { failures.push(error); } } + if (failures.length > 1) throw new AggregateError(failures, 'Owned shutdown and passive evidence failed'); + if (failures.length) throw failures[0]; + } else await shutdownServer(); if (bootClock) { clearRealInterval(bootClock); bootClock = undefined; } process.send?.({ id: message.id, kind: "stopped", outgoingHttpRequests }); if (fakeClock) mock.timers.reset(); @@ -93,6 +151,16 @@ try { } catch (error) { process.send?.({ id: message.id, kind: "error", error: error instanceof Error ? error.stack : String(error) }); } }); } catch (error) { + incomingHttp?.dispose(); + if (workCapture) { + try { + // Existing bounded DB capture performs no SQL while flushing metadata. + databaseCapture ??= (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv(); + databaseCapture?.flush(); + await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-startup-failure.json'), JSON.stringify(workCapture.summary(), null, 2)); + } catch { console.warn('Passive startup evidence could not be persisted; original error preserved'); } + finally { try { workCapture.dispose(); } catch { console.warn('Passive capture cleanup failed; original startup error preserved'); } } + } if (bootClock) clearRealInterval(bootClock); process.send?.({ kind: "error", error: error instanceof Error ? error.stack : String(error) }); process.exitCode = 1; diff --git a/server/src/__tests__/helpers/maintenance/server-harness.ts b/server/src/__tests__/helpers/maintenance/server-harness.ts index 922f90292a8..15e1c6ca04e 100644 --- a/server/src/__tests__/helpers/maintenance/server-harness.ts +++ b/server/src/__tests__/helpers/maintenance/server-harness.ts @@ -1,3 +1,5 @@ +import type { IdleWorkCaptureSnapshot } from './idle-work-capture.js'; +import type { IncomingHttpSnapshot } from '../../../../../scripts/sleep-qualification/evidence.mjs'; import { fork } from "node:child_process"; import type { Server as HttpServer } from "node:http"; import { randomBytes, randomUUID } from "node:crypto"; @@ -42,9 +44,54 @@ export async function withMaintenanceProvider(provider: HttpServer, run: (ori } } -type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number }; +type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number; + incomingHttp: IncomingHttpSnapshot; + managed?: { isCloudManaged: boolean; deploymentMode: string; storageProvider: string; apiActivityActive: boolean }; + workCapture?: IdleWorkCaptureSnapshot; + announcement?: { intercepted: boolean; requests: string[]; unmatched: string[]; boundary: string } }; -export async function startMaintenanceServer(databaseUrl: string, options: { artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string }; cloud?: { jwks: string; s3Origin: string; standby: boolean } } = {}) { +export interface MaintenanceConfiguration { + automaticBackups?: { intervalMinutes: number; idleCheckpoint: boolean }; + announcements?: boolean; + telemetryOrigin?: string; + otelOrigin?: string; + serveUi?: boolean; + managedInstance?: boolean; + deploymentMode?: "local_trusted" | "authenticated"; + storageProvider?: "local_disk" | "s3"; +} +export interface MaintenanceServerOptions { + artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; + connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string; instanceId?: string }; + cloud?: { jwks: string; s3Origin: string; standby: boolean }; + configuration?: MaintenanceConfiguration; + controlJwks?: string; + captureIdleWork?: boolean; + tenantActor?: { userId: string; email: string; name: string }; +} +export async function startMaintenanceServer(databaseUrl: string, options: MaintenanceServerOptions = {}) { + const configuration = options.configuration; + if (options.captureIdleWork && options.clock) throw new Error("Passive setup diagnostics require the real clock"); + if (options.tenantActor && (!options.cloud || !configuration?.managedInstance || configuration.deploymentMode !== 'authenticated' || configuration.storageProvider !== 's3')) { + throw new Error('Managed tenant fixture requires the supported authenticated S3 profile'); + } + const tenantHeaders: Record = options.tenantActor ? { + 'x-paperclip-cloud-tenant-token': 'maintenance-tenant-token', 'x-paperclip-cloud-stack-id': 'maintenance-stack', + 'x-paperclip-cloud-stack-role': 'owner', 'x-paperclip-cloud-user-id': options.tenantActor.userId, + 'x-paperclip-cloud-user-email': options.tenantActor.email, 'x-paperclip-cloud-user-name': options.tenantActor.name, + 'x-paperclip-cloud-paperclip-company-name': 'Configured sleep acceptance', + } : {}; + for (const origin of [configuration?.telemetryOrigin, configuration?.otelOrigin]) { + if (!origin) continue; + const endpoint = new URL(origin); + if (endpoint.protocol !== "http:" || endpoint.hostname !== "127.0.0.1" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) { + throw new Error("Maintenance collectors must be explicit loopback HTTP endpoints"); + } + } + if (configuration?.automaticBackups && (!Number.isInteger(configuration.automaticBackups.intervalMinutes) || configuration.automaticBackups.intervalMinutes < 1)) { + throw new Error("Maintenance backup interval must be a positive integer"); + } + if (configuration && options.clock) throw new Error("Configured acceptance requires the real clock"); const artifacts = await createMaintenanceArtifacts(options.artifactRoot); const { directory, retainedDirectory } = artifacts; const reservation = createServer(); @@ -72,43 +119,57 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art const configPath = path.join(directory, "config.json"); await writeFile(configPath, JSON.stringify({ $meta: { version: 1, updatedAt: new Date().toISOString(), source: "onboard" }, - database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: false, dir: path.join(directory, "backups") } }, + database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: Boolean(configuration?.automaticBackups), intervalMinutes: configuration?.automaticBackups?.intervalMinutes ?? 60, dir: path.join(directory, "backups") } }, logging: { mode: "file", logDir: path.join(directory, "logs") }, - server: { deploymentMode: "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: false }, + server: { deploymentMode: options.tenantActor ? "authenticated" : "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: configuration?.serveUi ?? false }, storage: options.cloud ? { provider: "s3", s3: { bucket: "maintenance-test", region: "us-east-1", endpoint: options.cloud.s3Origin, forcePathStyle: true } } : { provider: "local_disk", localDisk: { baseDir: path.join(directory, "storage") } }, secrets: { provider: "local_encrypted", strictMode: false, localEncrypted: { keyFilePath: path.join(directory, "master.key") } }, - telemetry: { enabled: false }, updates: { checkEnabled: false }, + ...(options.tenantActor ? { auth: { baseUrlMode: "explicit", publicBaseUrl: `http://127.0.0.1:${address.port}`, disableSignUp: true } } : {}), + telemetry: { enabled: Boolean(configuration?.telemetryOrigin) }, updates: { checkEnabled: false }, }), { mode: 0o600 }); await new Promise(resolve => reservation.close(() => resolve())); + const captureStartedAt = options.captureIdleWork ? Date.now() : undefined; + const captureSources = options.captureIdleWork ? new Map() : undefined; + let settledDiagnosticId = 0; const child = fork(fileURLToPath(new URL("./server-child.ts", import.meta.url)), [], { cwd: fileURLToPath(new URL("../../../../..", import.meta.url)), - execArgv: ["--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], + execArgv: [...((configuration?.announcements || options.captureIdleWork) ? ['--experimental-test-module-mocks'] : []), "--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], env: { + ...(captureStartedAt === undefined ? {} : { + PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE: '1', + PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW: `${new Date(captureStartedAt).toISOString()}/${new Date(captureStartedAt + 60_000).toISOString()}`, + }), PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: "production", PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test", PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"), - DATABASE_URL: proxy.url, DATABASE_POOL_MAX: "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", + DATABASE_URL: proxy.url, DATABASE_POOL_MAX: configuration ? "4" : "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), - CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false", - PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_UI_DEV_MIDDLEWARE: "false", + CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: String(configuration?.announcements ?? false), + ...(configuration?.announcements ? { PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE: '1' } : {}), + PAPERCLIP_DB_BACKUP_ENABLED: String(Boolean(configuration?.automaticBackups)), PAPERCLIP_UI_DEV_MIDDLEWARE: "false", + ...(configuration?.automaticBackups ? { PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED: configuration.automaticBackups.idleCheckpoint ? "1" : "0" } : {}), + ...(configuration?.telemetryOrigin ? { PAPERCLIP_TELEMETRY_ENDPOINT: configuration.telemetryOrigin } : {}), + ...(configuration?.otelOrigin ? { OTEL_EXPORTER_OTLP_ENDPOINT: configuration.otelOrigin, OTEL_EXPORTER_OTLP_PROTOCOL: "http/json" } : {}), HEARTBEAT_SCHEDULER_ENABLED: String(options.enabled ?? true), PAPERCLIP_MAINTENANCE_FAKE_CLOCK: String(options.clock ?? false), ...(options.suppressed ? { PAPERCLIP_IN_WORKTREE: "true" } : {}), ...(options.githubOrigin ? { PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN: options.githubOrigin } : {}), ...(options.connector ? { - PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", + PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: options.connector.instanceId ?? "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY: options.connector.signPrivateKey, PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY: options.connector.sealPrivateKey, PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: options.connector.origin, } : {}), + ...(options.controlJwks ? { PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.controlJwks } : {}), ...(options.cloud ? { PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } }), PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"), PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "maintenance-tenant-token", PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.cloud.jwks, PAPERCLIP_CLOUD_WARM_STANDBY: options.cloud.standby ? "1" : "0", - PAPERCLIP_PUBLIC_URL: "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: "pool-maintenance.staging.paperclip.app", + PAPERCLIP_PUBLIC_URL: options.tenantActor ? `http://127.0.0.1:${address.port}` : "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: options.tenantActor ? "127.0.0.1" : "pool-maintenance.staging.paperclip.app", + ...(options.tenantActor ? { PAPERCLIP_MAINTENANCE_MANAGED_TENANT: "1" } : {}), AWS_ACCESS_KEY_ID: "maintenance-fixture", AWS_SECRET_ACCESS_KEY: "maintenance-fixture", AWS_EC2_METADATA_DISABLED: "true", } : {}), PAPERCLIP_MIGRATION_PROMPT: "never", PAPERCLIP_MIGRATION_AUTO_APPLY: "true", PAPERCLIP_MANAGED_RUNTIME_HTTPS: "off", @@ -139,12 +200,18 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art for (const request of requests.values()) request.reject(error); requests.clear(); }); - async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } } = {}): Promise { + async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } } = {}): Promise { const id = ++requestId; const result = new Promise((resolve, reject) => requests.set(id, { resolve, reject })); child.send({ id, command, ...payload }); - try { return await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); } - finally { requests.delete(id); } + try { + const value = await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); + if (captureSources) { + const capture = (value as { workCapture?: IdleWorkCaptureSnapshot }).workCapture; + for (const source of capture?.newSources ?? []) captureSources.set(source.id, source.callers); + } + return value; + } finally { requests.delete(id); } } stopChild = async () => { try { @@ -165,7 +232,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art if (options.clock) await request("realtime"); try { const response = await fetch(`http://127.0.0.1:${port}/api${route}`, { method, signal: AbortSignal.timeout(30_000), - headers: { "Content-Type": "application/json", ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); + headers: { "Content-Type": "application/json", ...(Object.keys(headers ?? {}).some(key => ["authorization", "x-paperclip-cloud-control"].includes(key.toLowerCase())) ? {} : tenantHeaders), ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); const value = await response.json(); if (!response.ok) throw new Error(`Maintenance API ${method} ${route}: ${response.status} ${JSON.stringify(value)}`); return value as T; @@ -182,6 +249,16 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art finally { if (options.clock) await request("freeze"); } }, publishBackup: (publication: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }) => request("publishBackup", { publication }), + async prepareGitHub(companyId: string) { + const result = await request<{ result: { connectionId: string; grantId: string } }>("prepareGitHub", { github: { companyId } }); + return result.result; + }, + async settleGitHub(input: { connectionId: string; grantId: string }) { + return request("settleGitHub", { github: input }); + }, + tenantHeaders, + produceChatHistory: () => request("produceChatHistory"), + trace: () => request("trace"), snapshot: () => request("snapshot"), drain: () => request("drain"), resume: () => request("resume"), realtime: () => request("realtime"), @@ -197,17 +274,38 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art }, freeze: () => request("freeze"), get output() { return output; }, - async settled(quietMs = 500, deferredStartupHolds = 0) { + async settled(quietMs = 500, deferredStartupHolds = 0, settleOptions: { managedSetupRecovery?: true } = {}) { + // A successful managed health check self-notifies, then reconciles its + // future deadline on the next existing 30s pass. Observe both passes + + // 500ms quiet + 500ms margin without changing worker timing/admission. + const settleTimeoutMs = settleOptions.managedSetupRecovery ? 61_000 : 30_000; + const diagnostic = options.captureIdleWork ? (await import('./idle-work-capture.js')).createSettledWorkSamples( + () => [...captureSources!].map(([id, callers]) => ({ id, callers }))) : undefined; + const diagnosticId = diagnostic ? ++settledDiagnosticId : undefined; + let failure: { error: unknown } | undefined; let previous = -1, quietSince = Date.now(); - for (const deadline = Date.now() + 30_000; Date.now() < deadline;) { + try { + for (const deadline = Date.now() + settleTimeoutMs; Date.now() < deadline;) { + const snapshot = await request("snapshot"); + diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending); + if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed"); + if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now(); + previous = proxy.queries.length; + if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) { + diagnostic?.assertComplete(); return snapshot; + } + await delay(25); + } const snapshot = await request("snapshot"); - if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed"); - if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now(); - previous = proxy.queries.length; - if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) return snapshot; - await delay(25); + diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending); + throw new Error(`Maintenance did not settle: ${JSON.stringify({ timeoutMs: settleTimeoutMs, quietMs, queries: proxy.queries.length, pending: proxy.pending, snapshot })}\n${output.slice(-12_000)}`); + } catch (error) { failure = { error }; throw error; } + finally { + if (diagnostic) { + try { await writeFile(path.join(directory, `idle-work-settled-samples-${diagnosticId}.json`), JSON.stringify({ ...diagnostic.report(), settleDeadlineMs: settleTimeoutMs }, null, 2)); } + catch (error) { if (failure) throw new AggregateError([failure.error, error], 'Original settling and passive evidence failed'); throw error; } + } } - throw new Error(`Maintenance did not settle: ${JSON.stringify({ queries: proxy.queries.length, pending: proxy.pending, snapshot: await request("snapshot") })}\n${output.slice(-12_000)}`); }, }; } catch (error) { diff --git a/server/src/__tests__/instrumentation-signals.test.ts b/server/src/__tests__/instrumentation-signals.test.ts new file mode 100644 index 00000000000..44e5638b83a --- /dev/null +++ b/server/src/__tests__/instrumentation-signals.test.ts @@ -0,0 +1,30 @@ +import { afterEach, expect, it, vi } from 'vitest'; + +const sdk = vi.hoisted(() => ({ options: [] as Array>, start: vi.fn(), shutdown: vi.fn(async () => {}) })); +vi.mock('../peer-version-check.js', () => ({ checkExactPeerVersions: () => ({ ok: true }) })); +vi.mock('@opentelemetry/sdk-node', () => ({ NodeSDK: class { + constructor(options: Record) { sdk.options.push(options); } + start = sdk.start; + shutdown = sdk.shutdown; +} })); +vi.mock('@opentelemetry/auto-instrumentations-node', () => ({ getNodeAutoInstrumentations: () => [] })); +vi.mock('@opentelemetry/resources', () => ({ resourceFromAttributes: (attributes: unknown) => attributes })); +vi.mock('@opentelemetry/semantic-conventions', () => ({ ATTR_SERVICE_NAME: 'service.name', ATTR_SERVICE_VERSION: 'service.version' })); +vi.mock('@opentelemetry/exporter-trace-otlp-http', () => ({ OTLPTraceExporter: class {} })); + +afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); }); +it('configures current plural SDK signal options so an opted-in trace exporter cannot enable default metrics or logs', async () => { + vi.stubEnv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector.invalid:4318'); + vi.stubEnv('OTEL_EXPORTER_OTLP_PROTOCOL', 'http/json'); + vi.spyOn(process, 'once').mockReturnValue(process); + vi.resetModules(); + const instrumentation = await import('../instrumentation.js'); + await instrumentation.instrumentationReady; + expect(sdk.options).toHaveLength(1); + expect(sdk.options[0]).toMatchObject({ metricReaders: [], logRecordProcessors: [], traceExporter: expect.any(Object) }); + expect(sdk.options[0]).not.toHaveProperty('metricReader'); + expect(sdk.options[0]).not.toHaveProperty('logRecordProcessor'); + expect(sdk.start).toHaveBeenCalledOnce(); + await instrumentation.shutdownInstrumentation(); + expect(sdk.shutdown).toHaveBeenCalledOnce(); +}); diff --git a/server/src/__tests__/maintenance-harness-resources.test.ts b/server/src/__tests__/maintenance-harness-resources.test.ts index 179bb9e20eb..6ab58089901 100644 --- a/server/src/__tests__/maintenance-harness-resources.test.ts +++ b/server/src/__tests__/maintenance-harness-resources.test.ts @@ -125,3 +125,11 @@ describe("owned maintenance providers", () => { } }); }); + +describe("configured acceptance boundaries", () => { + it("rejects remote collectors and fake-clock configuration before opening PostgreSQL", async () => { + await expect(startMaintenanceServer("not-a-database-url", { configuration: { telemetryOrigin: "https://telemetry.example.test" } })).rejects.toThrow("loopback"); + await expect(startMaintenanceServer("not-a-database-url", { clock: true, configuration: { announcements: true } })).rejects.toThrow("real clock"); + await expect(startMaintenanceServer("not-a-database-url", { configuration: { automaticBackups: { intervalMinutes: 0, idleCheckpoint: true } } })).rejects.toThrow("positive integer"); + }); +}); diff --git a/server/src/__tests__/sleep-backup-observation.test.ts b/server/src/__tests__/sleep-backup-observation.test.ts new file mode 100644 index 00000000000..34bac7bbce2 --- /dev/null +++ b/server/src/__tests__/sleep-backup-observation.test.ts @@ -0,0 +1,61 @@ +import { expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, open, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js'; + +it('observes complete owned gzip bytes without retaining SQL and refuses missing, truncated, foreign, historical or failed checkpoints', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-observer-')); + const id = '12345678-1234-1234-1234-123456789012'; + const startedAt = new Date(Date.now() - 1000).toISOString(); + const backupDir = path.join(directory, 'backups'); await mkdir(backupDir); + try { + const observe = () => observeIdleBackupCheckpoint(directory, [], startedAt, id); + await expect(observe()).rejects.toThrow(); + await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker'); + await expect(observe()).rejects.toThrow('no new archive'); + const archive = path.join(backupDir, 'owned.sql.gz'), bytes = gzipSync(`COPY fixture;\n${id}\n`); + await writeFile(archive, bytes); + const receipt = await observe(); + expect(receipt.archives[0]).toMatchObject({ bytes: bytes.length, gzipVerified: true, fixtureIdentifierFound: true }); + expect(receipt.archives[0].sha256).toMatch(/^[a-f0-9]{64}$/); + expect(JSON.stringify(receipt)).not.toContain(id); + await expect(observeIdleBackupCheckpoint(directory, ['owned.sql.gz'], startedAt, id)).rejects.toThrow('no new archive'); + await expect(observeIdleBackupCheckpoint(directory, [], new Date(Date.now() + 1000).toISOString(), id)).rejects.toThrow('predates'); + await writeFile(archive, bytes.subarray(0, bytes.length - 4)); await expect(observe()).rejects.toThrow(); + await writeFile(archive, gzipSync('foreign fixture')); await expect(observe()).rejects.toThrow('owned fixture'); + await writeFile(archive, bytes); await writeFile(path.join(backupDir, 'db-backup-to-s3.failure'), 'failed'); + await expect(observe()).rejects.toThrow('failure marker'); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it('uses opened-file mtime when birthtime is zero and still rejects old or malformed provenance', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-no-birthtime-')); + const id = '12345678-1234-1234-1234-123456789012', startedAt = new Date(Date.now() - 1000).toISOString(); + const backupDir = path.join(directory, 'backups'); await mkdir(backupDir); + const archive = path.join(backupDir, 'owned.sql.gz'); + await writeFile(archive, gzipSync(`COPY fixture;\n${id}\n`)); + await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker'); + const handle = await open(archive, 'r'), prototype = Object.getPrototypeOf(handle), originalStat = prototype.stat; + await handle.close(); + let override: { birthtimeMs: number; mtimeMs?: number } = { birthtimeMs: 0 }; + const spy = vi.spyOn(prototype, 'stat').mockImplementation(async function (this: unknown, ...args: unknown[]) { + return Object.assign(await originalStat.apply(this, args), override); + }); + try { + const observed = await observeIdleBackupCheckpoint(directory, [], startedAt, id); + expect(observed.archives[0]).toMatchObject({ createdAtSource: 'mtime', gzipVerified: true, fixtureIdentifierFound: true }); + expect(Date.parse(observed.archives[0]!.createdAt)).toBeGreaterThanOrEqual(Date.parse(startedAt)); + for (const invalid of [ + { birthtimeMs: 0, mtimeMs: Date.parse(startedAt) - 1 }, { birthtimeMs: 0, mtimeMs: 0 }, + { birthtimeMs: 0, mtimeMs: NaN }, { birthtimeMs: NaN }, { birthtimeMs: -1 }, + { birthtimeMs: 0, mtimeMs: Date.now() + 60_000 }, + { birthtimeMs: Date.parse(startedAt) - 1, mtimeMs: Date.now() }, + ]) { + override = invalid; + await expect(observeIdleBackupCheckpoint(directory, [], startedAt, id)).rejects.toThrow('predates'); + } + await expect(observeIdleBackupCheckpoint(directory, [], 'invalid', id)).rejects.toThrow('checkpoint timestamp'); + } finally { spy.mockRestore(); await rm(directory, { recursive: true, force: true }); } +}); diff --git a/server/src/__tests__/sleep-configuration-acceptance.test.ts b/server/src/__tests__/sleep-configuration-acceptance.test.ts new file mode 100644 index 00000000000..a6d6a7a7ede --- /dev/null +++ b/server/src/__tests__/sleep-configuration-acceptance.test.ts @@ -0,0 +1,312 @@ +import { createHash, generateKeyPairSync, randomUUID, sign } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; +import { createServer } from 'node:http'; +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { configuredSleepFeatures, finalizeLocalSleepReceipt, observeConfiguredSignals, observeIncomingHttpWindow, observedNoIncomingHttp, sleepConfigurationDigest } from '../../../scripts/sleep-qualification/evidence.mjs'; +import { idleBackupArchiveNames, observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js'; +import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { managedSleepConfiguration, managedSleepServerOptions, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js'; +import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js'; +import { produceCompletedToolReview } from './helpers/maintenance/configuration-producers.js'; +import { observeMaintenanceBrowser } from './helpers/maintenance/browser-observation.js'; +import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js'; + +const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL'); +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !databaseUrl) throw new Error('Configured acceptance requires an explicit dedicated PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL; embedded allocation is forbidden'); +const cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA; +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !/^[a-f0-9]{40}$/.test(cloudSha ?? '')) throw new Error('Configured acceptance requires the exact PAPERCLIP_SLEEP_CLOUD_SHA'); +const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT; +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !cloudCheckout) throw new Error('Configured acceptance requires a reviewed PAPERCLIP_SLEEP_CLOUD_CHECKOUT with its built qualification broker adapter'); +const enabled = process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && Boolean(databaseUrl); +const quietMs = 90_000, apiCooldownMs = 600_000; +if (enabled && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Configured qualification requires retained raw evidence'); + +describe.skipIf(!enabled)('configured assembled sleep qualification (explicit owned external PostgreSQL only)', () => { + beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000); + it('retains normal credentials, completed work and a future routine through a real cooldown and quiet window', async () => { + const phase = (name: string) => console.info(JSON.stringify({ qualificationPhase: name, at: new Date().toISOString() })); + phase('starting-owned-fixtures'); + const collectorRequests: Array<{ path: string; body: unknown }> = []; + const collector = createServer(async (req, res) => { + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + collectorRequests.push({ path: req.url!, body: JSON.parse(Buffer.concat(chunks).toString('utf8')) }); + res.setHeader('Content-Type', 'application/json'); res.end('{}'); + }); + if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Qualification Cloud checkout does not match the exact requested SHA'); + const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href); + const broker = await startQualificationBroker(); + const s3 = createMaintenanceS3Boundary(); + try { + await withMaintenanceProvider(s3.server, async s3Origin => { + await withMaintenanceProvider(collector, async origin => { + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'sleep-qualification', alg: 'EdDSA', use: 'sig' }; + const configuration = managedSleepConfiguration(origin, process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1'); + // Independent intended logical contract; collector ports are observed separately. + const expected = { target: 'owned-local-assembled', appSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), cloudSha: cloudSha!, + configDigest: createHash('sha256').update(JSON.stringify({ automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, + telemetry: true, otel: { enabled: true, protocol: 'http/json' }, serveUi: process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1', + managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' })).digest('hex'), + collectorOrigins: { telemetry: `${origin}/telemetry`, otel: origin } }; + expect(sleepConfigurationDigest(configuration)).toBe(expected.configDigest); + const startedAt = new Date().toISOString(); + const server = await startMaintenanceServer(databaseUrl!, managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker)); + phase('actual-server-ready'); + let fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const control = (action: string, idleCapabilities?: unknown) => { + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url'); + const now = Math.floor(Date.now() / 1000); + const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid }); + const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) }); + return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` }; + }; + let failure: { error: unknown } | undefined; + try { + await writeFile(path.join(server.directory, 'sleep-managed-startup.json'), JSON.stringify(await server.snapshot(), null, 2)); + const company = await provisionManagedSleepCompany(server, fixture); + expect(await server.api('GET', '/announcements/current')).toMatchObject({ id: 'new-projects' }); + const announcement = (await server.snapshot()).announcement; + expect(announcement?.intercepted).toBe(true); + expect(announcement?.requests).toEqual(['https://pages.paperclip.ing/announcements/v1/current.json']); + expect(announcement?.unmatched).toEqual([]); + const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Owned process worker', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', "console.log('sleep-acceptance-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const chatHistory = await server.produceChatHistory(); + const toolHistory = await produceCompletedToolReview(server, fixture, company.id); + const github = await server.prepareGitHub(company.id); + await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true }); + const githubContinuity = await server.settleGitHub(github); + await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1); + await broker.enqueue(); + await broker.deliver(`http://127.0.0.1:${server.port}`, () => control('connector-events:wake')[CLOUD_CONTROL_HEADER]); + // The normally enabled connection owner leases and acknowledges the real sealed event. + await expect.poll(async () => { + const [delivery] = await fixture.$client`select status from connection_event_deliveries where provider_delivery_id='qualification-delivery'`; + return delivery?.status; + }, { timeout: 30_000 }).toBe('processed'); + expect(await broker.pending()).toEqual([]); + const brokerBinding = await broker.binding(); + expect(brokerBinding?.bindings).toHaveLength(1); + const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Retained qualification credential' }); + const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'Owned completed history', status: 'todo', assigneeAgentId: agent.id }); + await expect.poll(async () => { + const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; + return run?.status; + }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + const routine = await server.api<{ id: string }>('POST', `/companies/${company.id}/routines`, { title: 'Harmless future obligation', assigneeAgentId: agent.id }); + await server.api('POST', `/routines/${routine.id}/triggers`, { kind: 'schedule', cronExpression: (() => { const next = new Date(Date.now() + 24 * 60 * 60_000); return `0 ${next.getUTCHours()} ${next.getUTCDate()} ${next.getUTCMonth() + 1} *`; })(), timezone: 'UTC' }); + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); + const lastAuthenticatedApiAt = performance.now(); + await writeFile(path.join(server.directory, 'sleep-setup-receipt.json'), JSON.stringify({ ...expected, configuration, company, + agentId: agent.id, keyId: key.id, announcement, chatHistory, toolHistory, githubContinuity, brokerBinding, + brokerBoundary: broker.boundary, s3: s3.snapshot(), snapshot: await server.snapshot(), + boundary: 'Actual managed startup/tenant provisioning/producers; Cloud front door, GitHub HTTP and S3 service are owned simulated transports' }, null, 2)); + expect(s3.snapshot().errors).toEqual([]); + let browser; + try { + browser = process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1' ? await observeMaintenanceBrowser(`http://127.0.0.1:${server.port}`, 'Organize your work.', server.tenantHeaders) : undefined; + await writeFile(path.join(server.directory, 'sleep-browser-receipt.json'), JSON.stringify({ browser, observedAt: new Date().toISOString() }, null, 2)); + } catch (error) { + await writeFile(path.join(server.directory, 'sleep-browser-error.json'), JSON.stringify({ observedAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) }, null, 2)); + throw error; + } + // Setup-only owner settling never substitutes for the later full quiet window. + await server.settled(500, 0, { managedSetupRecovery: true }); + const earlyDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + try { + const earlyCapabilities = { incomingWake: await broker.capability(earlyDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const earlyRead = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:read', earlyCapabilities)); + const early = await stabilizeEarlyDiagnostics(server, { ownerId: earlyDrain.ownerId, apiSucceededAt: lastAuthenticatedApiAt, read: earlyRead, + persist: observations => writeFile(path.join(server.directory, 'sleep-early-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) }); + await writeFile(path.join(server.directory, 'sleep-early-diagnostics.json'), JSON.stringify(early, null, 2)); + } finally { await server.api('DELETE', `/instance/task-drain?ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:stop')); } + // Anchor the full cooldown to a genuine successful opaque-key request. + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); + const lastApiActivity = performance.now(); + await fixture.$client.end({ timeout: 1 }); + await server.trace(); + phase('first-real-ten-minute-api-cooldown'); + // The real authenticated-use cooldown is part of ordinary behavior. + // No fake clock, production timing override, tenant health poll or SQL observer. + await delay(apiCooldownMs + 100); + await server.settled(); + const activeSignals = observeConfiguredSignals(collectorRequests); + expect(activeSignals.qualificationSpans).toBeGreaterThan(0); + expect(activeSignals.telemetryRequests).toBeGreaterThan(0); + const before = await server.snapshot(), queries = server.proxy.queries.length, transport = server.proxy.transport; + const wallStart = performance.now(), quietStartedAt = new Date().toISOString(); + phase('first-real-ninety-second-quiet-window'); + await delay(quietMs); + const elapsedMs = performance.now() - wallStart, quietFinishedAt = new Date().toISOString(); + const after = await server.snapshot(), finalTransport = server.proxy.transport; + const quiet = { startedAt: quietStartedAt, finishedAt: quietFinishedAt, elapsedMs, clock: 'real', + sqlQueries: server.proxy.queries.length - queries, + connectionsOpened: finalTransport.connectionsOpened - transport.connectionsOpened, + connectionsRemaining: finalTransport.connectionsOpened - finalTransport.connectionsClosed, + nonDatabaseRequests: after.outgoingHttpRequests - before.outgoingHttpRequests, + nonDatabaseBytes: after.network.nonDatabaseBytesWritten - before.network.nonDatabaseBytesWritten, + observerRequests: observeIncomingHttpWindow(before.incomingHttp, after.incomingHttp) }; + await writeFile(path.join(server.directory, 'sleep-first-frozen-quiet.json'), JSON.stringify({ quiet, announcementBefore: before.announcement, announcementAfter: after.announcement }, null, 2)); + const signalObservation = observeConfiguredSignals(collectorRequests); + await writeFile(path.join(server.directory, 'sleep-first-signal-observation.json'), JSON.stringify(signalObservation, null, 2)); + expect(signalObservation.qualificationSpans).toBeGreaterThan(0); + expect(signalObservation.telemetryRequests).toBeGreaterThan(0); + expect(signalObservation.metricRequests).toBe(0); expect(signalObservation.logRequests).toBe(0); + expect(signalObservation.malformedTraceRequests).toBe(0); expect(signalObservation.unexpectedRequests).toBe(0); + expect(quiet.elapsedMs).toBeGreaterThanOrEqual(quietMs); + expect(quiet.sqlQueries).toBe(0); + expect(quiet.connectionsOpened).toBe(0); + expect(quiet.connectionsRemaining).toBe(0); + expect(quiet.nonDatabaseRequests).toBe(0); + expect(quiet.nonDatabaseBytes).toBe(0); + // The installed listener must have observed the actual setup HTTP traffic. + expect(quiet.observerRequests.before.requests).toBeGreaterThan(0); + expect(observedNoIncomingHttp(quiet.observerRequests, quiet)).toBe(true); + expect(after.announcement).toEqual(before.announcement); + const backupNamesBefore = await idleBackupArchiveNames(server.directory); + const checkpointStartedAt = new Date().toISOString(); + const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const read = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + const initial = await read(); + await writeFile(path.join(server.directory, 'sleep-initial-safety.json'), JSON.stringify(initial, null, 2)); + const final = await read(); + phase('first-owned-eligibility-reports-read'); + // Persist raw refusal evidence before any assertion can terminate the case. + await writeFile(path.join(server.directory, 'sleep-initial-final-safety.json'), JSON.stringify({ initial, final }, null, 2)); + if (initial.idleSleepSafety.backgroundWork !== 'none' || final.idleSleepSafety.backgroundWork !== 'none') { + const diagnostic = await server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + await writeFile(path.join(server.directory, 'sleep-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2)); + } + const { parseApiWakeAdmission, authorizeApiWakeAdmission } = await import('../services/idle-api-wake-admission.js'); + const assertIncomingProof = (report: any, incomingWake: any, ownerId: string) => { + expect(report.idleSleepSafety.incomingWake).toEqual(incomingWake); + const admission = parseApiWakeAdmission(report.idleSleepSafety.apiWakeAdmission); + expect(admission).toMatchObject({ stackId: 'maintenance-stack', ownerId, + capabilityGeneration: incomingWake.generation, challenge: incomingWake.apiKeyProof.challenge }); + expect(authorizeApiWakeAdmission(admission, { stackId: 'maintenance-stack', token: key.token, now: Date.now() })).toBeDefined(); + }; + assertIncomingProof(initial, capabilities.incomingWake, drain.ownerId); + assertIncomingProof(final, capabilities.incomingWake, drain.ownerId); + expect(final.idleSleepSafety.apiWakeAdmission).toEqual(initial.idleSleepSafety.apiWakeAdmission); + const proof = final.idleSleepSafety; + const backupCheckpoint = await observeIdleBackupCheckpoint(server.directory, backupNamesBefore, checkpointStartedAt, company.id); + const observations = { version: 1, ...expected, startedAt, + configuration, browser, s3: s3.snapshot(), announcement: after.announcement, chatHistory, toolHistory, githubContinuity, brokerBinding, brokerBoundary: broker.boundary, backupCheckpoint, + signalObservation, features: configuredSleepFeatures({ announcement: after.announcement, signalObservation, backupCheckpoint }), + quiet, + eligibility: { observedAt: new Date().toISOString(), backgroundWork: proof.backgroundWork, ownedHold: final.ownerId === drain.ownerId, apiCooldownElapsedMs: performance.now() - lastApiActivity, + unchangedGeneration: /^[a-f0-9]{64}$/.test(proof.deadlineProof?.generation ?? '') && initial.idleSleepSafety.deadlineProof?.generation === proof.deadlineProof?.generation }, initial, final, + live: { databaseSuspension: 'INCONCLUSIVE', hostSuspension: 'INCONCLUSIVE', backupPersistence: 'INCONCLUSIVE', reason: 'Local actual server with simulated Cloud signer; no deployed provider target authorized' }, + coverage: { announcements: 'Actual route, validated feed/cache and UI with one named injected provider transport fetch; DNS/HTTPS guard covered separately, not exercised here.', browser: browser ? 'Actual Chromium foreground/background measured, then closed before server quiet window' : 'Closed; set PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE=1 with built UI and headed Chromium for background/active evidence', broker: 'Retained enabled connection, actual encrypted OAuth vault, actual continuity/binding owners and signed/sealed production broker delivery; host capability from actual ConnectorStore', history: 'Normal task/credential/routine, actual tool review rejection/outbox retirement, actual signed chat ingress completion and endpoint archival' } }; + await writeFile(path.join(server.directory, 'sleep-configuration-observations.json'), JSON.stringify(observations, null, 2)); + const { receipt, report } = finalizeLocalSleepReceipt(observations, expected); + await writeFile(path.join(server.directory, 'sleep-configuration-receipt.json'), JSON.stringify(receipt, null, 2)); + await writeFile(path.join(server.directory, 'sleep-configuration-evaluation.json'), JSON.stringify(report, null, 2)); + expect(proof.version).toBe(3); expect(proof.backgroundWork).toBe('none'); + expect(proof.deadlineProof).toMatchObject({ catalog: 'domain_deadlines_v1', generation: expect.stringMatching(/^[a-f0-9]{64}$/) }); + fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const [retained] = await fixture.$client`select (select count(*)::int from agent_api_keys where id=${key.id}) as keys, (select count(*)::int from issues where id=${issue.id}) as issues, (select count(*)::int from routines where id=${routine.id}) as routines, (select enabled from tool_connections where id=${github.connectionId}) as github_enabled`; + expect(retained).toEqual({ keys: 1, issues: 1, routines: 1, github_enabled: true }); + await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop')); + const nextIssue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'First ordinary producer after quiet', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` }); + const reactivationApiAt = performance.now(); + await expect.poll(async () => { + const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${nextIssue.id} order by created_at desc limit 1`; + return run?.status; + }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${nextIssue.id}`, { status: 'done' }); + const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${nextIssue.id}`; + expect(executions.count).toBe(1); + await fixture.$client.end({ timeout: 1 }); + phase('reactivation-real-ten-minute-api-cooldown'); + await delay(apiCooldownMs + 100); + await server.settled(); + const resumed = await server.snapshot(), resumedQueries = server.proxy.queries.length, resumedTransport = server.proxy.transport; + const resumedStart = performance.now(), resumedStartedAt = new Date().toISOString(); + phase('reactivation-real-ninety-second-quiet-window'); + await delay(quietMs); + const resumedElapsedMs = performance.now() - resumedStart, resumedFinishedAt = new Date().toISOString(); + const resumedEnd = await server.snapshot(), resumedFinalTransport = server.proxy.transport; + // Freeze every measurement before signed drain/proof requests do SQL. + const resumedQuiet = { startedAt: resumedStartedAt, finishedAt: resumedFinishedAt, elapsedMs: resumedElapsedMs, clock: 'real', + sqlQueries: server.proxy.queries.length - resumedQueries, + connectionsOpened: resumedFinalTransport.connectionsOpened - resumedTransport.connectionsOpened, + connectionsRemaining: resumedFinalTransport.connectionsOpened - resumedFinalTransport.connectionsClosed, + nonDatabaseRequests: resumedEnd.outgoingHttpRequests - resumed.outgoingHttpRequests, + nonDatabaseBytes: resumedEnd.network.nonDatabaseBytesWritten - resumed.network.nonDatabaseBytesWritten, + observerRequests: observeIncomingHttpWindow(resumed.incomingHttp, resumedEnd.incomingHttp) }; + await writeFile(path.join(server.directory, 'sleep-resumed-frozen-quiet.json'), JSON.stringify({ quiet: resumedQuiet, announcementBefore: resumed.announcement, announcementAfter: resumedEnd.announcement }, null, 2)); + const resumedSignals = observeConfiguredSignals(collectorRequests); + await writeFile(path.join(server.directory, 'sleep-resumed-signal-observation.json'), JSON.stringify(resumedSignals, null, 2)); + expect(resumedSignals.qualificationSpans).toBeGreaterThan(0); + expect(resumedSignals.telemetryRequests).toBeGreaterThan(0); + expect(resumedSignals.metricRequests).toBe(0); expect(resumedSignals.logRequests).toBe(0); + expect(resumedSignals.malformedTraceRequests).toBe(0); expect(resumedSignals.unexpectedRequests).toBe(0); + expect(resumedQuiet.elapsedMs).toBeGreaterThanOrEqual(quietMs); + expect(resumedQuiet.sqlQueries).toBe(0); + expect(resumedQuiet.connectionsOpened).toBe(0); + expect(resumedQuiet.connectionsRemaining).toBe(0); + expect(resumedQuiet.nonDatabaseRequests).toBe(0); + expect(resumedQuiet.nonDatabaseBytes).toBe(0); + expect(resumedQuiet.observerRequests.before.requests).toBeGreaterThan(0); + expect(observedNoIncomingHttp(resumedQuiet.observerRequests, resumedQuiet)).toBe(true); + expect(resumedEnd.announcement).toEqual(resumed.announcement); + expect(resumedEnd.announcement?.unmatched).toEqual([]); + const resumedDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + const resumedCapabilities = { incomingWake: await broker.capability(resumedDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const readResumed = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities)); + const resumedInitial = await readResumed(); + await writeFile(path.join(server.directory, 'sleep-resumed-initial-safety.json'), JSON.stringify(resumedInitial, null, 2)); + const resumedProof = await readResumed(); + await writeFile(path.join(server.directory, 'sleep-resumed-initial-final-safety.json'), JSON.stringify({ initial: resumedInitial, final: resumedProof }, null, 2)); + if (resumedInitial.idleSleepSafety.backgroundWork !== 'none' || resumedProof.idleSleepSafety.backgroundWork !== 'none') { + const diagnostic = await server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities)); + await writeFile(path.join(server.directory, 'sleep-resumed-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2)); + } + assertIncomingProof(resumedInitial, resumedCapabilities.incomingWake, resumedDrain.ownerId); + assertIncomingProof(resumedProof, resumedCapabilities.incomingWake, resumedDrain.ownerId); + expect(resumedProof.idleSleepSafety.apiWakeAdmission).toEqual(resumedInitial.idleSleepSafety.apiWakeAdmission); + expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toMatch(/^[a-f0-9]{64}$/); + expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toBe(resumedInitial.idleSleepSafety.deadlineProof?.generation); + expect(resumedProof.idleSleepSafety.backgroundWork).toBe('none'); + await writeFile(path.join(server.directory, 'sleep-local-reactivation-receipt.json'), JSON.stringify({ + version: 1, actualProducer: 'POST company issues with process assignee using the retained opaque key after owned drain release', executions: executions.count, + realApiCooldownElapsedMs: performance.now() - reactivationApiAt, + quiet: resumedQuiet, + signalObservation: resumedSignals, + realElapsedMs: resumedQuiet.elapsedMs, sqlQueries: resumedQuiet.sqlQueries, + networkRequests: resumedQuiet.nonDatabaseRequests, + initial: resumedInitial, + eligibility: resumedProof.idleSleepSafety, + boundary: 'Actual local application reactivation, real cooldown and fresh eligibility; no provider suspension or deployed inbound-wake claim', + }, null, 2)); + const s3Evidence = s3.snapshot(); + await writeFile(path.join(server.directory, 'sleep-s3-boundary.json'), JSON.stringify(s3Evidence, null, 2)); + expect(s3Evidence.errors).toEqual([]); + phase('configured-qualification-receipts-complete'); + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + try { await writeFile(path.join(server.directory, 'sleep-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + try { await server.stop(); } catch (error) { errors.push(error); } + try { await fixture.$client.end(); } catch (error) { errors.push(error); } + // Shutdown can flush storage; capture and validate only after child exit. + try { await writeFile(path.join(server.directory, 'sleep-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Configured work or owned cleanup failed'); + } + }); + }); + } finally { await broker.stop(); } + }, 1_800_000); +}); diff --git a/server/src/__tests__/sleep-early-diagnostics.test.ts b/server/src/__tests__/sleep-early-diagnostics.test.ts new file mode 100644 index 00000000000..b9db5979378 --- /dev/null +++ b/server/src/__tests__/sleep-early-diagnostics.test.ts @@ -0,0 +1,15 @@ +import { expect, it } from 'vitest'; +import { assertEarlySleepDiagnostics } from './helpers/maintenance/early-sleep-diagnostics.js'; +const fixture = () => ({ idleSleepSafety: { diagnostics: { complete: true, checks: [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog', 'issues'].map(reason => ({ reason, state: 'none' })), +] } } }); +it('retains only explicit early cooldown/diagnostic-backup reasons and refuses every unexpected or unknown blocker', () => { + expect(() => assertEarlySleepDiagnostics(fixture(), 90_000)).not.toThrow(); + for (const [reason, state] of [['issues', 'present'], ['deadlines.catalog', 'unknown'], ['inspection.hold', 'not_checked'], ['local.registered_probes', 'unknown']]) { + const f = fixture(); f.idleSleepSafety.diagnostics.checks.find(check => check.reason === reason)!.state = state; + expect(() => assertEarlySleepDiagnostics(f, 90_000)).toThrow('Unexpected'); + } + const missing = fixture(); missing.idleSleepSafety.diagnostics.checks.shift(); expect(() => assertEarlySleepDiagnostics(missing, 90_000)).toThrow('Missing'); + for (const duration of [NaN, Infinity, -1, 600_000]) expect(() => assertEarlySleepDiagnostics(fixture(), duration)).toThrow('cooldown'); +}); diff --git a/server/src/__tests__/sleep-idle-work-capture.test.ts b/server/src/__tests__/sleep-idle-work-capture.test.ts new file mode 100644 index 00000000000..114b04057bb --- /dev/null +++ b/server/src/__tests__/sleep-idle-work-capture.test.ts @@ -0,0 +1,90 @@ +import { afterEach, expect, it } from 'vitest'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { createMaintenanceIdleWorkCapture as createCapture, createSettledWorkSamples } from './helpers/maintenance/idle-work-capture.js'; + +const captures: ReturnType[] = []; +function createMaintenanceIdleWorkCapture(...args: Parameters) { + const capture = createCapture(...args); captures.push(capture); return capture; +} +afterEach(() => { for (const capture of captures.splice(0)) capture.dispose(); }); + +const stack = () => 'Error\n at begin (/checkout/server/src/services/owned-work.ts:12:3)\n at owner (/checkout/server/src/index.ts:100:5)'; +function original() { + let active = 0, generation = 0, finishes = 0, tracked: Promise | undefined; + const beginIdleTrackedWork = () => { + active++; generation++; let done = false; + return () => { finishes++; if (!done) { done = true; active--; generation++; } }; + }; + const trackIdleWork = (work: Promise) => { + const finish = beginIdleTrackedWork(); const result = work.finally(finish); tracked = result; return result; + }; + return { beginIdleTrackedWork, trackIdleWork, idleWorkSnapshot: () => ({ active, generation }), + finishes: () => finishes, lastTracked: () => tracked }; +} +it('delegates original finish calls/idempotence and returns the exact original tracked promise and rejection', async () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack }); + const finish = capture.beginIdleTrackedWork(); expect(actual.idleWorkSnapshot().active).toBe(1); + finish(); finish(); expect(actual.finishes()).toBe(2); expect(actual.idleWorkSnapshot().active).toBe(0); + let resolve!: (value: string) => void; + const tracked = capture.trackIdleWork(new Promise(done => { resolve = done; })); + expect(tracked).toBe(actual.lastTracked()); expect(capture.snapshot().observedActive).toBe(1); + resolve('same'); await expect(tracked).resolves.toBe('same'); expect(capture.snapshot().observedActive).toBe(0); + const failure = new Error('original rejection'), rejected = capture.trackIdleWork(Promise.reject(failure)); + expect(rejected).toBe(actual.lastTracked()); await expect(rejected).rejects.toBe(failure); + expect(capture.snapshot().observedActive).toBe(0); expect(actual.idleWorkSnapshot().active).toBe(0); + capture.dispose(); capture.dispose(); expect(capture.snapshot().disposed).toBe(true); + await expect(capture.trackIdleWork(Promise.resolve('after-dispose'))).resolves.toBe('after-dispose'); + expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().observedActive).toBe(0); +}); +it('cannot throw from passive metadata into actual work and makes missing observation explicit', () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack, now: () => { throw new Error('private observer detail'); } }); + let finish!: () => void; expect(() => { finish = capture.beginIdleTrackedWork(); }).not.toThrow(); + expect(actual.idleWorkSnapshot().active).toBe(1); expect(capture.snapshot().missingPaths).toBeGreaterThan(0); + expect(() => finish()).not.toThrow(); expect(actual.idleWorkSnapshot().active).toBe(0); + expect(capture.snapshot().observerErrors).toBeGreaterThan(0); expect(JSON.stringify(capture.summary())).not.toContain('private observer detail'); +}); +it('bounds active entries, completion tail and filtered callsites without clearing original work', () => { + const actual = original(); + const callers = () => 'private outside value\n' + Array.from({ length: 8 }, (_, i) => ` at work (/root/server/src/services/owned-${i}.ts:1:2)`).join('\n'); + const capture = createMaintenanceIdleWorkCapture(actual, { stack: callers }); + const finish = Array.from({ length: 513 }, () => capture.beginIdleTrackedWork()); + const busy = capture.snapshot(); expect(busy.active).toHaveLength(512); expect(busy.overflow).toBe(1); + expect(busy.newSources[0]!.callers).toHaveLength(4); expect(JSON.stringify(busy)).not.toContain('private outside value'); + expect(actual.idleWorkSnapshot().active).toBe(513); expect(busy.missingPaths).toBeGreaterThan(0); + finish.forEach(done => done()); expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().completionTail).toHaveLength(64); +}); +it('bounds unique source metadata and preserves original finisher failures', () => { + const actual = original(); let source = 0; + const capture = createMaintenanceIdleWorkCapture(actual, { stack: () => `Error\n at owner (/checkout/server/src/services/source-${source++}.ts:1:1)` }); + for (let i = 0; i < 513; i++) capture.beginIdleTrackedWork()(); + const state = capture.snapshot(); expect(capture.summary().sourceCatalog).toHaveLength(512); expect(state.sourceOverflow).toBe(1); + expect(actual.idleWorkSnapshot().active).toBe(0); + const failure = new Error('original finisher'), begin = actual.beginIdleTrackedWork; + actual.beginIdleTrackedWork = () => { const finish = begin(); return () => { finish(); throw failure; }; }; + const observed = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = observed.beginIdleTrackedWork(); + expect(() => finish()).toThrow(failure); expect(actual.idleWorkSnapshot().active).toBe(0); expect(observed.snapshot().observedActive).toBe(0); +}); +it('bounds settle samples and stores callsites once, while missing paths forbid diagnostic completion', () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = capture.beginIdleTrackedWork(); + const state = capture.snapshot(), catalog = state.newSources; + const samples = createSettledWorkSamples(() => catalog); + for (let i = 0; i < 2442; i++) samples.sample({ now: 1000 + i, idle: actual.idleWorkSnapshot(), workCapture: state }, 20 + i, 0); + const report = samples.report(); expect(report.samples).toHaveLength(2441); expect(report.sampleLimit).toBe(2441); expect(report.overflowSamples).toBe(1); + expect(report.samples[1]!.queryDelta).toBe(1); expect(report.sourceCatalog).toHaveLength(1); + expect(JSON.stringify(report.samples)).not.toContain('owned-work'); expect(() => samples.assertComplete()).toThrow('incomplete'); + finish(); + const missing = createSettledWorkSamples(() => []); missing.sample({ now: 0, idle: actual.idleWorkSnapshot() }, 0, 0); + expect(() => missing.assertComplete()).toThrow('incomplete'); +}); +it('intercepts the actual Node24 TS module and preserves every other original export and existing state', async () => { + const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url))], { timeout: 15_000, maxBuffer: 128 * 1024 }); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ identity: true, originalStatePreserved: true, finalActive: 0 }); +}); +it('preserves actual Node unhandledRejection with the same original tracked promise and reason', async () => { + const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url)), 'unhandled'], { timeout: 15_000, maxBuffer: 128 * 1024 }); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 }); +}); diff --git a/server/src/__tests__/sleep-incoming-http-observation.test.ts b/server/src/__tests__/sleep-incoming-http-observation.test.ts new file mode 100644 index 00000000000..e42d215eb54 --- /dev/null +++ b/server/src/__tests__/sleep-incoming-http-observation.test.ts @@ -0,0 +1,100 @@ +import { createServer, request } from "node:http"; +import { setImmediate } from "node:timers/promises"; +import { expect, it } from "vitest"; +import { observeIncomingHttpWindow, observedNoIncomingHttp } from "../../../scripts/sleep-qualification/evidence.mjs"; +import { observeMaintenanceIncomingHttp } from "./helpers/maintenance/incoming-http-observation.js"; +import { withMaintenanceProvider } from "./helpers/maintenance/server-harness.js"; + +it("counts a direct SQL-free health fetch without retaining credentials or changing its response", async () => { + // This fixture has no database or outgoing client; its only work is answering HTTP. + const server = createServer((_req, res) => { res.setHeader("Content-Type", "application/json"); res.end('{"status":"ok"}'); }); + server.on("upgrade", (_req, socket) => socket.destroy()); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const before = capture.snapshot(), startedAt = new Date().toISOString(); + expect(before).toMatchObject({ installed: true, valid: true, requests: 0 }); + expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, before), { startedAt: before.observedAt, finishedAt: before.observedAt })).toBe(true); + const response = await fetch(`${origin}/api/health?fixture=private-query`, { signal: AbortSignal.timeout(5_000), headers: { Authorization: "Bearer fixture-private-token" } }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ status: "ok" }); + const finishedAt = new Date().toISOString(), after = capture.snapshot(); + expect(after).toMatchObject({ installed: true, valid: true, captureId: before.captureId }); + const observed = observeIncomingHttpWindow(before, after); + expect(observed.requests).toBe(1); + expect(observedNoIncomingHttp(observed, { startedAt, finishedAt })).toBe(false); + expect(JSON.stringify(observed)).not.toMatch(/private-query|fixture-private-token|Authorization|\/api\/health/); + }); + } finally { capture.dispose(); } +}); + +it("observes a genuinely request-free interval without making snapshot reads into HTTP", async () => { + const server = createServer((_req, res) => res.end()); + server.on("upgrade", (_req, socket) => socket.destroy()); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async () => { + const before = capture.snapshot(), startedAt = new Date().toISOString(); + capture.snapshot(); await setImmediate(); capture.snapshot(); + const finishedAt = new Date().toISOString(), after = capture.snapshot(); + expect(observeIncomingHttpWindow(before, after).requests).toBe(0); + expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, after), { startedAt, finishedAt })).toBe(true); + // This short boundary test does not claim the configured case's 90-second quiet proof. + }); + } finally { capture.dispose(); } +}); + +it("counts an actual HTTP upgrade as incoming activity", async () => { + const server = createServer((_req, res) => res.end()); + const upgrade = (_req: unknown, socket: import("node:stream").Duplex) => { + socket.end("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n"); + }; + server.on("upgrade", upgrade); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const before = capture.snapshot(); + const client = request(origin, { signal: AbortSignal.timeout(5_000), headers: { Connection: "Upgrade", Upgrade: "fixture" } }); + try { + await new Promise((resolve, reject) => { + client.once("error", reject); + client.once("upgrade", (_response, socket) => { socket.destroy(); resolve(); }); + client.end(); + }); + } finally { client.destroy(); } + expect(observeIncomingHttpWindow(before, capture.snapshot()).requests).toBe(1); + }); + } finally { capture.dispose(); server.off("upgrade", upgrade); } +}); + +it("invalidates a detached listener and disposes only its own listeners", async () => { + const app = (_req: unknown, res: import("node:http").ServerResponse) => res.end("app"); + const server = createServer(app); + const unrelatedUpgrade = () => {}; + server.on("upgrade", unrelatedUpgrade); + const originalRequest = server.listeners("request"), originalUpgrade = server.listeners("upgrade"); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const counter = server.listeners("request")[0]! as (...args: unknown[]) => void; + server.off("request", counter); + expect(capture.snapshot()).toMatchObject({ installed: true, valid: false }); + // Reattaching cannot erase an unobserved interval. + server.prependListener("request", counter); + expect(capture.snapshot().valid).toBe(false); + capture.dispose(); capture.dispose(); + expect(capture.snapshot()).toMatchObject({ installed: false, valid: false }); + expect(server.listeners("request")).toEqual(originalRequest); + expect(server.listeners("upgrade")).toEqual(originalUpgrade); + expect(await (await fetch(origin, { signal: AbortSignal.timeout(5_000) })).text()).toBe("app"); + }); + } finally { capture.dispose(); } +}); + +it("leaves Node's unsupported-upgrade behavior alone when the app has no upgrade handler", () => { + const server = createServer((_req, res) => res.end()); + const requestHandlers = server.listeners("request"), upgradeHandlers = server.listeners("upgrade"); + expect(() => observeMaintenanceIncomingHttp(server)).toThrow("requires existing app request and upgrade handlers"); + expect(server.listeners("request")).toEqual(requestHandlers); + expect(server.listeners("upgrade")).toEqual(upgradeHandlers); +}); diff --git a/server/src/__tests__/sleep-managed-fixture.test.ts b/server/src/__tests__/sleep-managed-fixture.test.ts new file mode 100644 index 00000000000..0f7a3ada9f5 --- /dev/null +++ b/server/src/__tests__/sleep-managed-fixture.test.ts @@ -0,0 +1,69 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { createMaintenanceS3Boundary, decodeMaintenanceAwsChunks } from './helpers/maintenance/s3-boundary.js'; +import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { createS3StorageProvider } from '../storage/s3-provider.js'; +import { classifyEarlyDiagnostic, stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; + +afterEach(() => vi.unstubAllEnvs()); +it('uses real small-object SDK HTTP with conditional bytes, reads and deletion; unknown operations fail closed', async () => { + vi.stubEnv('AWS_ACCESS_KEY_ID', 'owned-fixture'); vi.stubEnv('AWS_SECRET_ACCESS_KEY', 'owned-fixture'); vi.stubEnv('AWS_SESSION_TOKEN', undefined); + const fixture = createMaintenanceS3Boundary(); + await withMaintenanceProvider(fixture.server, async endpoint => { + const provider = createS3StorageProvider({ bucket: 'maintenance-test', region: 'us-east-1', endpoint, forcePathStyle: true }); + const input = { objectKey: 'owned/value', contentType: 'text/plain', contentLength: 3, body: Buffer.from('one') }; + const first = await provider.putObjectConditional!({ ...input, ifNoneMatch: '*' }); + expect(first.etag).toBeTruthy(); + await expect(provider.putObjectConditional!({ ...input, ifNoneMatch: '*' })).rejects.toMatchObject({ name: 'PreconditionFailed' }); + await expect(provider.putObjectConditional!({ ...input, ifMatch: 'wrong' })).rejects.toMatchObject({ name: 'PreconditionFailed' }); + await provider.putObjectConditional!({ ...input, body: Buffer.from('two'), ifMatch: first.etag }); + const object = await provider.getObject({ objectKey: input.objectKey }); const chunks: Buffer[] = []; + for await (const chunk of object.stream) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe('two'); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(true); + expect((await provider.listObjects!({ prefix: 'owned/' })).keys).toEqual([input.objectKey]); + await provider.deleteObject({ objectKey: input.objectKey }); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(false); + expect(fixture.snapshot().errors).toEqual([]); + const unknown = await fetch(`${endpoint}/maintenance-test/key?uploads`, { method: 'POST' }); expect(unknown.status).toBe(501); + expect(fixture.snapshot().errors).toHaveLength(1); + }); +}); +it('decodes only complete bounded AWS checksum chunks and refuses malformed framing', () => { + expect(decodeMaintenanceAwsChunks(Buffer.from('3\r\none\r\n3\r\ntwo\r\n0\r\nx-amz-checksum-crc32:YWJj\r\n\r\n')).toString()).toBe('onetwo'); + for (const wire of ['3\r\none', '3;signature=unknown\r\none\r\n0\r\n\r\n', '0\r\nunknown:secret\r\n\r\n']) expect(() => decodeMaintenanceAwsChunks(Buffer.from(wire))).toThrow(); +}); +it('retries only observed transient inspection states and rejects durable blockers even alongside them', () => { + const checks = [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, { reason: 'inspection.hold', state: 'none' }, + { reason: 'inspection.generation', state: 'unknown' }, { reason: 'local.tracked_work', state: 'present' }, + { reason: 'plugins.runtime_drain', state: 'not_checked' }, { reason: 'durable.snapshot', state: 'none' }, + { reason: 'deadlines.catalog', state: 'none' }, + ]; + const report = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: false, checks } } }; + expect(classifyEarlyDiagnostic(report, 90_000)).toBe('transient'); + const durable = structuredClone(report); durable.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' }); + expect(() => classifyEarlyDiagnostic(durable, 90_000)).toThrow('chat_endpoints'); + const unknown = structuredClone(report); unknown.idleSleepSafety.diagnostics.checks.push({ reason: 'deadlines.reader', state: 'unknown' }); + expect(() => classifyEarlyDiagnostic(unknown, 90_000)).toThrow('deadlines.reader'); + for (const check of report.idleSleepSafety.diagnostics.checks) if (['inspection.generation', 'local.tracked_work', 'plugins.runtime_drain'].includes(check.reason)) check.state = 'none'; + report.idleSleepSafety.diagnostics.complete = true; expect(classifyEarlyDiagnostic(report, 90_000)).toBe('strict'); +}); + +it('retains observer generation changes and still rejects a second durable report after a transient first', async () => { + const checks = [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog'].map(reason => ({ reason, state: 'none' })), + ]; + const strict = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: true, checks } } }; + const observations: unknown[][] = []; + const server = { settled: async () => ({ idle: { active: 0, generation: 10 }, managed: { apiActivityActive: true } }), + snapshot: async () => ({ idle: { active: 0, generation: 14 }, managed: { apiActivityActive: true } }) } as unknown as Parameters[0]; + const accepted = await stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => structuredClone(strict), persist: async values => { observations.push(structuredClone(values)); } }); + expect(accepted.rounds).toBe(1); expect(observations.at(-1)).toHaveLength(4); + const transient = structuredClone(strict); transient.idleSleepSafety.diagnostics.complete = false; + transient.idleSleepSafety.diagnostics.checks.find(check => check.reason === 'inspection.generation')!.state = 'unknown'; + const blocked = structuredClone(transient); blocked.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' }); + let reads = 0; + await expect(stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => reads++ ? blocked : transient, + persist: async values => { observations.push(structuredClone(values)); } })).rejects.toThrow('chat_endpoints'); + expect(reads).toBe(2); +}); diff --git a/server/src/__tests__/sleep-managed-preflight.test.ts b/server/src/__tests__/sleep-managed-preflight.test.ts new file mode 100644 index 00000000000..eef54f4fa15 --- /dev/null +++ b/server/src/__tests__/sleep-managed-preflight.test.ts @@ -0,0 +1,99 @@ +import { execFileSync } from 'node:child_process'; +import { generateKeyPairSync, randomUUID, sign } from 'node:crypto'; +import { createServer } from 'node:http'; +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js'; +import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { managedSleepConfiguration, managedSleepServerOptions, managedTenantActor, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; +import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js'; + +const requested = process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT === '1'; +const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_TEST_DATABASE_URL'); +if (requested && !databaseUrl) throw new Error('Managed preflight requires an explicit dedicated external *_test URL; no embedded allocation'); +if (requested && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Managed preflight requires retained raw evidence'); +const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT, cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA; +if (requested && (!cloudCheckout || !/^[a-f0-9]{40}$/.test(cloudSha ?? ''))) throw new Error('Managed preflight requires the exact reviewed private Cloud checkout/SHA'); + +describe.skipIf(!requested)('actual managed fixture preflight (no browser, cooldown or quiet claim)', () => { + beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000); + it('provisions through real managed auth and completes a process run before strict activity diagnostics', async () => { + if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Cloud source does not match the reviewed preflight SHA'); + const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href); + const broker = await startQualificationBroker(); const s3 = createMaintenanceS3Boundary(); + const collectorRequests: string[] = []; + const collector = createServer(async (req, res) => { for await (const _chunk of req) { /* Fully consume actual exports. */ } collectorRequests.push(req.url!); res.setHeader('content-type', 'application/json'); res.end('{}'); }); + try { + await withMaintenanceProvider(s3.server, async s3Origin => withMaintenanceProvider(collector, async origin => { + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'managed-preflight', alg: 'EdDSA', use: 'sig' }; + const configuration = managedSleepConfiguration(origin, true); + const server = await startMaintenanceServer(databaseUrl!, { ...managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker), + captureIdleWork: process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_DIAGNOSTICS === '1' }); + const fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const control = (action: string, idleCapabilities?: unknown) => { + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url'), now = Math.floor(Date.now() / 1000); + const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid }); + const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) }); + return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` }; + }; + let failure: { error: unknown } | undefined; + try { + await writeFile(path.join(server.directory, 'managed-preflight-startup.json'), JSON.stringify({ configuration, snapshot: await server.snapshot(), s3: s3.snapshot() }, null, 2)); + expect(s3.snapshot().errors).toEqual([]); + const company = await provisionManagedSleepCompany(server, fixture); + await expect(server.api('POST', '/companies', { name: 'Must remain platform managed' })).rejects.toThrow('cloud_managed'); + // Reuse the combined case's real vault/continuity/enrollment owners. + // The strict broker capability is unavailable before event-wake enrollment. + const github = await server.prepareGitHub(company.id); + await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true }); + const githubContinuity = await server.settleGitHub(github); + await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding: await broker.binding(), boundary: broker.boundary }, null, 2)); + await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1); + const brokerBinding = await broker.binding(); + await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding, boundary: broker.boundary }, null, 2)); + expect(brokerBinding?.instanceId).toBe(broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID); + expect(brokerBinding?.bindings).toEqual([[`${github.grantId}_101`, company.id, managedTenantActor.userId, + 'github.code', '101', github.connectionId, github.grantId]]); + const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Managed preflight process', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', "console.log('managed-preflight-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Actual managed preflight key' }); + const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'One actual managed process run', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` }); + await expect.poll(async () => { const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; return run?.status; }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${issue.id}`; expect(executions.count).toBe(1); + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); const apiSucceededAt = performance.now(); + const activity = await server.snapshot(); + await writeFile(path.join(server.directory, 'managed-preflight-setup.json'), JSON.stringify({ company, githubContinuity, brokerBinding, + agentId: agent.id, keyId: key.id, issueId: issue.id, executions: executions.count, activity, s3: s3.snapshot() }, null, 2)); + expect(activity.managed?.apiActivityActive).toBe(true); + await server.settled(500, 0, { managedSetupRecovery: true }); + const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + try { + const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const read = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + const proof = await stabilizeEarlyDiagnostics(server, { ownerId: drain.ownerId, apiSucceededAt, read, + persist: observations => writeFile(path.join(server.directory, 'managed-preflight-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) }); + await writeFile(path.join(server.directory, 'managed-preflight-receipt.json'), JSON.stringify({ companyId: company.id, proof, s3: s3.snapshot(), collectorRequests, + boundary: 'Actual managed startup/provisioning/key activity/process outcome; no browser, real cooldown/quiet, eligibility, provider suspension or durable S3 claim' }, null, 2)); + } finally { await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop')); } + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + try { await writeFile(path.join(server.directory, 'managed-preflight-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + try { await server.stop(); } catch (error) { errors.push(error); } + try { await fixture.$client.end({ timeout: 1 }); } catch (error) { errors.push(error); } + try { await writeFile(path.join(server.directory, 'managed-preflight-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Managed preflight or owned cleanup failed'); + } + })); + } finally { await broker.stop(); } + }, 180_000); +}); diff --git a/server/src/__tests__/sleep-otel-export.test.ts b/server/src/__tests__/sleep-otel-export.test.ts new file mode 100644 index 00000000000..f8276760850 --- /dev/null +++ b/server/src/__tests__/sleep-otel-export.test.ts @@ -0,0 +1,43 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { createServer } from 'node:http'; +import { it, expect } from 'vitest'; +import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; + +// Opt-in qualification must fail when the SDK falls back to no-op. The ordinary +// bootstrap tests separately retain coverage for the supported missing-peer mode. +it.skipIf(process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE !== '1')('exports an actual named span with no metric or log traffic over 90 real seconds', async () => { + const spans: string[] = [], paths: string[] = []; + const collector = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + paths.push(req.url ?? 'missing-path'); + const envelope = JSON.parse(Buffer.concat(chunks).toString('utf8')); + for (const resource of envelope.resourceSpans ?? []) for (const scope of resource.scopeSpans ?? []) for (const span of scope.spans ?? []) spans.push(span.name); + res.setHeader('Content-Type', 'application/json'); res.end('{}'); + }); + await withMaintenanceProvider(collector, async origin => { + const startedAt = performance.now(); + const { stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url)), '--observe-exporters'], { + env: { PATH: process.env.PATH, NODE_ENV: 'production', OTEL_EXPORTER_OTLP_ENDPOINT: origin, OTEL_EXPORTER_OTLP_PROTOCOL: 'http/json' }, + timeout: 110_000, maxBuffer: 2 * 1024 * 1024, + }); + expect(performance.now() - startedAt).toBeGreaterThanOrEqual(90_000); + expect(stderr).not.toContain('Continuing without tracing'); + expect(spans).toContain('sleep-qualification-stimulus'); + expect(paths).not.toContain('/v1/metrics'); + expect(paths).not.toContain('/v1/logs'); + }); +}, 120_000); + + +it('keeps the optional SDK and exporters inactive without an endpoint', async () => { + const { stdout, stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url))], { + env: { PATH: process.env.PATH, NODE_ENV: 'production' }, timeout: 15_000, maxBuffer: 2 * 1024 * 1024, + }); + expect(stdout).not.toContain('OpenTelemetry service.version='); + expect(stderr).not.toContain('Continuing without tracing'); +}); diff --git a/server/src/__tests__/voice-session-store.integration.test.ts b/server/src/__tests__/voice-session-store.integration.test.ts index 70903f3b786..8c0bb1f1d6c 100644 --- a/server/src/__tests__/voice-session-store.integration.test.ts +++ b/server/src/__tests__/voice-session-store.integration.test.ts @@ -1892,8 +1892,28 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }); it("retains one safe report per call and rejects history after access revocation", async () => { const f = await nativeServiceFixture(), first = await f.service.start(f.input); + const foreign = await nativeServiceFixture(); + const [foreignSession] = await db.select().from(chatVoiceSessions).where(eq(chatVoiceSessions.id, foreign.sessionId)); + await db.insert(chatVoiceReports).values({ companyId: foreign.companyId, sessionId: foreign.sessionId, nextCheckAt: new Date(0) }); + const foreignReports = await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId)); + const reportProviderIds: string[] = []; + const fetchReport = f.transport.callReport as (providerSessionId: string) => ReturnType; + const reports = voiceCallReports(db, session => { + reportProviderIds.push(session.providerSessionId!); + return fetchReport(session.providerSessionId!); + }); + expect(await reports.load({ kind: "company", id: foreign.companyId })).toEqual([ + expect.objectContaining({ id: foreign.sessionId, companyId: foreign.companyId, dueAt: 0 }), + ]); + // This case owns report/history behavior for its company. Global recovery + // also visits every predecessor fixture's live calls, replies and reports. + const reconcileReports = async () => { + const due = (await reports.load({ kind: "company", id: f.companyId })) + .filter(row => row.dueAt <= Date.now()).sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id)); + for (const row of due) await reports.check(row.id); + }; await f.service.end(f.companyId, first.session.id, f.input.caller); - await f.service.reconcile(1000); + await reconcileReports(); const detail = await f.service.callDetail(f.companyId, first.session.id, f.input.caller); expect(detail.report).toMatchObject({ status: "available", costMicroUsd: "120000", transcript: [{ id: "turn1", text: "A test request" }] }); expect(await f.service.history(f.companyId, f.endpointId, f.input.caller)).toEqual(expect.arrayContaining([detail])); @@ -1908,12 +1928,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const baseline = await f.transport.callReport(), latest = new Date(Date.now() + 10000); let response = { ...baseline, costMicroUsd: "240000", providerUpdatedAt: latest }; f.transport.callReport.mockImplementation(async (providerSessionId?: string) => providerSessionId === stored.providerSessionId ? response : baseline); - await f.service.reconcile(1000); + await reconcileReports(); expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000"); expect((await f.service.callDetail(f.companyId, other.session.id, f.input.caller)).report.costMicroUsd).toBe("120000"); await db.update(chatVoiceReports).set({ nextCheckAt: new Date(0) }).where(eq(chatVoiceReports.sessionId, first.session.id)); response = { ...baseline, costMicroUsd: "1", providerUpdatedAt: new Date(0) }; - await f.service.reconcile(1000); + await reconcileReports(); expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000"); expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.sessionId, first.session.id))).toHaveLength(1); await expect(f.service.callDetail(f.companyId, first.session.id, { id: randomUUID(), authority: "member" })).rejects.toMatchObject({ status: 403 }); @@ -1922,6 +1942,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await expect(f.service.history(f.companyId, f.endpointId, f.input.caller)).rejects.toMatchObject({ status: 403 }); await expect(f.service.callDetail(f.companyId, first.session.id, f.input.caller)).rejects.toMatchObject({ status: 403 }); expect(f.transport.createPhoneSession).not.toHaveBeenCalled(); + expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId))).toEqual(foreignReports); + expect(reportProviderIds).toContain(stored.providerSessionId); + expect(reportProviderIds).not.toContain(foreignSession.providerSessionId); }); it("does not create provider media when membership is revoked during credential resolution", async () => { const f = await nativeServiceFixture(); diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index b5aba0a7f0e..4e4ca127631 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -14,6 +14,7 @@ import { agents, companies, createDb, + closeRegisteredClients, executionWorkspaces, heartbeatRuns, issueComments, @@ -87,6 +88,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; const RUNTIME_OWNED_GIT_BRANCH_METADATA = { createdByRuntime: true, @@ -134,8 +136,12 @@ function workspaceBranchIncoherenceFingerprintForTest(input: { } const leasedRunIds = new Set(); -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); -const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const externalRuntimeControlDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_RUNTIME_CONTROL_TEST_DATABASE_URL"); +const embeddedPostgresSupport: Awaited> = externalRuntimeControlDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +// The explicit fixture belongs only to the service-control suite. Other DB +// suites must not allocate an embedded or admin-owned database in this mode. +const describeEmbeddedPostgres = !externalRuntimeControlDatabaseUrl && embeddedPostgresSupport.supported ? describe : describe.skip; +const describeRuntimeControl = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( @@ -6672,16 +6678,18 @@ describeEmbeddedPostgres("workspace dirty quarantine branch repair", () => { }, 20_000); }); -describeEmbeddedPostgres("workspace runtime service control persistence", () => { +describeRuntimeControl("workspace runtime service control persistence", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-"); - db = createDb(tempDb.connectionString); + if (!externalRuntimeControlDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-"); + db = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString, + externalRuntimeControlDatabaseUrl ? { maxConnections: 4 } : undefined); }, 20_000); afterAll(async () => { + if (externalRuntimeControlDatabaseUrl) await closeRegisteredClients(externalRuntimeControlDatabaseUrl, { scope: "database" }); await tempDb?.cleanup(); }); @@ -7284,14 +7292,14 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => return result[0]!; })); - // The two lanes claim ports concurrently in-process. The allocator guarantees - // distinct ports, but not which lane wins the base port. It depends on the - // scheduling order. So assert order-independent invariants, not array index. + // A free-port probe is not a reservation. Both siblings must own distinct + // live ports in the allocator's bounded range even if the preferred port + // becomes unavailable before allocation. const lowerPort = Math.min(first.port!, second.port!); const upperPort = Math.max(first.port!, second.port!); const maxAllocatablePort = basePort + WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS - 1; expect(first.port).not.toBe(second.port); - expect(lowerPort).toBe(basePort); + expect(lowerPort).toBeGreaterThanOrEqual(basePort); expect(upperPort).toBeGreaterThan(basePort); expect(upperPort).toBeLessThanOrEqual(maxAllocatablePort); expect(first.url).toBe(`http://127.0.0.1:${first.port}`); @@ -7333,7 +7341,7 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => it("commits fixed-port reservations across the configured database idle timeout", async () => { const fixture = await createRuntimeFixture(); - const readinessDb = createDb(tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 }); + const readinessDb = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 }); const [initialConnection] = await readinessDb.$client<{ pid: number }[]>`select pg_backend_pid() as pid`; const cleanupRuntimeHome = await createRuntimeHome(); const workspace = fixture.workspaces[0]!; diff --git a/server/src/instrumentation.ts b/server/src/instrumentation.ts index ac632a57e15..d26b09ceae4 100644 --- a/server/src/instrumentation.ts +++ b/server/src/instrumentation.ts @@ -499,6 +499,10 @@ async function bootstrapOtel(endpoint: string): Promise { console.log(`[paperclip] OpenTelemetry service.version=${serviceVersion}`); const sdk = new NodeSDK({ + // This integration exports traces only. NodeSDK otherwise creates + // default OTLP metric/log exporters even when none were configured. + metricReaders: [], + logRecordProcessors: [], resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME || "paperclip", [ATTR_SERVICE_VERSION]: serviceVersion, diff --git a/tests/e2e/disconnect-write-recovery.spec.ts b/tests/e2e/disconnect-write-recovery.spec.ts index 1d7cc5d2fe1..1450aec855d 100644 --- a/tests/e2e/disconnect-write-recovery.spec.ts +++ b/tests/e2e/disconnect-write-recovery.spec.ts @@ -178,8 +178,9 @@ test("replaying an accepted comment does not reopen completed work", async ({ pa const activity = await json(await request.get(`/api/issues/${issue.id}/activity`)); expect(activity.filter((entry: { action: string; details?: { commentId?: string } }) => entry.action === "issue.comment_added" && entry.details?.commentId === first.id)).toHaveLength(1); - await page.goto(`/${company.issuePrefix}/issues/${issue.id}`); - await expect(page.getByRole("heading", { name: issue.title, exact: true })).toBeVisible(); + await page.goto(`/${company.issuePrefix}/issues/${issue.identifier}`); + await expect(page.getByTestId("task-chat-history-content")).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId("issue-detail-header").getByRole("heading", { name: issue.title, exact: true })).toBeVisible(); await expect(page.getByText(body, { exact: true })).toBeVisible(); } finally { await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } }); diff --git a/tests/e2e/task-loading-frame.spec.ts b/tests/e2e/task-loading-frame.spec.ts index f607b245059..e784e61dfc1 100644 --- a/tests/e2e/task-loading-frame.spec.ts +++ b/tests/e2e/task-loading-frame.spec.ts @@ -155,13 +155,14 @@ for (const coordinator of [false, true]) test(`mobile chrome stays fixed when pe let release!: () => void; const preferenceReady = new Promise(resolve => { release = resolve; }); await page.addInitScript(() => { - const frames: number[][] = []; - Object.assign(window, { mobileShellFrames: frames }); + const frames: number[][] = [], frameTimes: number[] = []; + Object.assign(window, { mobileShellFrames: frames, mobileShellFrameTimes: frameTimes }); const sample = () => { const shell = document.querySelector(".paperclip-app-shell"); if (shell && shell.getBoundingClientRect().width > 0) { const rects = [".app-breadcrumb", ".mobile-bottom-nav", "#main-content"].map(selector => document.querySelector(selector)!.getBoundingClientRect()); frames.push([innerWidth, innerHeight, scrollX, scrollY, ...rects.flatMap((rect, index) => index === 2 ? [rect.x, rect.y, rect.width] : [rect.x, rect.y, rect.width, rect.height])]); + frameTimes.push(performance.now()); } requestAnimationFrame(sample); }; @@ -184,9 +185,27 @@ for (const coordinator of [false, true]) test(`mobile chrome stays fixed when pe release(); if (coordinator) await expect(page.getByTestId("task-chat-thread").locator(':scope > [aria-busy="false"]')).toBeVisible(); else await expect(page.getByTestId("issue-detail-header").getByRole("heading", { name: fixture.title, exact: true })).toBeVisible(); - await page.waitForTimeout(300); - const frames = await page.evaluate(() => (window as unknown as { mobileShellFrames: number[][] }).mobileShellFrames); - await info.attach("mobile-preference-chrome", { body: JSON.stringify(frames), contentType: "application/json" }); + const ready = await page.evaluate(() => ({ + count: (window as unknown as { mobileShellFrames: number[][] }).mobileShellFrames.length, + at: performance.now(), + })); + let frames: number[][] = []; + try { + // Observe six real post-ready frames and preserve the original 300ms + // interval. Wall time alone does not establish that rAF sampled it. + await expect.poll(() => page.evaluate(({ count, at }) => { + const state = window as unknown as { mobileShellFrames: number[][]; mobileShellFrameTimes: number[] }; + return state.mobileShellFrames.length >= count + 6 && (state.mobileShellFrameTimes.at(-1) ?? 0) >= at + 300; + }, ready)).toBe(true); + } finally { + const captured = await page.evaluate(() => { + const state = window as unknown as { mobileShellFrames: number[][]; mobileShellFrameTimes: number[] }; + return { frames: state.mobileShellFrames, times: state.mobileShellFrameTimes }; + }); + frames = captured.frames; + await info.attach("mobile-preference-chrome", { body: JSON.stringify(frames), contentType: "application/json" }); + await info.attach("mobile-preference-frame-times", { body: JSON.stringify({ ready, times: captured.times }), contentType: "application/json" }); + } expect(frames.length).toBeGreaterThan(5); for (let dimension = 0; dimension < frames[0].length; dimension++) { const values = frames.map(frame => frame[dimension]);