From ab2d2098a2764c511978d1c3e7a3fc5dfcef07f4 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:41:15 -0600 Subject: [PATCH 01/28] test: verify sleep producer readiness and CI contracts (#18795) Reviewed and verified at 9f87b490d1d25553c49570ba5074c4143b8e3f7d. Co-Authored-By: Paperclip --- doc/observability.md | 3 +- doc/sleep-qualification.md | 244 ++++++++++++++ scripts/sleep-qualification/evidence.d.mts | 54 +++ scripts/sleep-qualification/evidence.mjs | 231 +++++++++++++ scripts/sleep-qualification/setup-otel.mjs | 125 +++++++ server/scripts/sleep-otel-setup.test.mjs | 67 ++++ server/scripts/sleep-qualification.test.mjs | 233 +++++++++++++ .../maintenance/announcement-boundary.ts | 36 ++ .../helpers/maintenance/backup-observation.ts | 54 +++ .../maintenance/browser-observation.ts | 82 +++++ .../maintenance/chat-history-producer.ts | 21 ++ .../maintenance/configuration-producers.ts | 48 +++ .../early-diagnostic-stabilization.ts | 52 +++ .../maintenance/early-sleep-diagnostics.ts | 18 + .../github-configuration-producer.ts | 47 +++ .../maintenance/idle-work-capture-probe.ts | 50 +++ .../helpers/maintenance/idle-work-capture.ts | 137 ++++++++ .../maintenance/incoming-http-observation.ts | 40 +++ .../helpers/maintenance/managed-profile.ts | 34 ++ .../helpers/maintenance/native-browser.ts | 65 ++++ .../helpers/maintenance/otel-child.ts | 10 + .../helpers/maintenance/s3-boundary.ts | 79 +++++ .../helpers/maintenance/server-child.ts | 74 ++++- .../helpers/maintenance/server-harness.ts | 144 ++++++-- .../__tests__/instrumentation-signals.test.ts | 30 ++ .../maintenance-harness-resources.test.ts | 8 + .../sleep-backup-observation.test.ts | 61 ++++ .../sleep-configuration-acceptance.test.ts | 312 ++++++++++++++++++ .../__tests__/sleep-early-diagnostics.test.ts | 15 + .../__tests__/sleep-idle-work-capture.test.ts | 90 +++++ .../sleep-incoming-http-observation.test.ts | 100 ++++++ .../__tests__/sleep-managed-fixture.test.ts | 69 ++++ .../__tests__/sleep-managed-preflight.test.ts | 99 ++++++ .../src/__tests__/sleep-otel-export.test.ts | 43 +++ .../voice-session-store.integration.test.ts | 29 +- .../src/__tests__/workspace-runtime.test.ts | 28 +- server/src/instrumentation.ts | 4 + tests/e2e/disconnect-write-recovery.spec.ts | 5 +- tests/e2e/task-loading-frame.spec.ts | 29 +- 39 files changed, 2823 insertions(+), 47 deletions(-) create mode 100644 doc/sleep-qualification.md create mode 100644 scripts/sleep-qualification/evidence.d.mts create mode 100644 scripts/sleep-qualification/evidence.mjs create mode 100644 scripts/sleep-qualification/setup-otel.mjs create mode 100644 server/scripts/sleep-otel-setup.test.mjs create mode 100644 server/scripts/sleep-qualification.test.mjs create mode 100644 server/src/__tests__/helpers/maintenance/announcement-boundary.ts create mode 100644 server/src/__tests__/helpers/maintenance/backup-observation.ts create mode 100644 server/src/__tests__/helpers/maintenance/browser-observation.ts create mode 100644 server/src/__tests__/helpers/maintenance/chat-history-producer.ts create mode 100644 server/src/__tests__/helpers/maintenance/configuration-producers.ts create mode 100644 server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts create mode 100644 server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts create mode 100644 server/src/__tests__/helpers/maintenance/github-configuration-producer.ts create mode 100644 server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts create mode 100644 server/src/__tests__/helpers/maintenance/idle-work-capture.ts create mode 100644 server/src/__tests__/helpers/maintenance/incoming-http-observation.ts create mode 100644 server/src/__tests__/helpers/maintenance/managed-profile.ts create mode 100644 server/src/__tests__/helpers/maintenance/native-browser.ts create mode 100644 server/src/__tests__/helpers/maintenance/otel-child.ts create mode 100644 server/src/__tests__/helpers/maintenance/s3-boundary.ts create mode 100644 server/src/__tests__/instrumentation-signals.test.ts create mode 100644 server/src/__tests__/sleep-backup-observation.test.ts create mode 100644 server/src/__tests__/sleep-configuration-acceptance.test.ts create mode 100644 server/src/__tests__/sleep-early-diagnostics.test.ts create mode 100644 server/src/__tests__/sleep-idle-work-capture.test.ts create mode 100644 server/src/__tests__/sleep-incoming-http-observation.test.ts create mode 100644 server/src/__tests__/sleep-managed-fixture.test.ts create mode 100644 server/src/__tests__/sleep-managed-preflight.test.ts create mode 100644 server/src/__tests__/sleep-otel-export.test.ts diff --git a/doc/observability.md b/doc/observability.md index 3c0ba327c29..300743ae057 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -210,7 +210,8 @@ diagnostic line on boot and continues without tracing — your server stays up. ## Scope The OpenTelemetry export carries **traces only**. Metrics and log exporters -are out of scope and intentionally not configured here. Auto-instrumentations +are out of scope. The SDK receives explicit empty metric-reader and log-processor +lists so its default OTLP exporters cannot emit unrequested metrics or logs. Auto-instrumentations for `fs`, `dns`, and `net` are disabled by default because they are too chatty for this workload; everything else from `@opentelemetry/auto-instrumentations-node` is on (HTTP, Express, PG, etc.). diff --git a/doc/sleep-qualification.md b/doc/sleep-qualification.md new file mode 100644 index 00000000000..4bbacc05d00 --- /dev/null +++ b/doc/sleep-qualification.md @@ -0,0 +1,244 @@ +# Sleep configuration qualification + +This opt-in harness boots the actual server against an explicitly supplied fresh +test database. It measures local SQL, connection and network quiet. A local pass +does not establish Railway or Neon suspension. Keep a separate verdict for each +layer: quiet, eligibility, database suspension, host suspension, backup persistence, +inbound wake, timed wake and return to idle. + +## Local configured services + +Use a reviewed integrated source containing the incoming-wake and domain-deadline +contracts. Provision and migrate a fresh dedicated PostgreSQL database ending in +`_test` through the normal test operator. The suite does not allocate a database, +start embedded PostgreSQL or delete a database. It preserves its fixture rows. +Never supply a live tenant database. + +The configured exporter needs the exact optional peer versions already declared +in `server/package.json`. With Node 24 and npm available, explicitly install the +fixture peers into a new temporary prefix and link only missing packages in this +checkout: + +```sh +node scripts/sleep-qualification/setup-otel.mjs +``` + +The setup leaves application manifests and lockfiles unchanged. Retain the printed +setup receipt and temporary package lock. A version mismatch fails without +replacing an installed package. An opt-in CI job must run the same setup command; +setting an OTLP endpoint alone does not qualify an exporter. + +The helper records its exact symlink targets in +`server/node_modules/.paperclip-sleep-otel-links.json`. If an owned temporary prefix +disappears, rerunning repairs only those matching recorded symlinks. Existing +matching packages are left alone. An unrecorded dangling link or other entry is +preserved with a recovery error; its pathname is not evidence of ownership. +`node scripts/sleep-qualification/setup-otel.mjs --cleanup` unlinks only matching +owned links. It preserves foreign replacements and keeps temporary install +prefixes for setup evidence. Remove those owned prefixes separately when their +receipts are no longer needed. + +Set `PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE=1`, +`PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL` to the owned migrated fixture and +`PAPERCLIP_SLEEP_CLOUD_SHA` to the exact reviewed Cloud source SHA. Set +`PAPERCLIP_SLEEP_CLOUD_CHECKOUT` to that built private checkout; the suite verifies +its HEAD and imports its production broker adapter without copying private source. +Retain artifacts +with `PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS=1`. Run the two scoped Vitest files with +one worker and no file parallelism: + +```sh +pnpm exec vitest run server/src/__tests__/sleep-otel-export.test.ts \ + server/src/__tests__/sleep-configuration-acceptance.test.ts \ + --maxWorkers=1 --no-file-parallelism +``` + +The first case requires a successful named span at the real local OTLP collector +and no metric or log export during 90 real seconds, beyond the SDK's default +60-second metric cadence. It fails if the production SDK falls back to no-op. +The assembled case keeps +normal automatic backups, announcements, first-party telemetry and OTel enabled. +It creates an agent, useful completed process execution, API credential and future +routine through normal APIs. It waits the real ten-minute authenticated-use +cooldown, then measures at least 90 real seconds without tenant HTTP/SQL observer +polling. A passive listener on the actual app HTTP server counts incoming requests +and WebSocket upgrades, including direct clients outside the harness API helper. +It requires the app's existing request and upgrade handlers before installing; +it cannot become the sole upgrade listener and change unsupported-upgrade handling. +IPC snapshots record its capture identity, installation/validity, cumulative safe-integer +counts and observation times. Both snapshots must enclose the complete quiet interval, +remain valid and yield an exact zero delta. No URL, headers or body are recorded; +snapshot reads themselves make no HTTP requests or SQL queries. Missing or detached +capture, malformed counts and the legacy empty `observerRequests` list cannot prove +quiet. The passive protocol proxy separately measures database effects. It obtains +two owned signed safety reports after the window and preserves fixture history. +After ordinary API reactivation it waits another full real ten-minute cooldown, +measures another 90 seconds of quiet and requests fresh owned eligibility proof. +The helper keeps its legacy defaults for existing maintenance tests. + +Announcements are enabled and the actual route/feed service validates and caches +the known fixture. An owned test-child module wrapper injects the existing feed +transport seam for the exact default HTTPS URL; unmatched requests fail. Startup +fails if the wrapper did not intercept the actual app constructor. The receipt +requires one named fetch and no subsequent calls during quiet. Browser mode also +requires the fixture copy to render. The production DNS/HTTPS address guard stays +unchanged and is not exercised in this combined case; its owning tests cover that +boundary separately. Active user activity can legitimately keep an instance awake. + +The assembled scenario includes real tool approval rejection/delivery retirement +and signed chat-webhook completion followed by ordinary endpoint removal. That +removed-endpoint subcase is separate from the retained enabled GitHub connection: +the latter uses actual encrypted user credentials, refresh/continuity owners, +signed broker bindings and the production event delivery worker. The host wake +capability comes from the production connector store. Local in-memory broker +storage and synthetic GitHub HTTP responses remain explicit external boundaries; +this does not establish deployed broker durability or host suspension. These +assembled paths require a successful run against the reviewed integration before +their receipts can qualify them; source presence alone is not evidence. +Unknown plugins, persistent provider sessions, unfinished delivery/accounting, +uncertain writes and cleanup remain real blockers. Every inventory family needs +an explicit disposition before claiming all configurations qualified. + +## Evaluate a receipt + +```sh +pnpm exec vitest run server/scripts/sleep-qualification.test.mjs --maxWorkers=1 +node scripts/sleep-qualification/evidence.mjs receipt.json expected.json report.json +``` + +`expected.json` binds `appSha`, `cloudSha`, `target` and a SHA-256 `configDigest`. +The digest uses `logicalSleepConfiguration(configuration)`: explicit backup interval +and checkpoint policy, announcements, telemetry activation, OTLP activation/protocol, +and UI activation in a fixed order. It excludes ephemeral loopback ports. Compute +the expected logical contract independently before starting the fixture; the +evaluator recomputes the receipt digest. Keep actual `collectorOrigins.telemetry` +and `.otel` separately, matching the actual configuration and optional expected +origin bindings. + +Configured coverage must also have recorded observations. Announcements require +actual interception, a request for the expected feed, and no unmatched URL. +`signalObservation` records telemetry requests, trace requests, actual OTLP span +records and the named qualification span separately. Metric, log, malformed-trace +and unexpected request counts must be explicitly zero. The assembled case freezes +these observations after each quiet window; feature booleans are derived from +them and the observed backup checkpoint. Missing or malformed counts fail local +qualification even when a feature flag claims success. + +Before the normal cooldown, a signed diagnostic-only inventory saves both raw +reports. It explicitly expects the active API cooldown aggregate and the skipped +backup diagnostic; every other reported blocker must be absent. Aggregate local +probes cannot identify which probe is present, so this early inventory never +proves eligibility. Its hold is released before the full unchanged cooldown. +After quiet, raw ordinary reports are saved before assertions; a refusal also saves +a diagnostic report under that hold. + +Automatic backup coverage requires a new archive from the normal idle checkpoint, +not an enabled flag. After quiet counters freeze, the harness reads every new +owned gzip through its trailer, hashes compressed bytes, checks nonempty SQL for +the fixture identifier without retaining contents, and observes the wake marker +and absence of backup failure markers. This local evidence carries no provider +volume or restore claim. The owning backup suite separately qualifies restoration. +Archive provenance uses a positive finite opened-file birthtime, falling back to +positive finite content mtime only when the filesystem reports birthtime as zero. +The selected timestamp and its source are recorded. Existing names, old positive +birthtimes, malformed timestamps and future chronology remain disqualifying. +Original receipts are retained as produced; the new evidence fields are not +retroactively added to earlier runs. +The historical Full Y at `487e450b` and short composition at `57744f4e` wrote an +unmeasured empty observer-request list. Their recorded SQL, connection, outbound, +browser and producer observations remain pinned to those runs, but that list does +not meet the corrected inbound-observation contract. A new combined qualification +requires both real cooldowns and quiet windows with the installed incoming counter; +a short injected-poll regression cannot retroactively establish observer absence. +Receipt finalization follows all observations and runs the same evaluator before +writing the final receipt. A return-to-idle claim also needs explicit in-run quiet +start/end timestamps spanning at least 90 real seconds after the latest wake; +a duration field alone cannot prove it. +The evaluator rejects mismatched identities and chronology. Missing evidence stays +`INCONCLUSIVE`; a measured failed condition is `NOT VERIFIED`, including an +identity-verified active Neon endpoint or exact Railway service with running deployments. It does not produce +provider proof from local counters, fake time, a registry sleep flag or a successful +control response. Preserve original failures and their raw logs after a retry. + +## Deployed boundary + +No deployment target is authorized by this document. The private Cloud companion +provides a gated read-only provider observer. An operator must first approve one +exact owned disposable staging target, revisions, configuration, provider refs, +fixture operations and finite observation deadline. A named connector or shared +QA workspace is not that authorization. + +Capture app/Cloud identity and effective policy while awake. Use only provider and +central control-plane metadata during sleep; tenant health/API/SQL probes can wake +the target. Confirm the exact Neon endpoint is idle and all owned Railway services +have stopped deployments. Observe ordinary authorized API, broker and deadline +wake cycles separately. Verify useful persisted results and return to idle. +Automatic backup persistence additionally requires an exact attached volume +covering the configured home/backup directory and a fsynced sentinel/archive +continuity receipt across the owned stop/wake. A writable local fallback directory +is insufficient. Missing live access leaves these layers `INCONCLUSIVE`. + +For actual foreground/background browser coverage, first build the shipped UI into +`server/ui-dist` in the same reviewed checkout. Install the normal Playwright +Chromium runtime and run on a display-capable worker. Set +`PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE=1` for the configured suite. The harness loads +the actual UI, records an ordinary active reload, opens another real foreground +tab and requires Chromium to report the app tab hidden. It observes 90 real +seconds without app API traffic, then closes the browser before the server quiet +window. Failure to establish real visibility fails that mode; it never substitutes +a fake visibility event or frozen page. Without this explicit mode, the receipt +claims closed-browser coverage only. + +The browser uses a fresh owned Chromium profile and loopback debugging endpoint. +It attaches with `connectOverCDP({ noDefaults: true })` to the native default +context. Normal Playwright contexts install focus emulation that kept real +background tabs visible in the measured failure. This path avoids that override. +The receipt includes native window IDs, initial/final visibility and passive +visibility events for the entire 90-second interval. An intervening activation +fails the observation. Cleanup disconnects, terminates only the owned process and +removes its temporary profile. It never attaches to a user's existing browser. + + +## Managed fixture preflight + +Configured qualification uses the supported managed authenticated profile. The +actual app runs on loopback, with a lifetime-stable fixture vault key and the +normal managed configuration. A fixture Cloud front door supplies trusted tenant +headers to invoke the real primary-company/user/membership provisioning path; +ordinary company creation remains forbidden. Bearer-key and signed-control +requests omit tenant headers. Native browser headers can reach only the owned +app origin; foreign HTTP/WebSocket transports are blocked and fail qualification. + +Managed application storage requires S3. A bounded loopback protocol fixture +supplies the remote object service for actual SDK requests; unsupported operations, +framing and conditional requests fail closed. This establishes neither provider +IAM nor durable S3 storage. The instance home and database archives remain real +local filesystem paths. Normal managed migration, historical cleanup and routine +owners remain enabled. The logical digest binds managed activation, auth mode and +storage provider; actual ephemeral transport origins are recorded separately. + +Before another long campaign, run the opt-in `sleep-managed-preflight.test.ts` on +its own root-leased fresh external database using +`PAPERCLIP_SLEEP_MANAGED_PREFLIGHT=1` and an explicit dedicated +`PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_TEST_DATABASE_URL`. Retain artifacts with +`PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS=1`. It uses the same profile and negotiated +broker receiver, provisions real authority, admits and completes one process run, +and proves real key activity through signed diagnostics. It performs no browser, +ten-minute wait or quiet-window claim. No embedded allocation or automatic drop. +Loopback execution requires the host tool's explicit network/runtime permission. + +Setup and successful browser receipts are written immediately before later +assertions. Early diagnostics first use the existing owner-settled contract; they +reject durable blockers immediately and retry only observed local in-flight / +generation inspection incompleteness, up to three rounds. Every attempt is saved. +Both accepted reports must be strict, complete, structurally matching and under +the exact live hold, with each internal `inspection.generation=none` and no active +work before/after. Real HTTP authentication itself changes the global work +counter, so equality across deliberate observer requests is not asserted. + +A fresh successful opaque-key request anchors the later full real cooldown. +Neither setup stabilization nor the fast preflight substitutes for both real +600100ms cooldowns, both untouched 90000ms windows and useful reactivation. S3 +observations are saved before and after owned app cleanup, including failures; +cleanup preserves the original error. Full provider sleep remains a separate +unqualified layer until an exact deployed target is authorized and observed. diff --git a/scripts/sleep-qualification/evidence.d.mts b/scripts/sleep-qualification/evidence.d.mts new file mode 100644 index 00000000000..25a091fe9d0 --- /dev/null +++ b/scripts/sleep-qualification/evidence.d.mts @@ -0,0 +1,54 @@ +export interface SleepQualificationIdentity { + appSha: string; + cloudSha: string; + target: string; + configDigest: string; + collectorOrigins?: { telemetry: string; otel: string }; +} +export interface SleepQualificationReport { + version: number; + target: string; + appSha: string; + cloudSha: string; + configDigest: string; + layers: Record; +} +export function logicalSleepConfiguration(configuration: unknown): { + automaticBackups: { intervalMinutes: number; idleCheckpoint: boolean }; + announcements: boolean; + telemetry: boolean; + otel: { enabled: boolean; protocol: 'http/json' }; + serveUi: boolean; + managedInstance: boolean; + deploymentMode: 'authenticated' | 'local_trusted'; + storageProvider: 's3' | 'local_disk'; +}; +export function sleepConfigurationDigest(configuration: unknown): string; +export interface SleepSignalObservation { + telemetryRequests: number; traceRequests: number; exportedSpans: number; qualificationSpans: number; + metricRequests: number; logRequests: number; malformedTraceRequests: number; unexpectedRequests: number; +} +export interface IncomingHttpSnapshot { + source: 'node-http-server-events'; + captureId: string; + installed: boolean; + valid: boolean; + requests: number; + observedAt: string; +} +export interface IncomingHttpWindow { + before: IncomingHttpSnapshot; + after: IncomingHttpSnapshot; + requests: number; +} +export function observeIncomingHttpWindow(before: IncomingHttpSnapshot, after: IncomingHttpSnapshot): IncomingHttpWindow; +export function observedNoIncomingHttp(observation: unknown, quiet: { startedAt: string; finishedAt: string }): boolean; +export function observeConfiguredSignals(requests: ReadonlyArray<{ path: string; body: unknown }>): SleepSignalObservation; +export function configuredSleepFeatures(observations: Record): { + telemetry: boolean; announcements: boolean; automaticBackups: boolean; otel: boolean; exportedSpans: number; +}; +export function evaluateSleepQualification(receipt: unknown, expected: SleepQualificationIdentity): SleepQualificationReport; +export function finalizeLocalSleepReceipt>(observations: T, expected: SleepQualificationIdentity, now?: () => string): { + receipt: T & { finishedAt: string }; + report: SleepQualificationReport; +}; diff --git a/scripts/sleep-qualification/evidence.mjs b/scripts/sleep-qualification/evidence.mjs new file mode 100644 index 00000000000..d19b54fb034 --- /dev/null +++ b/scripts/sleep-qualification/evidence.mjs @@ -0,0 +1,231 @@ +import { createHash } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +const sha = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value); +const finite = value => typeof value === 'number' && Number.isFinite(value) && value >= 0; +const stamp = value => typeof value === 'string' && Number.isFinite(Date.parse(value)); +const layers = ['workerQuiet', 'eligibility', 'databaseSuspension', 'hostSuspension', 'backupPersistence', 'inboundWake', 'timedWake', 'returnToIdle']; +const result = (verdict, reason) => ({ verdict, reason }); +const announcementFeed = 'https://pages.paperclip.ing/announcements/v1/current.json'; +const observedAnnouncement = announcement => announcement?.intercepted === true + && Array.isArray(announcement.requests) && announcement.requests.length > 0 + && announcement.requests.every(url => url === announcementFeed) + && Array.isArray(announcement.unmatched) && announcement.unmatched.length === 0; +const signalKeys = ['telemetryRequests', 'traceRequests', 'exportedSpans', 'qualificationSpans', 'metricRequests', 'logRequests', 'malformedTraceRequests', 'unexpectedRequests']; +const knownSignalCounts = signal => signal && signalKeys.every(key => Number.isSafeInteger(signal[key]) && signal[key] >= 0); + +/** Preserve actual cumulative app-server snapshots; IPC reads create no HTTP traffic. */ +export function observeIncomingHttpWindow(before, after) { + return { before, after, requests: after?.requests - before?.requests }; +} +const knownIncomingHttpSnapshot = snapshot => snapshot?.source === 'node-http-server-events' + && typeof snapshot.captureId === 'string' && /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/.test(snapshot.captureId) + && snapshot.installed === true && snapshot.valid === true + && Number.isSafeInteger(snapshot.requests) && snapshot.requests >= 0 && stamp(snapshot.observedAt); +export const observedNoIncomingHttp = (observation, quiet) => knownIncomingHttpSnapshot(observation?.before) + && knownIncomingHttpSnapshot(observation?.after) && observation.before.captureId === observation.after.captureId + && stamp(quiet?.startedAt) && stamp(quiet?.finishedAt) && Date.parse(quiet.finishedAt) >= Date.parse(quiet.startedAt) + && Date.parse(observation.before.observedAt) <= Date.parse(quiet.startedAt) + && Date.parse(observation.after.observedAt) >= Date.parse(quiet.finishedAt) + && Number.isSafeInteger(observation.requests) && observation.requests === 0 + && observation.after.requests - observation.before.requests === observation.requests; + +/** Count actual collector requests and OTLP span records, never request-as-span estimates. */ +export function observeConfiguredSignals(requests) { + const observed = Object.fromEntries(signalKeys.map(key => [key, 0])); + for (const request of requests) { + if (typeof request?.path !== 'string' || !request.path.startsWith('/') || request.path.startsWith('//')) { observed.unexpectedRequests++; continue; } + const pathname = new URL(request.path, 'http://owned-collector').pathname; + if (pathname === '/telemetry') { observed.telemetryRequests++; continue; } + if (pathname === '/v1/metrics') { observed.metricRequests++; continue; } + if (pathname === '/v1/logs') { observed.logRequests++; continue; } + if (pathname !== '/v1/traces') { observed.unexpectedRequests++; continue; } + observed.traceRequests++; + try { + if (!Array.isArray(request.body?.resourceSpans)) throw new Error('Missing resourceSpans'); + let spans = 0, qualificationSpans = 0; + for (const resource of request.body.resourceSpans) { + if (!Array.isArray(resource?.scopeSpans)) throw new Error('Missing scopeSpans'); + for (const scope of resource.scopeSpans) { + if (!Array.isArray(scope?.spans)) throw new Error('Missing spans'); + for (const span of scope.spans) { + if (typeof span?.name !== 'string' || !span.name) throw new Error('Invalid span'); + spans++; if (span.name === 'sleep-qualification-stimulus') qualificationSpans++; + } + } + } + observed.exportedSpans += spans; observed.qualificationSpans += qualificationSpans; + } catch { observed.malformedTraceRequests++; } + } + return observed; +} + +/** Coverage booleans come from recorded activity; the evaluator also checks their provenance. */ +export function configuredSleepFeatures({ announcement, signalObservation, backupCheckpoint }) { + const known = knownSignalCounts(signalObservation); + return { telemetry: Boolean(known && signalObservation.telemetryRequests > 0), announcements: Boolean(observedAnnouncement(announcement)), + automaticBackups: backupCheckpoint?.owner === 'normal-idle-checkpoint' && backupCheckpoint.wakeMarkerObserved === true + && backupCheckpoint.failureObserved === false && Array.isArray(backupCheckpoint.archives) && backupCheckpoint.archives.length > 0, + otel: Boolean(known && signalObservation.traceRequests > 0 && signalObservation.exportedSpans > 0 + && signalObservation.qualificationSpans > 0 && signalObservation.qualificationSpans <= signalObservation.exportedSpans + && signalObservation.metricRequests === 0 && signalObservation.logRequests === 0 + && signalObservation.malformedTraceRequests === 0 && signalObservation.unexpectedRequests === 0), + exportedSpans: known ? signalObservation.exportedSpans : 0 }; +} +const validBackupCheckpoint = (backup, inRun, quietFinishedAt) => backup?.owner === 'normal-idle-checkpoint' && backup.wakeMarkerObserved === true + && backup.failureObserved === false && inRun(backup.startedAt) && inRun(backup.observedAt) + && Date.parse(backup.startedAt) >= Date.parse(quietFinishedAt) && Date.parse(backup.observedAt) >= Date.parse(backup.startedAt) && Array.isArray(backup.archives) && backup.archives.length > 0 + && backup.archives.every(archive => typeof archive.name === 'string' && /^[^/\\]+\.sql\.gz$/.test(archive.name) + && finite(archive.bytes) && archive.bytes > 0 && finite(archive.sqlBytes) && archive.sqlBytes > 0 + && /^[a-f0-9]{64}$/.test(archive.sha256 ?? '') && archive.gzipVerified === true && archive.fixtureIdentifierFound === true + && ['birthtime', 'mtime'].includes(archive.createdAtSource) + && inRun(archive.createdAt) && Date.parse(archive.createdAt) >= Date.parse(backup.startedAt) && Date.parse(archive.createdAt) <= Date.parse(backup.observedAt)); + +/** Canonical logical configuration: ephemeral collector addresses are bound separately. */ +export function logicalSleepConfiguration(configuration) { + if (!configuration || typeof configuration.announcements !== 'boolean' || typeof configuration.serveUi !== 'boolean' + || typeof configuration.telemetryOrigin !== 'string' || typeof configuration.otelOrigin !== 'string' + || (configuration.managedInstance !== undefined && typeof configuration.managedInstance !== 'boolean') + || (configuration.deploymentMode !== undefined && !['authenticated', 'local_trusted'].includes(configuration.deploymentMode)) + || (configuration.storageProvider !== undefined && !['s3', 'local_disk'].includes(configuration.storageProvider)) + || typeof configuration.automaticBackups?.idleCheckpoint !== 'boolean' + || !finite(configuration.automaticBackups?.intervalMinutes) || configuration.automaticBackups.intervalMinutes < 1) { + throw new Error('sleep_qualification_configuration_invalid'); + } + return { automaticBackups: { intervalMinutes: configuration.automaticBackups.intervalMinutes, + idleCheckpoint: configuration.automaticBackups.idleCheckpoint }, announcements: configuration.announcements, + telemetry: Boolean(configuration.telemetryOrigin), otel: { enabled: Boolean(configuration.otelOrigin), protocol: 'http/json' }, serveUi: configuration.serveUi, managedInstance: configuration.managedInstance ?? false, + deploymentMode: configuration.deploymentMode ?? 'local_trusted', storageProvider: configuration.storageProvider ?? 'local_disk' }; +} +export const sleepConfigurationDigest = configuration => createHash('sha256').update(JSON.stringify(logicalSleepConfiguration(configuration))).digest('hex'); + +/** Finish after observations, then validate the exact serialized local receipt. */ +export function finalizeLocalSleepReceipt(observations, expected, now = () => new Date().toISOString()) { + const receipt = { ...observations, finishedAt: now() }; + const report = evaluateSleepQualification(receipt, expected); + if (report.layers.workerQuiet.verdict !== 'VERIFIED' || report.layers.eligibility.verdict !== 'VERIFIED') { + throw new Error('sleep_qualification_local_evidence_invalid'); + } + return { receipt, report }; +} + +/** Evaluate this campaign's measured boundaries. A local run cannot certify a provider. */ +export function evaluateSleepQualification(receipt, expected) { + if (!receipt || receipt.version !== 1 || !expected || !sha(expected.appSha) + || !sha(expected.cloudSha) || typeof expected.target !== 'string' || !expected.target + || typeof expected.configDigest !== 'string' || !/^[a-f0-9]{64}$/.test(expected.configDigest) + || !stamp(receipt.startedAt) || !stamp(receipt.finishedAt) + || Date.parse(receipt.finishedAt) < Date.parse(receipt.startedAt) + || ['appSha', 'cloudSha', 'target', 'configDigest'].some(key => receipt[key] !== expected[key])) { + throw new Error('sleep_qualification_identity_or_window_invalid'); + } + if (sleepConfigurationDigest(receipt.configuration) !== expected.configDigest) throw new Error('sleep_qualification_configuration_digest_invalid'); + const origins = receipt.collectorOrigins; + if (!origins || origins.telemetry !== receipt.configuration.telemetryOrigin || origins.otel !== receipt.configuration.otelOrigin + || (expected.collectorOrigins && (origins.telemetry !== expected.collectorOrigins.telemetry || origins.otel !== expected.collectorOrigins.otel))) { + throw new Error('sleep_qualification_collector_origins_invalid'); + } + const output = Object.fromEntries(layers.map(layer => [layer, result('INCONCLUSIVE', 'No evidence for this layer')])); + const inRun = value => stamp(value) && Date.parse(value) >= Date.parse(receipt.startedAt) && Date.parse(value) <= Date.parse(receipt.finishedAt); + const quiet = receipt.quiet; + if (quiet) { + const observedFeatures = configuredSleepFeatures(receipt); + const valid = inRun(quiet.startedAt) && inRun(quiet.finishedAt) && finite(quiet.elapsedMs) + && Date.parse(quiet.finishedAt) - Date.parse(quiet.startedAt) >= 90_000 && quiet.elapsedMs >= 90_000 + && quiet.clock === 'real' && finite(quiet.sqlQueries) && finite(quiet.connectionsOpened) + && finite(quiet.connectionsRemaining) && finite(quiet.nonDatabaseRequests) && finite(quiet.nonDatabaseBytes) + && observedNoIncomingHttp(quiet.observerRequests, quiet) + && inRun(quiet.observerRequests.before.observedAt) && inRun(quiet.observerRequests.after.observedAt) + && observedFeatures.telemetry && observedFeatures.announcements && observedFeatures.otel + && Object.entries(observedFeatures).every(([key, value]) => receipt.features?.[key] === value) + && receipt.features?.telemetry === true && receipt.features?.announcements === true + && receipt.features?.automaticBackups === true && validBackupCheckpoint(receipt.backupCheckpoint, inRun, quiet.finishedAt) + && receipt.features?.otel === true && receipt.features?.exportedSpans > 0; + const idle = valid && quiet.sqlQueries === 0 && quiet.connectionsOpened === 0 && quiet.connectionsRemaining === 0 + && quiet.nonDatabaseRequests === 0 && quiet.nonDatabaseBytes === 0; + output.workerQuiet = result(idle ? 'VERIFIED' : 'NOT VERIFIED', idle ? 'Real configured quiet window, including closed DB connections' : 'Configured real quiet window missing, contaminated, or active'); + } + const eligibility = receipt.eligibility; + if (eligibility) { + const valid = inRun(eligibility.observedAt) && eligibility.backgroundWork === 'none' + && eligibility.ownedHold === true && eligibility.unchangedGeneration === true + && finite(eligibility.apiCooldownElapsedMs) && eligibility.apiCooldownElapsedMs >= 600_000; + output.eligibility = result(valid ? 'VERIFIED' : 'NOT VERIFIED', valid ? 'Owned stable safety proof after real API activity cooldown' : 'Safety proof or real API activity cooldown not established'); + } + const provider = receipt.provider; + if (provider) { + if (!provider.live || provider.target !== expected.target || provider.configDigest !== expected.configDigest + || provider.appSha !== expected.appSha || provider.cloudSha !== expected.cloudSha + || !inRun(provider.observedAt)) { + throw new Error('sleep_qualification_provider_identity_invalid'); + } + const untouched = Array.isArray(provider.observerRequests) + && provider.observerRequests.every(entry => entry.boundary === 'provider-control-plane' && (entry.method === 'GET' || (entry.provider === 'railway' && entry.method === 'POST' && entry.operation === 'query-only'))); + const endpoint = provider.neon; + if (untouched && endpoint?.identityVerified === true && endpoint.currentState === 'idle' && endpoint.pendingState === null) { + output.databaseSuspension = result('VERIFIED', 'Exact endpoint idle from live Neon control-plane metadata'); + } + if (untouched && endpoint?.identityVerified === true && (endpoint.currentState === 'active' + || (endpoint.currentState === 'idle' && endpoint.pendingState === 'active'))) { + output.databaseSuspension = result('NOT VERIFIED', 'Exact Neon endpoint is active or becoming active'); + } + const services = provider.railway?.services; + if (untouched && provider.railway?.identityVerified === true && provider.railway.state === 'OBSERVED' && Array.isArray(services) && services.length > 0 + && services.every(service => service.identityVerified === true && service.state === 'OBSERVED' && service.allDeploymentsStopped === true + && typeof service.backupsEnabled === 'boolean' && (service.backupsEnabled === false || service.backupStorageVerified === true))) { + output.hostSuspension = result('VERIFIED', 'Every exact owned service has stopped deployments'); + } + if (untouched && provider.railway?.identityVerified === true && Array.isArray(services) + && services.some(service => service.identityVerified === true && service.state === 'OBSERVED' && service.allDeploymentsStopped === false)) { + output.hostSuspension = result('NOT VERIFIED', 'An exact observed owned service still has running deployments'); + } + const storage = provider.storage; + if (output.hostSuspension.verdict === 'VERIFIED' && storage?.identityVerified === true + && storage.volumeId && storage.beforeVolumeId === storage.volumeId && storage.afterVolumeId === storage.volumeId + && storage.mountPath === '/paperclip' && storage.backupsEnabledBefore === true && storage.backupsEnabledAfter === true + && /^[a-f0-9]{64}$/.test(storage.sentinelBefore ?? '') && storage.sentinelBefore === storage.sentinelAfter + && inRun(storage.beforeAt) && inRun(storage.afterAt) && Date.parse(storage.beforeAt) < Date.parse(provider.observedAt) + && Date.parse(storage.afterAt) > Date.parse(provider.observedAt)) { + output.backupPersistence = result('VERIFIED', 'Attached volume and fsynced sentinel survive owned stop/wake with backups enabled'); + } + } + for (const [layer, kind] of [['inboundWake', 'inbound'], ['timedWake', 'deadline']]) { + const wake = receipt[layer]; + if (!wake) continue; + const sleeping = output.databaseSuspension.verdict === 'VERIFIED' && output.hostSuspension.verdict === 'VERIFIED'; + const chronology = provider && inRun(wake.stimulusAt) && inRun(wake.finishedAt) + && Date.parse(wake.stimulusAt) > Date.parse(provider.observedAt) && Date.parse(wake.finishedAt) >= Date.parse(wake.stimulusAt); + const alarm = kind !== 'deadline' || (inRun(wake.alarmPersistedAt) + && Date.parse(wake.alarmPersistedAt) < Date.parse(provider.observedAt) && wake.sameFinalGeneration === true); + if (!sleeping) { output[layer] = result('INCONCLUSIVE', 'Exact provider sleep and known backup policy have not been established'); continue; } + const verified = sleeping && chronology && alarm && wake.ordinaryProducer === true + && wake.authorizationVerified === true && wake.persistedOutcomeVerified === true && wake.duplicateEffects === 0; + output[layer] = result(verified ? 'VERIFIED' : 'NOT VERIFIED', verified ? `Useful authorized ${kind} wake after provider sleep` : `Missing provider sleep, chronology, authorization or useful ${kind} outcome`); + } + if (receipt.returnToIdle) { + const presentWakes = ['inboundWake', 'timedWake'].filter(layer => Object.hasOwn(receipt, layer)); + const everyWakeInRun = presentWakes.every(layer => inRun(receipt[layer]?.stimulusAt) && inRun(receipt[layer]?.finishedAt) + && Date.parse(receipt[layer].finishedAt) >= Date.parse(receipt[layer].stimulusAt)); + const lastWake = Math.max(...presentWakes.map(layer => Date.parse(receipt[layer]?.finishedAt))); + const idle = receipt.returnToIdle; + const usefulWake = ['inboundWake', 'timedWake'].some(layer => output[layer].verdict === 'VERIFIED'); + const valid = usefulWake && everyWakeInRun && Number.isFinite(lastWake) && inRun(idle.observedAt) && Date.parse(idle.observedAt) > lastWake + && inRun(idle.quietStartedAt) && inRun(idle.quietFinishedAt) + && Date.parse(idle.quietStartedAt) >= lastWake && Date.parse(idle.quietFinishedAt) <= Date.parse(idle.observedAt) + && Date.parse(idle.quietFinishedAt) - Date.parse(idle.quietStartedAt) >= 90_000 + && finite(idle.freshQuietWindowMs) && idle.freshQuietWindowMs >= 90_000 + && idle.freshQuietWindowMs <= Date.parse(idle.quietFinishedAt) - Date.parse(idle.quietStartedAt) && idle.backgroundWork === 'none' && idle.ownedHold === true; + output.returnToIdle = result(valid ? 'VERIFIED' : 'NOT VERIFIED', valid ? 'Fresh quiet owned proof after useful wake' : 'No fresh quiet proof after useful wake'); + } + return { version: 1, target: expected.target, appSha: expected.appSha, cloudSha: expected.cloudSha, configDigest: expected.configDigest, layers: output }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [receiptPath, expectedPath, outputPath] = process.argv.slice(2); + if (!receiptPath || !expectedPath || !outputPath) throw new Error('Usage: node scripts/sleep-qualification/evidence.mjs receipt.json expected.json output.json'); + const report = evaluateSleepQualification(JSON.parse(await readFile(receiptPath, 'utf8')), JSON.parse(await readFile(expectedPath, 'utf8'))); + await writeFile(outputPath, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 }); + process.stdout.write(JSON.stringify(report) + '\n'); + if (Object.values(report.layers).some(layer => layer.verdict === 'NOT VERIFIED')) process.exitCode = 1; +} diff --git a/scripts/sleep-qualification/setup-otel.mjs b/scripts/sleep-qualification/setup-otel.mjs new file mode 100644 index 00000000000..684d3b119fb --- /dev/null +++ b/scripts/sleep-qualification/setup-otel.mjs @@ -0,0 +1,125 @@ +import { lstat, mkdtemp, mkdir, readFile, readlink, rename, rm, symlink, unlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { randomUUID } from 'node:crypto'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +// Explicit opt-in test setup. No manifests, lockfiles or other worktrees change. +const server = fileURLToPath(new URL('../../server/', import.meta.url)); +export const otelPeerNames = ['@opentelemetry/sdk-node', '@opentelemetry/auto-instrumentations-node', '@opentelemetry/exporter-trace-otlp-http', '@opentelemetry/resources', '@opentelemetry/semantic-conventions']; +const ledgerPath = root => path.join(root, 'node_modules', '.paperclip-sleep-otel-links.json'); +async function entryAt(destination) { + try { return await lstat(destination); } catch (error) { if (error.code !== 'ENOENT') throw error; } +} +async function versionAt(root, name) { + try { return JSON.parse(await readFile(path.join(root, 'node_modules', name, 'package.json'), 'utf8')).version; } + catch (error) { if (error.code !== 'ENOENT') throw error; return undefined; } +} +async function readOwnership(root) { + const file = ledgerPath(root), entry = await entryAt(file); + if (!entry) return []; + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error('OTel ownership ledger must be a regular file'); + const record = JSON.parse(await readFile(file, 'utf8')); + if (record.version !== 1 || record.serverDirectory !== root || !Array.isArray(record.links) + || record.links.length > otelPeerNames.length || new Set(record.links.map(link => link.name)).size !== record.links.length + || record.links.some(link => !otelPeerNames.includes(link.name) || typeof link.target !== 'string' + || !path.isAbsolute(link.target) || path.resolve(link.target) !== link.target + || !link.target.endsWith(path.join('node_modules', link.name)))) throw new Error('Invalid OTel ownership ledger'); + return record.links; +} +async function writeOwnership(root, links) { + const file = ledgerPath(root); + if (!links.length) { await rm(file, { force: true }); return; } + await mkdir(path.dirname(file), { recursive: true }); + const temporary = `${file}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify({ version: 1, serverDirectory: root, links }, null, 2), { flag: 'wx' }); + await rename(temporary, file); + } finally { await rm(temporary, { force: true }); } +} +async function ownsLink(destination, record) { + const entry = await entryAt(destination); + return Boolean(record && entry?.isSymbolicLink() + && path.resolve(path.dirname(destination), await readlink(destination)) === record.target); +} +async function installWithNpm(directory) { + await new Promise((resolve, reject) => { + const child = spawn('npm', ['install', '--prefix', directory, '--ignore-scripts', '--no-audit', '--no-fund', '--prefer-offline'], { stdio: 'inherit', shell: false }); + child.once('error', reject); child.once('exit', code => code === 0 ? resolve() : reject(new Error(`OTel fixture installation failed (${code})`))); + }); +} + +/** Only the exact recorded symlink may be removed; install prefixes retain evidence. */ +export async function cleanupSleepOtelLinks({ serverDirectory = server } = {}) { + const root = path.resolve(serverDirectory), records = await readOwnership(root), remaining = [], removed = [], errors = []; + for (const record of records) { + const destination = path.join(root, 'node_modules', record.name); + if (!await entryAt(destination)) continue; + if (!await ownsLink(destination, record)) { + remaining.push(record); errors.push(new Error(`Refusing to clean unrelated ${record.name}`)); continue; + } + await unlink(destination); removed.push(record.name); + } + await writeOwnership(root, remaining); + if (errors.length) throw new AggregateError(errors, 'Some OTel destinations no longer match owned links'); + return { removed, installPrefixes: [...new Set(records.map(record => path.dirname(path.dirname(path.dirname(record.target)))))], boundary: 'Only recorded matching symlinks removed; temporary install prefixes are not deleted' }; +} + +export async function setupSleepOtelPeers({ serverDirectory = server, temporaryRoot = tmpdir(), install = installWithNpm } = {}) { + const root = path.resolve(serverDirectory), manifest = JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8')); + const owned = await readOwnership(root), missing = []; + for (const name of otelPeerNames) { + const exact = manifest.peerDependencies[name]; + if (!/^\d+\.\d+\.\d+$/.test(exact ?? '')) throw new Error(`Exact existing peer version required for ${name}`); + const installed = await versionAt(root, name); + if (installed && installed !== exact) throw new Error(`Installed ${name} does not match ${exact}; refusing to replace it`); + if (installed) continue; + const destination = path.join(root, 'node_modules', name); + if (await entryAt(destination) && !await ownsLink(destination, owned.find(link => link.name === name))) { + throw new Error(`Refusing to replace unrelated ${name}; restore its target or verify ownership before removing the unrecorded entry`); + } + missing.push(name); + } + if (!missing.length) return { linked: [], alreadyInstalled: true }; + const directory = await mkdtemp(path.join(temporaryRoot, 'paperclip-sleep-otel-')); + let linked = 0; + try { + await writeFile(path.join(directory, 'package.json'), JSON.stringify({ private: true, dependencies: Object.fromEntries(otelPeerNames.map(name => [name, manifest.peerDependencies[name]])) }, null, 2)); + await install(directory); + const resolved = Object.fromEntries(await Promise.all(otelPeerNames.map(async name => [name, await versionAt(directory, name)]))); + for (const name of otelPeerNames) if (resolved[name] !== manifest.peerDependencies[name]) throw new Error(`Resolved ${name} differs from pinned peer`); + const records = [...owned]; + for (const name of missing) { + const destination = path.join(root, 'node_modules', name), previous = owned.find(record => record.name === name); + await mkdir(path.dirname(destination), { recursive: true }); + if (await entryAt(destination)) { + if (!await ownsLink(destination, previous)) throw new Error(`OTel destination changed; refusing to replace ${name}`); + await unlink(destination); + } + const target = path.join(directory, 'node_modules', name); + await symlink(target, destination, 'dir'); linked++; + const updated = [...records.filter(record => record.name !== name), { name, target }]; + try { await writeOwnership(root, updated); } + catch (error) { + if (await ownsLink(destination, { name, target })) { await unlink(destination); linked--; } + throw error; + } + records.splice(0, records.length, ...updated); + } + const receipt = { version: 1, directory, packages: resolved, linked: missing, ownershipLedger: ledgerPath(root), cleanup: 'node scripts/sleep-qualification/setup-otel.mjs --cleanup' }; + await writeFile(path.join(directory, 'qualification-setup.json'), JSON.stringify(receipt, null, 2)); + return receipt; + } catch (error) { + // A prefix with published links stays available for explicit owned cleanup. + if (!linked) await rm(directory, { recursive: true, force: true }); + throw error; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const args = process.argv.slice(2); + if (args.length > 1 || (args.length && args[0] !== '--cleanup')) throw new Error('Usage: node scripts/sleep-qualification/setup-otel.mjs [--cleanup]'); + const receipt = args[0] === '--cleanup' ? await cleanupSleepOtelLinks() : await setupSleepOtelPeers(); + process.stdout.write(JSON.stringify(receipt) + '\n'); +} diff --git a/server/scripts/sleep-otel-setup.test.mjs b/server/scripts/sleep-otel-setup.test.mjs new file mode 100644 index 00000000000..ffbedb038f1 --- /dev/null +++ b/server/scripts/sleep-otel-setup.test.mjs @@ -0,0 +1,67 @@ +import { it } from 'vitest'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, readlink, readdir, lstat, rm, symlink, unlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { cleanupSleepOtelLinks, otelPeerNames, setupSleepOtelPeers } from '../../scripts/sleep-qualification/setup-otel.mjs'; + +async function fixture(run) { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-otel-setup-test-')); + const serverDirectory = path.join(directory, 'server'), temporaryRoot = path.join(directory, 'installs'); + await mkdir(serverDirectory); await mkdir(temporaryRoot); + await writeFile(path.join(serverDirectory, 'package.json'), JSON.stringify({ peerDependencies: Object.fromEntries(otelPeerNames.map(name => [name, '1.2.3'])) })); + let installs = 0; + const install = async prefix => { + installs++; + const manifest = JSON.parse(await readFile(path.join(prefix, 'package.json'), 'utf8')); + for (const [name, version] of Object.entries(manifest.dependencies)) { + const packageDirectory = path.join(prefix, 'node_modules', name); + await mkdir(packageDirectory, { recursive: true }); + await writeFile(path.join(packageDirectory, 'package.json'), JSON.stringify({ version })); + } + }; + const options = { serverDirectory, temporaryRoot, install }; + try { await run({ directory, options, install, installs: () => installs, destination: name => path.join(serverDirectory, 'node_modules', name) }); } + finally { await rm(directory, { recursive: true, force: true }); } +} +it('reruns after losing its temporary peers and cleans only its recorded matching links', async () => fixture(async f => { + const first = await setupSleepOtelPeers(f.options); + assert.equal(first.linked.length, 5); assert.equal(f.installs(), 1); + assert.equal((await setupSleepOtelPeers(f.options)).alreadyInstalled, true); assert.equal(f.installs(), 1); + await rm(first.directory, { recursive: true }); + assert.equal((await lstat(f.destination(otelPeerNames[0]))).isSymbolicLink(), true); + const repaired = await setupSleepOtelPeers(f.options); + assert.equal(f.installs(), 2); assert.notEqual(repaired.directory, first.directory); + for (const name of otelPeerNames) assert.equal(await readlink(f.destination(name)), path.join(repaired.directory, 'node_modules', name)); + const cleaned = await cleanupSleepOtelLinks(f.options); assert.equal(cleaned.removed.length, 5); + for (const name of otelPeerNames) await assert.rejects(lstat(f.destination(name)), { code: 'ENOENT' }); + assert.ok(await readFile(path.join(repaired.directory, 'qualification-setup.json'), 'utf8')); // Evidence retained. +})); +it('preserves unowned dangling links, incomplete directories and mismatched installed packages', async () => { + for (const kind of ['dangling', 'directory', 'mismatch']) await fixture(async f => { + const destination = f.destination(otelPeerNames[0]); await mkdir(path.dirname(destination), { recursive: true }); + if (kind === 'dangling') await symlink(path.join(f.directory, 'foreign-missing'), destination); + else { await mkdir(destination); await writeFile(path.join(destination, kind === 'mismatch' ? 'package.json' : 'keep.txt'), kind === 'mismatch' ? JSON.stringify({ version: '9.9.9' }) : 'foreign'); } + await assert.rejects(setupSleepOtelPeers(f.options), /refusing|Refusing/); assert.equal(f.installs(), 0); + if (kind === 'dangling') assert.equal(await readlink(destination), path.join(f.directory, 'foreign-missing')); + else assert.ok(await readFile(path.join(destination, kind === 'mismatch' ? 'package.json' : 'keep.txt'), 'utf8')); + }); +}); +it('keeps healthy unrelated packages and refuses cleanup after an owned link is replaced', async () => fixture(async f => { + const destination = f.destination(otelPeerNames[0]); await mkdir(destination, { recursive: true }); + await writeFile(path.join(destination, 'package.json'), JSON.stringify({ version: '1.2.3' })); + const receipt = await setupSleepOtelPeers(f.options); assert.equal(receipt.linked.length, 4); + const replaced = f.destination(otelPeerNames[1]); await unlink(replaced); + const foreign = path.join(f.directory, 'foreign'); await mkdir(foreign); await writeFile(path.join(foreign, 'keep.txt'), 'foreign'); + await symlink(foreign, replaced); + await assert.rejects(cleanupSleepOtelLinks(f.options), /no longer match/); + assert.equal(await readlink(replaced), foreign); assert.equal(await readFile(path.join(foreign, 'keep.txt'), 'utf8'), 'foreign'); + assert.equal(JSON.parse(await readFile(path.join(destination, 'package.json'), 'utf8')).version, '1.2.3'); +})); +it('preserves an entry introduced during installation and removes an unpublished failed install prefix', async () => fixture(async f => { + await assert.rejects(setupSleepOtelPeers({ ...f.options, install: async prefix => { + await f.install(prefix); const destination = f.destination(otelPeerNames[0]); await mkdir(destination, { recursive: true }); await writeFile(path.join(destination, 'keep.txt'), 'new foreign entry'); + } }), /destination changed/); + assert.equal(await readFile(path.join(f.destination(otelPeerNames[0]), 'keep.txt'), 'utf8'), 'new foreign entry'); + assert.equal((await readdir(f.options.temporaryRoot)).length, 0); +})); diff --git a/server/scripts/sleep-qualification.test.mjs b/server/scripts/sleep-qualification.test.mjs new file mode 100644 index 00000000000..c0a275152a5 --- /dev/null +++ b/server/scripts/sleep-qualification.test.mjs @@ -0,0 +1,233 @@ +import { it as test } from 'vitest'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { configuredSleepFeatures, evaluateSleepQualification, observeConfiguredSignals, observeIncomingHttpWindow, sleepConfigurationDigest, finalizeLocalSleepReceipt } from '../../scripts/sleep-qualification/evidence.mjs'; +const configuration = { automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, telemetryOrigin: 'http://127.0.0.1:4318/telemetry', otelOrigin: 'http://127.0.0.1:4318', serveUi: true }; +// Independent preflight contract, not copied from the receipt/digest implementation. +const configDigest = createHash('sha256').update(JSON.stringify({ automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, telemetry: true, otel: { enabled: true, protocol: 'http/json' }, serveUi: true, managedInstance: false, deploymentMode: 'local_trusted', storageProvider: 'local_disk' })).digest('hex'); +const expected = { appSha: 'a'.repeat(40), cloudSha: 'b'.repeat(40), target: 'owned-fixture', configDigest }; +function fixture() { + return { version: 1, ...expected, configuration: structuredClone(configuration), collectorOrigins: { telemetry: configuration.telemetryOrigin, otel: configuration.otelOrigin }, + announcement: { intercepted: true, requests: ['https://pages.paperclip.ing/announcements/v1/current.json'], unmatched: [] }, + signalObservation: { telemetryRequests: 1, traceRequests: 1, exportedSpans: 1, qualificationSpans: 1, metricRequests: 0, logRequests: 0, malformedTraceRequests: 0, unexpectedRequests: 0 }, + backupCheckpoint: { owner: 'normal-idle-checkpoint', startedAt: '2099-01-01T00:12:31Z', observedAt: '2099-01-01T00:13:00Z', wakeMarkerObserved: true, failureObserved: false, archives: [{ name: 'owned.sql.gz', bytes: 128, sqlBytes: 256, sha256: 'd'.repeat(64), gzipVerified: true, fixtureIdentifierFound: true, createdAt: '2099-01-01T00:12:45Z', createdAtSource: 'birthtime' }] }, startedAt: '2099-01-01T00:00:00Z', finishedAt: '2099-01-01T01:00:00Z', + features: { telemetry: true, announcements: true, automaticBackups: true, otel: true, exportedSpans: 1 }, + quiet: { startedAt: '2099-01-01T00:11:00Z', finishedAt: '2099-01-01T00:12:30Z', elapsedMs: 90_000, clock: 'real', sqlQueries: 0, connectionsOpened: 0, connectionsRemaining: 0, nonDatabaseRequests: 0, nonDatabaseBytes: 0, + observerRequests: observeIncomingHttpWindow( + { source: 'node-http-server-events', captureId: '11111111-1111-4111-8111-111111111111', installed: true, valid: true, requests: 12, observedAt: '2099-01-01T00:11:00Z' }, + { source: 'node-http-server-events', captureId: '11111111-1111-4111-8111-111111111111', installed: true, valid: true, requests: 12, observedAt: '2099-01-01T00:12:30Z' }) }, + eligibility: { observedAt: '2099-01-01T00:13:00Z', backgroundWork: 'none', ownedHold: true, unchangedGeneration: true, apiCooldownElapsedMs: 600_000 } }; +} +function deployed() { + const f = fixture(); + f.provider = { ...expected, live: true, observedAt: '2099-01-01T00:14:00Z', observerRequests: [{ method: 'GET', boundary: 'provider-control-plane' }], + neon: { identityVerified: true, currentState: 'idle', pendingState: null }, + railway: { identityVerified: true, state: 'OBSERVED', services: [{ identityVerified: true, state: 'OBSERVED', backupsEnabled: false, backupStorageVerified: false, allDeploymentsStopped: true }] } }; + return f; +} +test('local configured quiet leaves every provider and wake claim inconclusive', () => { + const r = evaluateSleepQualification(fixture(), expected).layers; + assert.equal(r.workerQuiet.verdict, 'VERIFIED'); assert.equal(r.eligibility.verdict, 'VERIFIED'); + for (const key of ['hostSuspension', 'databaseSuspension', 'backupPersistence', 'inboundWake', 'timedWake']) assert.equal(r[key].verdict, 'INCONCLUSIVE'); +}); +test('wrong source, target and configuration evidence cannot be combined', () => { + for (const key of Object.keys(expected)) { const f = fixture(); f[key] = 'wrong'; assert.throws(() => evaluateSleepQualification(f, expected), /identity/); } + const f = deployed(); f.provider.target = 'foreign'; assert.throws(() => evaluateSleepQualification(f, expected), /provider_identity/); +}); +test('fake time, a no-op exporter, open connections, observer health requests and a shortened cooldown fail', () => { + for (const change of [f => f.quiet.clock = 'qualified', f => f.features.exportedSpans = 0, f => f.quiet.connectionsRemaining = 1, + f => { f.quiet.observerRequests.after.requests++; f.quiet.observerRequests.requests++; }, f => f.quiet.nonDatabaseBytes = 1, f => f.quiet.elapsedMs = 89_999]) { + const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } + const f = fixture(); f.eligibility.apiCooldownElapsedMs = 599_999; assert.equal(evaluateSleepQualification(f, expected).layers.eligibility.verdict, 'NOT VERIFIED'); +}); +test('incoming HTTP observation must be installed, intact and cover the whole window; legacy empty lists are not evidence', () => { + const changes = [f => f.quiet.observerRequests = [], f => delete f.quiet.observerRequests, + f => f.quiet.observerRequests.before.installed = false, f => f.quiet.observerRequests.after.installed = false, + f => f.quiet.observerRequests.before.valid = false, f => f.quiet.observerRequests.after.valid = false, + f => delete f.quiet.observerRequests.before, f => delete f.quiet.observerRequests.after, + f => f.quiet.observerRequests.before.source = 'harness-api-only', f => f.quiet.observerRequests.after.source = 'unknown', + f => f.quiet.observerRequests.after.captureId = '22222222-2222-4222-8222-222222222222', + f => f.quiet.observerRequests.before.captureId = '', + f => f.quiet.observerRequests.before.observedAt = '2099-01-01T00:11:00.001Z', + f => f.quiet.observerRequests.after.observedAt = '2099-01-01T00:12:29.999Z', + f => f.quiet.observerRequests.before.observedAt = '2098-12-31T23:59:59Z', + f => f.quiet.observerRequests.after.observedAt = '2099-01-01T01:00:01Z', + f => f.quiet.observerRequests.after.observedAt = 'invalid', + f => f.quiet.observerRequests.after.requests--, + // A forged zero cannot hide an actual request while SQL and outbound counters remain zero. + f => f.quiet.observerRequests.after.requests++, + f => f.quiet.observerRequests.requests = 1]; + for (const point of ['before', 'after']) { + for (const value of [-1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '12']) { + changes.push(f => f.quiet.observerRequests[point].requests = value); + } + } + for (const value of [-1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '0']) changes.push(f => f.quiet.observerRequests.requests = value); + for (const change of changes) { + const f = fixture(); change(f); + assert.equal(f.quiet.sqlQueries, 0); assert.equal(f.quiet.nonDatabaseRequests, 0); + assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } +}); +test('configured feature claims require complete announcement and integer collector observations', () => { + const changes = [f => delete f.announcement, f => f.announcement.intercepted = false, + f => f.announcement.requests = [], f => f.announcement.requests = ['https://foreign.example/feed'], + f => f.announcement.unmatched.push('https://foreign.example/feed'), f => delete f.announcement.unmatched, + f => delete f.signalObservation, f => f.signalObservation.telemetryRequests = 0, + f => f.signalObservation.traceRequests = 0, f => f.signalObservation.exportedSpans = 0, + f => f.signalObservation.qualificationSpans = 0, f => f.signalObservation.qualificationSpans = 2, + f => f.signalObservation.metricRequests = 1, f => f.signalObservation.logRequests = 1, + f => f.signalObservation.malformedTraceRequests = 1, f => f.signalObservation.unexpectedRequests = 1, + f => f.features.telemetry = false, f => f.features.announcements = false, f => f.features.otel = false]; + for (const key of Object.keys(fixture().signalObservation)) { + changes.push(f => delete f.signalObservation[key]); + for (const value of [-1, 0.5, NaN, Infinity, '1']) changes.push(f => f.signalObservation[key] = value); + } + for (const change of changes) { const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); } +}); +test('assembled receipts derive coverage from actual named spans and requests, including unwanted exporters', () => { + const requests = [{ path: '/telemetry', body: {} }, { path: '/v1/traces', body: { resourceSpans: [{ scopeSpans: [{ spans: [ + { name: 'ordinary-http' }, { name: 'sleep-qualification-stimulus' }, + ] }] }] } }]; + const f = fixture(); f.signalObservation = observeConfiguredSignals(requests); f.features = configuredSleepFeatures(f); + assert.deepEqual(f.signalObservation, { telemetryRequests: 1, traceRequests: 1, exportedSpans: 2, qualificationSpans: 1, + metricRequests: 0, logRequests: 0, malformedTraceRequests: 0, unexpectedRequests: 0 }); + assert.equal(f.features.exportedSpans, 2); // One trace request contained two real span records. + assert.equal(finalizeLocalSleepReceipt(f, expected, () => f.finishedAt).report.layers.workerQuiet.verdict, 'VERIFIED'); + for (const extra of [{ path: '/v1/metrics?batch=1', body: {} }, { path: '/v1/logs', body: {} }, + { path: '/unmatched', body: {} }, { path: '/v1/traces', body: { resourceSpans: [{}] } }]) { + const changed = fixture(); changed.signalObservation = observeConfiguredSignals([...requests, extra]); changed.features = configuredSleepFeatures(changed); + assert.equal(evaluateSleepQualification(changed, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } + const noop = fixture(); noop.signalObservation = observeConfiguredSignals([{ path: '/telemetry', body: {} }, + { path: '/v1/traces', body: { resourceSpans: [{ scopeSpans: [{ spans: [{ name: 'other', attributes: [{ value: 'sleep-qualification-stimulus' }] }] }] }] } }]); + noop.features = configuredSleepFeatures(noop); assert.equal(noop.signalObservation.qualificationSpans, 0); + assert.equal(evaluateSleepQualification(noop, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); +}); +test('registry sleep, incomplete deployment inventory and a failed metadata read cannot prove providers stopped', () => { + const f = deployed(); f.provider.neon = { currentState: 'sleeping', identityVerified: true }; f.provider.railway.services = []; + let r = evaluateSleepQualification(f, expected).layers; assert.equal(r.databaseSuspension.verdict, 'INCONCLUSIVE'); assert.equal(r.hostSuspension.verdict, 'INCONCLUSIVE'); + f.provider.neon = { currentState: 'idle', pendingState: 'active', identityVerified: true }; r = evaluateSleepQualification(f, expected).layers; assert.equal(r.databaseSuspension.verdict, 'NOT VERIFIED'); +}); +test('useful authorized deadline outcome requires a durable alarm before the exact stop proof', () => { + const f = deployed(); f.timedWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0, sameFinalGeneration: true, alarmPersistedAt: '2099-01-01T00:13:30Z' }; + assert.equal(evaluateSleepQualification(f, expected).layers.timedWake.verdict, 'VERIFIED'); + for (const change of [x => x.timedWake.alarmPersistedAt = '2099-01-01T00:15:00Z', x => x.timedWake.sameFinalGeneration = false, x => x.timedWake.persistedOutcomeVerified = false, x => x.timedWake.duplicateEffects = 1]) { + const x = structuredClone(f); change(x); assert.equal(evaluateSleepQualification(x, expected).layers.timedWake.verdict, 'NOT VERIFIED'); + } +}); +test('writable path alone cannot establish backup persistence across stop and wake', () => { + const f = deployed(); f.provider.storage = { identityVerified: true, volumeId: 'volume', beforeVolumeId: 'volume', afterVolumeId: 'volume', mountPath: '/paperclip', backupsEnabledBefore: true, backupsEnabledAfter: true, sentinelBefore: 'd'.repeat(64), sentinelAfter: 'd'.repeat(64), beforeAt: '2099-01-01T00:13:00Z', afterAt: '2099-01-01T00:25:00Z' }; + assert.equal(evaluateSleepQualification(f, expected).layers.backupPersistence.verdict, 'VERIFIED'); + for (const change of [x => delete x.provider.storage.volumeId, x => x.provider.storage.afterVolumeId = 'replacement', x => x.provider.storage.backupsEnabledAfter = false, x => x.provider.storage.sentinelAfter = 'e'.repeat(64)]) { + const x = structuredClone(f); change(x); assert.equal(evaluateSleepQualification(x, expected).layers.backupPersistence.verdict, 'INCONCLUSIVE'); + } +}); + +test('unknown backup policy and unverified enabled storage cannot certify a stopped host or wake', () => { + for (const change of [x => delete x.provider.railway.state, x => x.provider.railway.state = 'INCONCLUSIVE', + x => delete x.provider.railway.services[0].backupsEnabled, x => x.provider.railway.services[0].backupsEnabled = null, + x => x.provider.railway.services[0].state = 'INCONCLUSIVE', x => x.provider.railway.services[0].backupsEnabled = true]) { + const f = deployed(); change(f); + f.inboundWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0 }; + f.timedWake = { ...f.inboundWake, alarmPersistedAt: '2099-01-01T00:13:30Z', sameFinalGeneration: true }; + const r = evaluateSleepQualification(f, expected).layers; + assert.equal(r.hostSuspension.verdict, 'INCONCLUSIVE'); assert.equal(r.inboundWake.verdict, 'INCONCLUSIVE'); assert.equal(r.timedWake.verdict, 'INCONCLUSIVE'); + } + const f = deployed(); assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'VERIFIED'); + f.provider.railway.services[0].backupsEnabled = true; f.provider.railway.services[0].backupStorageVerified = true; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'VERIFIED'); +}); + +function woken() { + const f = deployed(); + f.inboundWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0 }; + f.returnToIdle = { quietStartedAt: '2099-01-01T00:31:00Z', quietFinishedAt: '2099-01-01T00:32:30Z', observedAt: '2099-01-01T00:33:00Z', freshQuietWindowMs: 90_000, backgroundWork: 'none', ownedHold: true }; + return f; +} +test('return to idle requires a real 90-second window after the latest wake', () => { + assert.equal(evaluateSleepQualification(woken(), expected).layers.returnToIdle.verdict, 'VERIFIED'); + for (const finishedAt of ['invalid', '2098-12-31T23:59:59Z', '2099-01-01T01:00:01Z']) { + const f = woken(); f.timedWake = { ...f.inboundWake, finishedAt, alarmPersistedAt: '2099-01-01T00:13:30Z', sameFinalGeneration: true }; + assert.equal(evaluateSleepQualification(f, expected).layers.inboundWake.verdict, 'VERIFIED'); + assert.equal(evaluateSleepQualification(f, expected).layers.returnToIdle.verdict, 'NOT VERIFIED'); + } + + for (const change of [ + f => f.returnToIdle.quietFinishedAt = '2099-01-01T00:31:00.001Z', + f => { f.returnToIdle.quietStartedAt = '2099-01-01T00:19:00Z'; f.returnToIdle.quietFinishedAt = '2099-01-01T00:20:30Z'; }, + f => f.returnToIdle.quietStartedAt = '2098-12-31T23:00:00Z', + f => f.returnToIdle.quietFinishedAt = '2099-01-01T01:00:01Z', + f => f.returnToIdle.quietFinishedAt = '2099-01-01T00:30:00Z', + f => f.returnToIdle.observedAt = '2099-01-01T00:32:00Z', + f => f.returnToIdle.freshQuietWindowMs = Infinity, + f => f.returnToIdle.freshQuietWindowMs = 90_001, + f => delete f.returnToIdle.quietStartedAt, + ]) { + const f = woken(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.returnToIdle.verdict, 'NOT VERIFIED'); + } +}); +test('exact observed active providers are negative evidence while unknown identity stays inconclusive', () => { + for (const state of [{ currentState: 'active', pendingState: null }, { currentState: 'idle', pendingState: 'active' }]) { + const f = deployed(); Object.assign(f.provider.neon, state); + assert.equal(evaluateSleepQualification(f, expected).layers.databaseSuspension.verdict, 'NOT VERIFIED'); + f.provider.neon.identityVerified = false; + assert.equal(evaluateSleepQualification(f, expected).layers.databaseSuspension.verdict, 'INCONCLUSIVE'); + } + const f = deployed(); f.provider.railway.services[0].allDeploymentsStopped = false; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'NOT VERIFIED'); + f.provider.railway.services[0].identityVerified = false; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'INCONCLUSIVE'); +}); + +test('logical digest is stable across origins and detects changed enabled configuration', () => { + assert.equal(sleepConfigurationDigest(configuration), configDigest); + const f = fixture(); f.configuration.telemetryOrigin = 'http://127.0.0.1:5555/telemetry'; f.configuration.otelOrigin = 'http://127.0.0.1:5555'; + f.collectorOrigins = { telemetry: f.configuration.telemetryOrigin, otel: f.configuration.otelOrigin }; + assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'VERIFIED'); + assert.throws(() => evaluateSleepQualification(f, { ...expected, collectorOrigins: fixture().collectorOrigins }), /collector_origins/); + for (const change of [x => x.configuration.announcements = false, x => x.configuration.serveUi = false, + x => x.configuration.automaticBackups.intervalMinutes = 5, x => x.configuration.automaticBackups.idleCheckpoint = false, + x => x.configuration.managedInstance = true, x => x.configuration.deploymentMode = 'authenticated', x => x.configuration.storageProvider = 's3', + x => x.configuration.otelOrigin = '', x => x.configuration.telemetryOrigin = '']) { + const changed = fixture(); change(changed); assert.throws(() => evaluateSleepQualification(changed, expected), /configuration_digest/); + } +}); +test('configured backup feature requires observed verified owned archives and a wake marker', () => { + for (const change of [f => delete f.backupCheckpoint, f => f.backupCheckpoint.archives = [], + f => f.backupCheckpoint.wakeMarkerObserved = false, f => f.backupCheckpoint.failureObserved = true, + f => f.backupCheckpoint.archives[0].gzipVerified = false, f => f.backupCheckpoint.archives[0].fixtureIdentifierFound = false, + f => f.backupCheckpoint.archives[0].bytes = 0, f => f.backupCheckpoint.archives[0].sha256 = 'invalid', + f => f.backupCheckpoint.archives[0].createdAt = '2098-01-01T00:00:00Z', + f => f.backupCheckpoint.archives[0].createdAtSource = 'unknown', + f => f.backupCheckpoint.startedAt = '2099-01-01T00:11:00Z']) { + const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } +}); +test('receipt finalization follows observations and validates the actual resulting receipt', () => { + const f = fixture(); delete f.finishedAt; + let tick = Date.parse(f.eligibility.observedAt); + const { receipt, report } = finalizeLocalSleepReceipt(f, expected, () => new Date(++tick).toISOString()); + assert.ok(Date.parse(receipt.finishedAt) > Date.parse(receipt.eligibility.observedAt)); + assert.equal(report.layers.eligibility.verdict, 'VERIFIED'); + assert.throws(() => finalizeLocalSleepReceipt(f, expected, () => f.startedAt), /local_evidence/); +}); +test('CLI exits one for known active providers and zero for inconclusive identity', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-evidence-cli-')); + try { + const input = path.join(directory, 'receipt.json'), expectedPath = path.join(directory, 'expected.json'), output = path.join(directory, 'report.json'); + await writeFile(expectedPath, JSON.stringify(expected)); + const f = deployed(); f.provider.neon.currentState = 'active'; + for (const [identityVerified, status] of [[true, 1], [false, 0]]) { + f.provider.neon.identityVerified = identityVerified; await writeFile(input, JSON.stringify(f)); + const child = spawnSync(process.execPath, [fileURLToPath(new URL('../../scripts/sleep-qualification/evidence.mjs', import.meta.url)), input, expectedPath, output], { encoding: 'utf8' }); + assert.equal(child.status, status, child.stderr); + } + } finally { await rm(directory, { recursive: true, force: true }); } +}); diff --git a/server/src/__tests__/helpers/maintenance/announcement-boundary.ts b/server/src/__tests__/helpers/maintenance/announcement-boundary.ts new file mode 100644 index 00000000000..44d642fe9c0 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/announcement-boundary.ts @@ -0,0 +1,36 @@ +import { mock } from 'node:test'; +import { DEFAULT_ANNOUNCEMENT_FEED_URL, announcementManifestSchema } from '@paperclipai/shared'; + +/** Test-child transport seam: actual route/feed/cache, no DNS or HTTPS claim. */ +export async function installMaintenanceAnnouncementBoundary() { + const actual = await import('../../../app.js'); + const requests: string[] = [], unmatched: string[] = []; + let intercepted = false; + const manifest = announcementManifestSchema.parse({ schemaVersion: 1, announcement: { + id: 'new-projects', eyebrow: 'New', title: 'Projects', description: 'Organize your work.', + primaryAction: { kind: 'route', label: 'Open', path: '/projects' }, + } }); + const wrapped: typeof actual.createApp = (db, options) => { + const announcements = options.announcements; + if (!announcements?.enabled || announcements.feedUrl !== DEFAULT_ANNOUNCEMENT_FEED_URL) { + throw new Error('Configured announcement boundary requires the exact enabled default feed'); + } + intercepted = true; + const fetch = async (url: URL, init: RequestInit) => { + if (url.href !== DEFAULT_ANNOUNCEMENT_FEED_URL || init.method !== 'GET') { + unmatched.push(url.href); + throw new Error('Unmatched announcement transport is forbidden'); + } + requests.push(url.href); + return Response.json(manifest, { headers: { ETag: '"qualification-v1"' } }); + }; + const withTransport = { ...options, announcements: { ...announcements, fetch } }; + return actual.createApp(db, withTransport); + }; + mock.module(new URL('../../../app.ts', import.meta.url).href, { namedExports: { ...actual, createApp: wrapped } }); + // Fail before server startup if the TS loader/module cache bypasses the mock. + if ((await import('../../../app.js')).createApp !== wrapped) throw new Error('Announcement module boundary did not intercept the startup import'); + return { assertIntercepted() { if (!intercepted) throw new Error('Actual server bypassed the announcement boundary'); }, + snapshot: () => ({ intercepted, requests: [...requests], unmatched: [...unmatched], + boundary: 'Feed transport injected through existing fetch seam; actual route/feed validation/cache/UI. DNS and HTTPS guard are not exercised in this combined case.' }) }; +} diff --git a/server/src/__tests__/helpers/maintenance/backup-observation.ts b/server/src/__tests__/helpers/maintenance/backup-observation.ts new file mode 100644 index 00000000000..c3a273c1346 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/backup-observation.ts @@ -0,0 +1,54 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { lstat, open, readdir } from 'node:fs/promises'; +import path from 'node:path'; +import { Transform, Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { createGunzip } from 'node:zlib'; + +/** Inspect only archives created by the normal idle owner after the quiet counters freeze. + * Stream through EOF to validate the gzip trailer; retain no SQL/credential contents. */ +export async function observeIdleBackupCheckpoint(directory: string, beforeNames: string[], startedAt: string, fixtureId: string) { + if (!/^[a-f0-9-]{36}$/.test(fixtureId)) throw new Error('Invalid owned backup fixture identifier'); + const checkpointMs = Date.parse(startedAt); + if (!Number.isFinite(checkpointMs) || checkpointMs <= 0) throw new Error('Invalid owned checkpoint timestamp'); + const backupDir = path.join(directory, 'backups'); + const marker = await lstat(path.join(backupDir, '.idle-backup-wake-required')); + if (!marker.isFile() || marker.isSymbolicLink() || marker.size === 0) throw new Error('Missing normal idle backup wake marker'); + for (const failure of [path.join(directory, 'health/db-backup-to-s3.failure'), path.join(directory, 'db-backup-to-s3.failure'), path.join(backupDir, 'db-backup-to-s3.failure')]) { + try { await lstat(failure); throw new Error('Observed database backup failure marker'); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + } + const names = (await readdir(backupDir)).filter(name => name.endsWith('.sql.gz') && !beforeNames.includes(name)).sort(); + if (!names.length) throw new Error('Normal idle owner produced no new archive'); + const archives = []; + for (const name of names) { + const handle = await open(path.join(backupDir, name), constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await handle.stat(); + // Some filesystems report birthtime=0. The new-name filter and opened + // non-symlink file still apply; use its content mtime only in that case. + // An old positive birthtime must not be rescued by a recent modification. + const createdAtSource = stat.birthtimeMs === 0 ? 'mtime' : 'birthtime'; + const createdMs = stat.birthtimeMs === 0 ? stat.mtimeMs : stat.birthtimeMs; + if (!stat.isFile() || stat.size <= 0 || !Number.isFinite(createdMs) || createdMs <= 0 + || createdMs < checkpointMs || Math.trunc(createdMs) > Date.now()) throw new Error('Archive predates owned checkpoint or is invalid'); + const hash = createHash('sha256'); let sqlBytes = 0, tail = '', fixtureIdentifierFound = false; + await pipeline(handle.createReadStream({ autoClose: false }), + new Transform({ transform(chunk, _encoding, done) { hash.update(chunk); done(null, chunk); } }), createGunzip(), + new Writable({ write(chunk, _encoding, done) { + sqlBytes += chunk.length; + if (sqlBytes > 512 * 1024 * 1024) { done(new Error('Owned fixture archive exceeded bounded observation size')); return; } + const text = tail + chunk.toString('utf8'); fixtureIdentifierFound ||= text.includes(fixtureId); + tail = text.slice(-fixtureId.length); done(); + } })); + if (!sqlBytes || !fixtureIdentifierFound) throw new Error('Archive does not contain the owned fixture'); + archives.push({ name, bytes: stat.size, sqlBytes, sha256: hash.digest('hex'), createdAt: new Date(createdMs).toISOString(), createdAtSource, gzipVerified: true, fixtureIdentifierFound }); + } finally { await handle.close(); } + } + return { owner: 'normal-idle-checkpoint', startedAt, observedAt: new Date().toISOString(), archives, wakeMarkerObserved: true, failureObserved: false }; +} +export async function idleBackupArchiveNames(directory: string) { + try { return (await readdir(path.join(directory, 'backups'))).filter(name => name.endsWith('.sql.gz')); } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error; } +} diff --git a/server/src/__tests__/helpers/maintenance/browser-observation.ts b/server/src/__tests__/helpers/maintenance/browser-observation.ts new file mode 100644 index 00000000000..76b4488d434 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/browser-observation.ts @@ -0,0 +1,82 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { withNativeMaintenanceBrowser } from './native-browser.js'; + +/** Actual Chromium visibility; never substitutes a fake document or frozen lifecycle. */ +export async function observeMaintenanceBrowser(origin: string, announcementDescription?: string, tenantHeaders?: Record) { + return withNativeMaintenanceBrowser(async (browser, context) => { + const unmatchedOrigins: string[] = []; + if (tenantHeaders) { + // Both HTTP and native WS need the simulated Cloud front door identity. + // Gate every possible foreign transport BEFORE networking; no fixture + // header can reach another origin. about:blank does not use networking. + await context.route('**/*', async route => { + if (new URL(route.request().url()).origin !== origin) { unmatchedOrigins.push(route.request().url()); await route.abort('blockedbyclient'); return; } + await route.continue(); + }); + await context.routeWebSocket(url => url.origin.replace(/^ws/, 'http') !== origin, route => { + unmatchedOrigins.push(route.url()); route.close({ code: 1008, reason: 'Outside owned app origin' }); + }); + await context.setExtraHTTPHeaders(tenantHeaders); + } + const page = await context.newPage(); + const requests: Array<{ at: number; path: string }> = []; + page.on('request', request => { const url = new URL(request.url()); if (url.origin === origin) requests.push({ at: performance.now(), path: url.pathname }); }); + await page.goto(origin, { waitUntil: 'networkidle' }); + if (announcementDescription) await page.getByText(announcementDescription, { exact: true }).waitFor({ state: 'visible' }); + await page.bringToFront(); + if (await page.evaluate(() => document.visibilityState) !== 'visible') throw new Error('Browser active-use observation requires a visible page'); + const beforeActive = requests.length; + await page.reload({ waitUntil: 'networkidle' }); + if (!requests.slice(beforeActive).some(request => request.path.startsWith('/api/'))) throw new Error('Active browser did not make ordinary app requests'); + const visibilityEvents: Array<{ state: string; rendererAt: number }> = []; + const prefix = 'paperclip-sleep-visibility:'; + page.on('console', message => { if (message.text().startsWith(prefix)) visibilityEvents.push(JSON.parse(message.text().slice(prefix.length))); }); + // Passive native events only; never replace document visibility properties. + await page.evaluate(prefix => document.addEventListener('visibilitychange', () => console.log(prefix + JSON.stringify({ state: document.visibilityState, rendererAt: performance.now() }))), prefix); + const visibility = () => page.evaluate(() => ({ state: document.visibilityState, focused: document.hasFocus(), rendererAt: performance.now(), + ownVisibilityOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'visibilityState')), ownHiddenOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hidden')), + ownFocusOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hasFocus')) })); + const initialVisibility = await visibility(); + if (initialVisibility.state !== 'visible') throw new Error('Native browser is not visible before the actual tab switch'); + const foreground = await context.newPage(); + await foreground.goto('about:blank'); await foreground.bringToFront(); + const native = await browser.newBrowserCDPSession(); + const windowIdentity = async (target: typeof page) => { + const session = await context.newCDPSession(target); + try { + const { targetInfo } = await session.send('Target.getTargetInfo'); + const { windowId } = await native.send('Browser.getWindowForTarget', { targetId: targetInfo.targetId }); + return { targetId: targetInfo.targetId, windowId }; + } finally { await session.detach(); } + }; + const appWindow = await windowIdentity(page), foregroundWindow = await windowIdentity(foreground); + if (appWindow.windowId !== foregroundWindow.windowId) throw new Error('Background browser qualification requires an actual same-window tab switch'); + const hiddenDeadline = performance.now() + 5_000; + while (await page.evaluate(() => document.visibilityState) !== 'hidden') { + if (performance.now() > hiddenDeadline) throw new Error('Browser could not establish a real background tab; visibility simulation is not qualification'); + await delay(50); + } + // Let requests already admitted by the transition settle before the untouched window. + await delay(2_000); + const initialHidden = await visibility(); + if (initialHidden.state !== 'hidden') throw new Error('Browser lost native hidden state during settling'); + if (!visibilityEvents.some(event => event.state === 'hidden' && event.rendererAt <= initialHidden.rendererAt)) throw new Error('Native hidden event was not observed'); + const hiddenRequests = requests.length, started = performance.now(); + await delay(90_000); + const finalHidden = await visibility(); + const hiddenRealElapsedMs = performance.now() - started; + if (hiddenRealElapsedMs < 90_000) throw new Error('Native hidden observation was shorter than 90 real seconds'); + const intervalVisibilityEvents = visibilityEvents.filter(event => event.rendererAt >= initialHidden.rendererAt && event.rendererAt <= finalHidden.rendererAt); + if (finalHidden.state !== 'hidden' || intervalVisibilityEvents.some(event => event.state !== 'hidden')) throw new Error(`Browser left native hidden state: ${JSON.stringify({ initialHidden, finalHidden, intervalVisibilityEvents })}`); + if ([initialVisibility, initialHidden, finalHidden].some(state => state.ownVisibilityOverride || state.ownHiddenOverride || state.ownFocusOverride)) throw new Error('Browser visibility or focus properties were overridden'); + const hidden = requests.slice(hiddenRequests); + if (hidden.some(request => request.path.startsWith('/api/'))) throw new Error(`Background tab produced tenant traffic: ${JSON.stringify(hidden)}`); + if (unmatchedOrigins.length) throw new Error(`Browser attempted non-owned transport: ${JSON.stringify(unmatchedOrigins)}`); + await page.bringToFront(); const restoredVisibility = await visibility(); + if (restoredVisibility.state !== 'visible') throw new Error('Native app tab did not become visible after activation'); + if (!visibilityEvents.some(event => event.state === 'visible' && event.rendererAt >= finalHidden.rendererAt)) throw new Error('Native visible restoration event was not observed'); + return { activeRequests: requests.slice(beforeActive, hiddenRequests).length, hiddenRealElapsedMs, + hiddenRequests: hidden, unmatchedOrigins, visibility: 'Native Chromium default context with no Playwright focus override', initialVisibility, initialHidden, finalHidden, + intervalVisibilityEvents, visibilityEvents, restoredVisibility, appWindow, foregroundWindow, announcementRendered: Boolean(announcementDescription), authBoundary: tenantHeaders ? 'Owned Cloud front-door headers; actual tenant resolver/membership' : 'Legacy local board', closed: true }; + }); +} diff --git a/server/src/__tests__/helpers/maintenance/chat-history-producer.ts b/server/src/__tests__/helpers/maintenance/chat-history-producer.ts new file mode 100644 index 00000000000..d2050cc5ba9 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/chat-history-producer.ts @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import type { Db } from '@paperclipai/db'; +import { githubIngressFixture } from '../github-webhook-ingress.js'; +import { chatChannelService } from '../../../services/chat-channels.js'; + +/** The fixture supplies the named external signed-webhook boundary; domain owners write receipts. */ +export async function produceChatHistory(db: Db) { + const fixture = await githubIngressFixture(db); + const action = await fixture.insertAction(); + await fixture.service.processGitHubWebhookIngress(action.id); + const finished = await fixture.readAction(action.id); + assert.equal(fixture.calls.length, 1); + assert.equal(finished.status, 'processed'); + const service = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error('Unexpected chat turn'); } }, + fetch: async () => { throw new Error('Unexpected provider HTTP outside signed-webhook fixture'); } }); + try { + await service.configure(fixture.endpoint.id, { action: 'remove' }, 'local-board'); + } finally { await service.shutdown(); } + return { companyId: fixture.companyId, actionId: action.id, endpointId: fixture.endpoint.id, + boundary: 'Existing signed webhook fixture input; actual chat ingress completion and ordinary endpoint removal preserve history' }; +} diff --git a/server/src/__tests__/helpers/maintenance/configuration-producers.ts b/server/src/__tests__/helpers/maintenance/configuration-producers.ts new file mode 100644 index 00000000000..a67d82912ec --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/configuration-producers.ts @@ -0,0 +1,48 @@ +import type { createDb } from '@paperclipai/db'; +import { expect } from 'vitest'; +import { randomUUID } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { startMaintenanceServer } from './server-harness.js'; + +/** Real routes own request creation, human rejection, run cancellation and outbox retirement. */ +export async function produceCompletedToolReview(server: Awaited>, db: ReturnType, companyId: string) { + await server.api('POST', `/companies/${companyId}/tools/policies`, { name: 'Review harmless qualification writes', policyType: 'require_approval', selectors: { toolName: 'mcp-remote-fixture:update_note' } }); + const readyPath = path.join(server.directory, `tool-review-ready-${randomUUID()}`); + const readyProgram = `const fs=require('node:fs');const marker=${JSON.stringify(readyPath)};fs.writeFileSync(marker+'.pending',process.env.PAPERCLIP_RUN_ID,{flag:'wx'});fs.renameSync(marker+'.pending',marker);setTimeout(()=>process.exit(0),60000)`; + const agent = await server.api<{ id: string }>('POST', `/companies/${companyId}/agents`, { name: 'Owned tool review process', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', readyProgram], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const issue = await server.api<{ id: string }>('POST', `/companies/${companyId}/issues`, { title: 'Harmless tool review history', status: 'todo', assigneeAgentId: agent.id }); + let runId: string | undefined; + await expect.poll(async () => { + const [run] = await db.$client`select id,status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; + runId = run?.id; + if (run?.status !== 'running') return false; + // The process runs only after its awaited adapter.invoke write can commit + // behind the final dispatch gate's run lock. Status alone precedes setup. + try { return await readFile(readyPath, 'utf8') === run.id; } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; } + }, { timeout: 30_000 }).toBe(true); + const readiness = { runId, marker: path.basename(readyPath), observedAt: new Date().toISOString(), source: 'actual-process-run-id-marker' }; + await writeFile(path.join(server.directory, 'sleep-tool-review-readiness.json'), JSON.stringify(readiness, null, 2)); + const session = await server.api<{ sessionId: string; token: string }>('POST', '/tool-gateway/sessions', { companyId, agentId: agent.id, runId, issueId: issue.id }); + const call = await fetch(`http://127.0.0.1:${server.port}/api/tool-gateway/tools/call`, { method: 'POST', signal: AbortSignal.timeout(15_000), + headers: { 'Content-Type': 'application/json', 'x-paperclip-tool-gateway-token': session.token }, + body: JSON.stringify({ tool: 'mcp-remote-fixture:update_note', parameters: { noteId: 'qualification', body: 'Harmless denied operation' } }) }); + expect(call.ok).toBe(false); + expect(JSON.stringify(await call.json())).toContain('approval_required'); + const [request] = await db.$client`select id,status from tool_action_requests where issue_id=${issue.id}`; + expect(request.status).toBe('pending'); + await server.api('POST', `/tool-gateway/action-requests/${request.id}/decline`, { companyId, reason: 'Qualification declines the harmless write' }); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + await expect.poll(async () => { + const [run] = await db.$client`select status from heartbeat_runs where id=${runId!}`; + return ['succeeded', 'failed', 'cancelled', 'timed_out'].includes(run?.status); + }, { timeout: 75_000 }).toBe(true); + await expect.poll(async () => { + const [delivery] = await db.$client`select delivered_at is not null as delivered from tool_action_deliveries where action_request_id=${request.id}`; + return delivery?.delivered; + }, { timeout: 30_000 }).toBe(true); + await server.api('POST', `/tool-gateway/sessions/${session.sessionId}/revoke`, { companyId }); + return { requestId: request.id, issueId: issue.id, runId, readiness, boundary: 'Actual request/rejection/session revocation and closed-task retirement; no provider write executed' }; +} diff --git a/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts b/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts new file mode 100644 index 00000000000..c51c4e337c1 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { assertEarlySleepDiagnostics } from './early-sleep-diagnostics.js'; +import type { startMaintenanceServer } from './server-harness.js'; + +type Report = { ownerId: string; draining: boolean; quiescent: boolean; expiresAt: string; + idleSleepSafety: { diagnostics?: { complete?: boolean; checks?: Array<{ reason: string; state: string }> } } }; +const transientStates = new Map([['inspection.generation', 'unknown'], ['local.tracked_work', 'present'], ['plugins.runtime_drain', 'not_checked']]); + +export function classifyEarlyDiagnostic(report: Report, elapsedSinceApiMs: number): 'strict' | 'transient' { + const checks = report.idleSleepSafety.diagnostics?.checks; + if (!Array.isArray(checks)) throw new Error('Missing early diagnostic checks'); + // A real durable blocker fails now, even if unrelated inspection work is in flight. + const expectedStates = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]); + for (const check of checks) { + if (check.state === (expectedStates.get(check.reason) ?? 'none')) continue; + if (transientStates.get(check.reason) !== check.state) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`); + } + if (checks.some(check => transientStates.get(check.reason) === check.state)) return 'transient'; + assertEarlySleepDiagnostics(report, elapsedSinceApiMs); + return 'strict'; +} + +/** Bounded setup-only observation, never a quiet-window poll or an uncertainty exemption. */ +export async function stabilizeEarlyDiagnostics(server: Awaited>, options: { + ownerId: string; apiSucceededAt: number; read: () => Promise; persist: (observations: unknown[]) => Promise; +}) { + const observations: unknown[] = []; + for (let round = 1; round <= 3; round++) { + const before = await server.settled(); + observations.push({ round, kind: 'settled-before', at: new Date().toISOString(), snapshot: before }); await options.persist(observations); + assert(before.managed?.apiActivityActive, 'Actual successful managed API key activity is not held'); + const reports: Report[] = []; let transient = false; + for (const position of ['initial', 'final']) { + const report = await options.read(); reports.push(report); + observations.push({ round, kind: position, at: new Date().toISOString(), report }); await options.persist(observations); + assert(report.ownerId === options.ownerId && report.draining && report.quiescent && Date.parse(report.expiresAt) > Date.now(), 'Early diagnostic lost exact live hold'); + const classification = classifyEarlyDiagnostic(report, performance.now() - options.apiSucceededAt); + transient = classification === 'transient' || transient; + } + const after = await server.snapshot(); + observations.push({ round, kind: 'snapshot-after', at: new Date().toISOString(), snapshot: after }); await options.persist(observations); + if (transient) continue; + assert(before.idle.active === 0 && after.idle.active === 0, 'Accepted early diagnostic has active owned work'); + // Each real HTTP authentication is tracked work. Its generation changes are + // observer stimuli; each report separately proves its own scan stayed stable. + assert.deepEqual(reports[0]!.idleSleepSafety, reports[1]!.idleSleepSafety, 'Accepted early diagnostic pair differs'); + assert(after.managed?.apiActivityActive, 'Real managed API activity expired during setup'); + return { rounds: round, initial: reports[0]!, final: reports[1]!, observations, + boundary: 'Strict complete stable setup inventory with real API cooldown active; not sleep eligibility' }; + } + throw new Error('Early inspection did not stabilize in three bounded owner-settled rounds'); +} diff --git a/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts b/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts new file mode 100644 index 00000000000..f7bda4c2369 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts @@ -0,0 +1,18 @@ +type Check = { reason: string; state: string }; +/** Early diagnostics are only a fail-fast blocker inventory, never sleep proof. + * The registered-probe aggregate includes the real 10-minute API activity hold; + * it cannot distinguish other probes, so the later full cooldown/proof is mandatory. */ +export function assertEarlySleepDiagnostics(report: { idleSleepSafety?: { diagnostics?: { complete?: boolean; checks?: Check[] } } }, elapsedSinceApiMs: number) { + const diagnostics = report.idleSleepSafety?.diagnostics; + if (!Number.isFinite(elapsedSinceApiMs) || elapsedSinceApiMs < 0 || elapsedSinceApiMs >= 600_000) throw new Error('Early diagnostic requires the actual API cooldown still active'); + if (diagnostics?.complete !== true || !Array.isArray(diagnostics.checks)) throw new Error('Early diagnostic coverage incomplete'); + const expected = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]); + const seen = new Set(); + for (const check of diagnostics.checks) { + if (seen.has(check.reason) || check.state !== (expected.get(check.reason) ?? 'none')) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`); + seen.add(check.reason); + } + for (const required of [...expected.keys(), 'inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog']) { + if (!seen.has(required)) throw new Error(`Missing early sleep diagnostic: ${required}`); + } +} diff --git a/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts b/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts new file mode 100644 index 00000000000..8431d6dbbc4 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts @@ -0,0 +1,47 @@ +import { eq } from 'drizzle-orm'; +import { randomUUID } from 'node:crypto'; +import { toolApplications, toolConnections, connectionGrants, type Db } from '@paperclipai/db'; +import { writeConnectionCredential } from '../../../services/connection-credentials.js'; +import { secretService } from '../../../services/secrets.js'; +import { createOAuthCredentialStore } from '../../../services/tool-access/oauth/credential-store.js'; +import { createOAuthRefreshLeases } from '../../../services/tool-access/oauth/refresh-leases.js'; +import { createOAuthGrantRefresh } from '../../../services/tool-access/oauth/grant-refresh.js'; +import { createGitHubGrantContinuity } from '../../../services/tool-access/github-grant-continuity.js'; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from '../../../services/paperclip-cloud-connector.js'; + +/** Bootstrap the named OAuth consent boundary, then use real vault, binding and continuity owners. */ +export async function prepareGitHubConfiguration(db: Db, companyId: string) { + const owner = 'local-board', now = Date.now(); + const [application] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: 'GitHub qualification', type: 'mcp_http', status: 'active' }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: application.id, uid: randomUUID(), name: 'Retained GitHub', status: 'active', enabled: false, + transport: 'mcp_remote', authKind: 'oauth', credentialPolicy: 'per_user', credentialSource: 'paperclip_vault', transportConfig: {}, + config: { url: 'https://api.github.com/mcp', sourceTemplateKey: 'github', oauth: { strategy: 'paperclip_cloud_connector', connectorProfile: 'github.code' } } }).returning(); + const refs = []; + for (const [configPath, value] of [['oauth.access_token', 'qualification-access'], ['oauth.refresh_token', 'qualification-refresh']] as const) { + const saved = await writeConnectionCredential(db, { companyId, connectionName: 'GitHub', configPath, label: configPath, value, ownerUserId: owner }); + refs.push({ configPath, secretId: saved.secret.id, versionSelector: 'latest' as const, required: true }); + } + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection.id, kind: 'user', subjectUserId: owner, status: 'active', credentialSecretRefs: refs, + providerTenant: { oauth: { strategy: 'paperclip_cloud_connector', accessTokenExpiresAt: new Date(now + 8 * 3_600_000).toISOString(), refreshedAt: new Date(now).toISOString() } } }).returning(); + await createOAuthCredentialStore(db, secretService(db)).syncCredentialBindings(connection, refs); + return { connectionId: connection.id, grantId: grant.id }; +} + +export async function settleGitHubConfiguration(db: Db, input: { connectionId: string; grantId: string }) { + const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, input.connectionId)))[0]; + const connection = await getConnection(); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, input.grantId)); + const config = paperclipCloudConnectorConfigFromEnv(); + if (!config) throw new Error('Owned qualification connector is not configured'); + const connector = createPaperclipCloudConnector({ config }); + const credentials = createOAuthCredentialStore(db, secretService(db)); + const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, credentials.getOAuthGrantRow); + const refresh = createOAuthGrantRefresh(db, { now: () => new Date(), store: credentials, leases, getConnectionRow: getConnection, + currentCloudConnector: () => connector, exchangeOAuthToken: async () => { throw new Error('Unexpected direct OAuth outside managed GitHub fixture'); } }); + const continuity = createGitHubGrantContinuity(db, { now: () => new Date(), currentCloudConnector: () => connector, + refreshOAuthGrantCredentials: refresh, resolveOAuthGrantSecret: credentials.resolveOAuthGrantSecret }); + const settled = await continuity.refreshManagedGitHubGrantAccess(connection, grant, { actorType: 'system', actorId: 'qualification' }); + if (settled.providerTenant?.github?.bindingSync || settled.providerTenant?.github?.continuityRetry) throw new Error('Real GitHub binding debt did not settle'); + return { connectionId: connection.id, grantId: grant.id, credentialRefs: settled.credentialSecretRefs.map(ref => ref.configPath), + boundary: 'Actual encrypted user credential/version/binding owners and GitHub authority refresh; real signed broker binding; named external GitHub HTTP only' }; +} diff --git a/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts b/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts new file mode 100644 index 00000000000..99218823a88 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict'; +import { installMaintenanceIdleWorkCapture } from './idle-work-capture.js'; + +// Isolated Node/TS-loader identity probe. Never imports app/index or opens a DB. +const original = await import('../../../services/task-admission.js'); +const before = process.argv[2] === 'unhandled' ? undefined : original.beginIdleTrackedWork(); +const capture = await installMaintenanceIdleWorkCapture(); +try { + const observed = await import('../../../services/task-admission.js'); + assert.equal(observed.idleWorkSnapshot, original.idleWorkSnapshot); + if (process.argv[2] === 'unhandled') { + const reason = new Error('original unhandled tracked rejection'); + let listener!: (reason: unknown, promise: Promise) => void; + let timer: ReturnType | undefined; + const event = new Promise<{ reason: unknown; promise: Promise }>((resolve, reject) => { + listener = (reason, promise) => resolve({ reason, promise }); + process.once('unhandledRejection', listener); + timer = setTimeout(() => reject(new Error('Original tracked rejection did not reach unhandledRejection')), 1500); + }); + try { + // Deliberately never await or attach a handler to this original promise. + const tracked = observed.trackIdleWork(Promise.reject(reason)); + const unhandled = await event; + assert.equal(unhandled.promise, tracked); + assert.equal(unhandled.reason, reason); + assert.equal(original.idleWorkSnapshot().active, 0); + assert.equal(capture.snapshot().observedActive, 0); + console.log(JSON.stringify({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 })); + } finally { + if (timer) clearTimeout(timer); + process.removeListener('unhandledRejection', listener); + } + } else { + assert.equal(observed.idleWorkSnapshot().active, 1); // Existing state was not reset. + assert.equal(capture.snapshot().observedActive, 0); // The pre-install path is explicitly missing. + before!(); + const finish = observed.beginIdleTrackedWork(); + assert.equal(original.idleWorkSnapshot().active, 1); + finish(); finish(); + assert.equal(original.idleWorkSnapshot().active, 0); + assert.equal(await observed.trackIdleWork(Promise.resolve('original-result')), 'original-result'); + assert.equal(original.idleWorkSnapshot().active, 0); + const result = capture.summary(); + assert(result.snapshot.missingPaths > 0); + console.log(JSON.stringify({ identity: true, originalStatePreserved: true, finalActive: original.idleWorkSnapshot().active, missingPathsExplicit: result.snapshot.missingPaths })); + } +} finally { + before?.(); + capture.dispose(); +} diff --git a/server/src/__tests__/helpers/maintenance/idle-work-capture.ts b/server/src/__tests__/helpers/maintenance/idle-work-capture.ts new file mode 100644 index 00000000000..732d1da4f27 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/idle-work-capture.ts @@ -0,0 +1,137 @@ +import { mock } from 'node:test'; +import { promiseHooks } from 'node:v8'; + +type Admission = typeof import('../../../services/task-admission.js'); +type Entry = { id: number; kind: 'begin' | 'track'; sourceId: number; startedAt: number; generation: number }; +type Completion = Entry & { finishedAt: number; finishedGeneration: number }; +export type IdleWorkCaptureSnapshot = { + actual: { active: number; generation: number }; observedActive: number; active: Entry[]; completionTail: Completion[]; + newSources: Array<{ id: number; callers: string[] }>; overflow: number; sourceOverflow: number; + missingCallsites: number; missingPaths: number; observerErrors: number; disposed: boolean; +}; + +/** Test-only metadata. No original work, promise, drain or counter is changed. */ +export function createMaintenanceIdleWorkCapture(actual: Pick, + controls: { now?: () => number; stack?: () => string | undefined } = {}) { + const active = new Map(), sources = new Map(); + const tail: Completion[] = []; + let trackedPromises = new WeakMap, number>(); + let disposed = false; + let serial = 0, emittedSourceId = 0, overflow = 0, sourceOverflow = 0, missingCallsites = 0, missingPaths = 0, observerErrors = 0; + const now = controls.now ?? Date.now; + function start(kind: Entry['kind']): number | undefined { + try { + if (disposed) return; + if (active.size >= 512) { overflow++; return; } + const callers: string[] = []; + for (const line of (controls.stack?.() ?? new Error().stack ?? '').split('\n')) { + const match = line.match(/[/\\]server[/\\](?:src|dist)[/\\]((?:(?:services|middleware|routes|modules)[/\\][A-Za-z0-9_./\\-]+|(?:app|index))\.[cm]?[jt]s):(\d+):(\d+)/); + if (!match || match[1]!.split(/[/\\]/).includes('..')) continue; + const caller = `${match[1]!.replaceAll('\\', '/')}:${match[2]}:${match[3]}`; + if (!callers.includes(caller)) callers.push(caller); + if (callers.length === 4) break; + } + if (!callers.length) missingCallsites++; + const key = JSON.stringify(callers); let source = sources.get(key); + if (!source) { + if (sources.size >= 512) { sourceOverflow++; return; } + source = { id: sources.size + 1, callers }; sources.set(key, source); + } + const id = ++serial; + active.set(id, { id, kind, sourceId: source.id, startedAt: now(), generation: actual.idleWorkSnapshot().generation }); + return id; + } catch { observerErrors++; return; } + } + // V8 observes settlement without attaching a continuation or marking a rejected + // original promise handled. Unrelated promises only perform a WeakMap lookup. + const stopSettlementHook = promiseHooks.onSettled(promise => { + try { + const id = trackedPromises.get(promise); + if (id === undefined) return; + trackedPromises.delete(promise); + finish(id); + } catch { observerErrors++; } + }); + function finish(id: number | undefined) { + try { + if (id === undefined) return; + const entry = active.get(id); if (!entry) return; + active.delete(id); + tail.push({ ...entry, finishedAt: now(), finishedGeneration: actual.idleWorkSnapshot().generation }); + if (tail.length > 64) tail.shift(); + } catch { observerErrors++; } + } + const beginIdleTrackedWork: Admission['beginIdleTrackedWork'] = () => { + const originalFinish = actual.beginIdleTrackedWork(); const id = start('begin'); + return () => { try { return originalFinish(); } finally { finish(id); } }; + }; + const trackIdleWork: Admission['trackIdleWork'] = (work: Promise) => { + const original = actual.trackIdleWork(work); const id = start('track'); + try { if (id !== undefined) trackedPromises.set(original, id); } catch { observerErrors++; } + return original; + }; + return { beginIdleTrackedWork, trackIdleWork, + dispose() { + if (disposed) return; + disposed = true; + try { stopSettlementHook(); } finally { trackedPromises = new WeakMap(); } + }, + snapshot(): IdleWorkCaptureSnapshot { + const state = actual.idleWorkSnapshot(); + if (state.active !== active.size) missingPaths++; + const newSources = [...sources.values()].filter(source => source.id > emittedSourceId).map(source => ({ ...source, callers: [...source.callers] })); + emittedSourceId = sources.size; + return { actual: state, observedActive: active.size, active: [...active.values()].map(entry => ({ ...entry })), completionTail: tail.map(entry => ({ ...entry })), + newSources, overflow, sourceOverflow, missingCallsites, missingPaths, observerErrors, disposed }; + }, + summary() { return { sourceCatalog: [...sources.values()], snapshot: this.snapshot() }; }, + }; +} + +export async function installMaintenanceIdleWorkCapture() { + const actual = await import('../../../services/task-admission.js'); + const capture = createMaintenanceIdleWorkCapture(actual); + let moduleCapture: ReturnType | undefined; + const disposeCapture = capture.dispose; + capture.dispose = () => { try { disposeCapture(); } finally { const installed = moduleCapture; moduleCapture = undefined; installed?.restore(); } }; + try { + moduleCapture = mock.module(new URL('../../../services/task-admission.ts', import.meta.url).href, { + namedExports: { ...actual, beginIdleTrackedWork: capture.beginIdleTrackedWork, trackIdleWork: capture.trackIdleWork }, + }); + const observed = await import('../../../services/task-admission.js'); + if (observed.beginIdleTrackedWork !== capture.beginIdleTrackedWork || observed.trackIdleWork !== capture.trackIdleWork + || Object.keys(observed).sort().join(',') !== Object.keys(actual).sort().join(',') + || Object.keys(actual).some(key => !['beginIdleTrackedWork', 'trackIdleWork'].includes(key) + && observed[key as keyof Admission] !== actual[key as keyof Admission])) throw new Error('Idle work module capture did not preserve original exports/identity'); + return capture; + } catch (error) { + try { capture.dispose(); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Idle work capture installation and cleanup failed'); } + throw error; + } +} + +/** Reuse existing settle-loop observations. No extra poll, timer, SQL or raw stack per sample. */ +export function createSettledWorkSamples(sourceCatalog: () => Array<{ id: number; callers: string[] }>) { + // Finite longest setup observation at the existing 25ms cadence, plus its + // terminal sample. Overflow still makes diagnostic completion fail closed. + const sampleLimit = Math.ceil(61_000 / 25) + 1; + const samples: Array<{ id: number; at: number; queryTotal: number; queryDelta: number; pending: number; + idle: { active: number; generation: number }; activeIds: number[] }> = []; + let previous: number | undefined, overflowSamples = 0, lastCapture: IdleWorkCaptureSnapshot | undefined; + const startedAt = Date.now(); + return { + sample(snapshot: { now: number; idle: { active: number; generation: number }; workCapture?: IdleWorkCaptureSnapshot }, queryTotal: number, pending: number) { + lastCapture = snapshot.workCapture; + if (samples.length < sampleLimit) samples.push({ id: samples.length + 1, at: snapshot.now, queryTotal, + queryDelta: previous === undefined ? 0 : queryTotal - previous, pending, idle: { ...snapshot.idle }, activeIds: lastCapture?.active.map(entry => entry.id) ?? [] }); + else overflowSamples++; + previous = queryTotal; + }, + report() { return { startedAt, finishedAt: Date.now(), settleDeadlineMs: 30_000, sampleLimit, overflowSamples, + samples, sourceCatalog: sourceCatalog(), lastCapture }; }, + assertComplete() { + if (!lastCapture || overflowSamples || lastCapture.overflow || lastCapture.sourceOverflow || lastCapture.missingCallsites + || lastCapture.missingPaths || lastCapture.observerErrors || lastCapture.disposed) throw new Error('Passive idle work diagnostic capture incomplete'); + }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts b/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts new file mode 100644 index 00000000000..e317165325f --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts @@ -0,0 +1,40 @@ +import { randomUUID } from "node:crypto"; +import type { Server } from "node:http"; +import type { IncomingHttpSnapshot } from "../../../../../scripts/sleep-qualification/evidence.mjs"; + +/** Passive test-child observation of the actual app listener, including direct clients. */ +export function observeMaintenanceIncomingHttp(server: Server) { + // A sole upgrade listener suppresses Node's default unsupported-upgrade close. + // The real app installs its WebSocket handlers before startServer returns. + if (server.listenerCount("request") === 0 || server.listenerCount("upgrade") === 0) { + throw new Error("Maintenance HTTP observation requires existing app request and upgrade handlers"); + } + const captureId = randomUUID(); + let requests = 0, installed = true, valid = true; + const count = () => { + if (requests === Number.MAX_SAFE_INTEGER) valid = false; + else requests++; + }; + const detached = (event: string | symbol, listener: unknown) => { + if ((event === "request" || event === "upgrade") && listener === count) valid = false; + }; + // Count before application handlers without inspecting or retaining request data. + server.prependListener("request", count); + server.prependListener("upgrade", count); + server.on("removeListener", detached); + return { + snapshot(): IncomingHttpSnapshot { + return { source: "node-http-server-events", captureId, installed, + valid: valid && installed && server.listening + && server.listeners("request").includes(count) && server.listeners("upgrade").includes(count), + requests, observedAt: new Date().toISOString() }; + }, + dispose() { + if (!installed) return; + installed = false; valid = false; + server.off("removeListener", detached); + server.off("request", count); + server.off("upgrade", count); + }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/managed-profile.ts b/server/src/__tests__/helpers/maintenance/managed-profile.ts new file mode 100644 index 00000000000..ca8ff553f84 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/managed-profile.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { authUsers, companyMemberships, type Db } from '@paperclipai/db'; +import { and, eq } from 'drizzle-orm'; +import { cloudTenantPrimaryCompanyId } from '../../../services/cloud-instance.js'; +import type { MaintenanceConfiguration, MaintenanceServerOptions, startMaintenanceServer } from './server-harness.js'; + +export const managedTenantActor = { userId: 'local-board', email: 'sleep-owner@example.test', name: 'Owned sleep operator' }; +export function managedSleepConfiguration(collectorOrigin: string, serveUi: boolean): MaintenanceConfiguration { + return { automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, + telemetryOrigin: `${collectorOrigin}/telemetry`, otelOrigin: collectorOrigin, serveUi, + managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' }; +} +export function managedSleepServerOptions(configuration: MaintenanceConfiguration, jwks: string, s3Origin: string, + broker: { origin: string; variables: Record }): MaintenanceServerOptions { + return { configuration, controlJwks: jwks, cloud: { jwks, s3Origin, standby: false }, tenantActor: managedTenantActor, + githubOrigin: broker.origin, connector: { origin: broker.origin, + instanceId: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID, + signPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY, + sealPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY } }; +} +export async function provisionManagedSleepCompany(server: Awaited>, db: Db) { + // This real request invokes resolveCloudTenantActor, never a table seed or a bypassed company-create route. + const companies = await server.api>('GET', '/companies'); + const id = cloudTenantPrimaryCompanyId('maintenance-stack'); + assert(companies.some(company => company.id === id), 'Actual managed primary company was not provisioned'); + const [authority] = await db.select({ userId: authUsers.id, status: companyMemberships.status, membershipRole: companyMemberships.membershipRole }) + .from(authUsers).innerJoin(companyMemberships, and(eq(companyMemberships.principalId, authUsers.id), eq(companyMemberships.principalType, 'user'))) + .where(and(eq(authUsers.id, managedTenantActor.userId), eq(companyMemberships.companyId, id))); + assert.deepEqual(authority, { userId: managedTenantActor.userId, status: 'active', membershipRole: 'owner' }); + await server.api('PATCH', `/companies/${id}`, { defaultResponsibleUserId: managedTenantActor.userId }); + const observation = (await server.snapshot()).managed; + assert(observation?.isCloudManaged && observation.deploymentMode === 'authenticated' && observation.storageProvider === 's3', 'Actual supported managed profile not active'); + return { id, authority, observation, boundary: 'Fixture Cloud front-door headers; actual resolver creates canonical company/user/member/default grants' }; +} diff --git a/server/src/__tests__/helpers/maintenance/native-browser.ts b/server/src/__tests__/helpers/maintenance/native-browser.ts new file mode 100644 index 00000000000..189e7993509 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/native-browser.ts @@ -0,0 +1,65 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { Browser, BrowserContext } from '@playwright/test'; + +/** A fresh owned browser, not a user's profile. Never terminate by process name. */ +export async function withNativeMaintenanceBrowser(run: (browser: Browser, context: BrowserContext) => Promise): Promise { + const { chromium } = await import('@playwright/test'); + const profile = await mkdtemp(path.join(tmpdir(), 'paperclip-sleep-browser-')); + let child: ChildProcess | undefined, browser: Browser | undefined, exited: Promise | undefined; + let exitObserved = false, launchError: Error | undefined, stderr = '', failure: { error: unknown } | undefined; + try { + child = spawn(chromium.executablePath(), [`--user-data-dir=${profile}`, '--remote-debugging-address=127.0.0.1', '--remote-debugging-port=0', + '--no-first-run', '--no-default-browser-check', '--disable-background-networking', '--disable-component-update', '--disable-sync', 'about:blank'], { stdio: ['ignore', 'ignore', 'pipe'] }); + const ownedChild = child; + exited = new Promise(resolve => { + ownedChild.once('exit', () => { exitObserved = true; resolve(); }); + ownedChild.once('error', error => { launchError = error; if (!ownedChild.pid) resolve(); }); + }); + child.stderr?.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-16_384); }); + const deadline = performance.now() + 10_000; + let port: number | undefined; + while (!port) { + if (launchError) throw launchError; + if (child.exitCode !== null || child.signalCode !== null || performance.now() > deadline) throw new Error(`Owned Chromium startup failed: ${stderr}`); + try { + const candidate = Number((await readFile(path.join(profile, 'DevToolsActivePort'), 'utf8')).split('\n')[0]); + if (Number.isInteger(candidate) && candidate > 0 && candidate < 65536) port = candidate; + } catch { /* The owned startup file is not ready yet. */ } + if (!port) await delay(50); + } + // Playwright1.62 installs focus emulation on normal pages. Measured native + // tab switches stayed visible there. The supported noDefaults/default-context + // path avoids installing that override (types.d.ts ConnectOverCDPOptions). + browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`, { noDefaults: true, timeout: 10_000 }); + const context = browser.contexts()[0]; + if (!context) throw new Error('Owned Chromium has no native default context'); + return await run(browser, context); + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + let disconnectTimer: ReturnType | undefined; + try { + await Promise.race([browser?.close(), new Promise((_, reject) => { disconnectTimer = setTimeout(() => reject(new Error('Owned browser disconnect timed out')), 5_000); })]); + } catch (error) { errors.push(error); } + finally { if (disconnectTimer) clearTimeout(disconnectTimer); } + if (child && exited) { + const waitExit = async () => { + let timer: ReturnType | undefined; + try { return await Promise.race([exited!.then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), 5_000); })]); } + finally { if (timer) clearTimeout(timer); } + }; + try { + if (child.exitCode === null && child.signalCode === null && !launchError) child.kill('SIGTERM'); + if (!(await waitExit())) { child.kill('SIGKILL'); if (!(await waitExit())) throw new Error('Owned Chromium did not exit after termination'); } + } catch (error) { errors.push(error); } + } + if (!child?.pid || exitObserved) { + try { await rm(profile, { recursive: true, force: true }); } catch (error) { errors.push(error); } + } else errors.push(new Error(`Owned Chromium exit was not confirmed; profile preserved at ${profile}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Native browser work or cleanup failed'); + } +} diff --git a/server/src/__tests__/helpers/maintenance/otel-child.ts b/server/src/__tests__/helpers/maintenance/otel-child.ts new file mode 100644 index 00000000000..507fa5e529a --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/otel-child.ts @@ -0,0 +1,10 @@ +import { getStartupTracer, instrumentationReady, shutdownInstrumentation } from '../../../instrumentation.js'; +await instrumentationReady; +const span = getStartupTracer('paperclip.sleep-qualification').startSpan('sleep-qualification-stimulus'); +span.end(); +// Keep the real SDK alive beyond its 60s default metric-reader interval. This +// observes unwanted exporter traffic without disabling any exporter through env. +if (process.argv.includes('--observe-exporters')) { + await new Promise(resolve => setTimeout(resolve, 90_000)); +} +await shutdownInstrumentation(); diff --git a/server/src/__tests__/helpers/maintenance/s3-boundary.ts b/server/src/__tests__/helpers/maintenance/s3-boundary.ts new file mode 100644 index 00000000000..dc857f5d187 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/s3-boundary.ts @@ -0,0 +1,79 @@ +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; + +/** Adapted from the real-SDK loopback fixture in managed-cloud-prerequisites.test.ts. + * Small owned objects only; no provider credentials/IAM/durability qualification. */ +export function createMaintenanceS3Boundary() { + const objects = new Map(); + const requests: Array<{ method: string; path: string; status: number }> = []; + const errors: string[] = []; + const xml = (text: string) => text.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>'); + const server = createServer(async (req, res) => { + const url = new URL(req.url!, 'http://fixture.invalid'); + const operation = { method: req.method!, path: url.pathname + url.search, status: 0 }; requests.push(operation); + const reply = (status: number, body?: string | Buffer) => { operation.status = status; res.statusCode = status; res.end(body); }; + try { + if (!url.pathname.startsWith('/maintenance-test/') && url.pathname !== '/maintenance-test') throw new Error('Unknown S3 fixture bucket'); + const key = decodeURIComponent(url.pathname.slice('/maintenance-test/'.length)); + if (req.method === 'GET' && ['/maintenance-test', '/maintenance-test/'].includes(url.pathname) && url.searchParams.get('list-type') === '2' + && [...url.searchParams.keys()].every(key => ['list-type', 'prefix', 'max-keys', 'x-id'].includes(key))) { + const keys = [...objects.keys()].filter(key => key.startsWith(url.searchParams.get('prefix') ?? '')).sort(); + if (keys.length > Math.min(1000, Number(url.searchParams.get('max-keys') ?? 1000))) throw new Error('Listing exceeds bounded single-page fixture'); + res.setHeader('content-type', 'application/xml'); + reply(200, `false${keys.map(key => `${xml(key)}${objects.get(key)!.bytes.length}`).join('')}`); return; + } + if (!key || [...url.searchParams.keys()].some(key => key !== 'x-id')) throw new Error('Unknown S3 fixture operation/query'); + if (req.method === 'PUT') { + const chunks: Buffer[] = []; let length = 0; + for await (const chunk of req) { length += chunk.length; if (length > 17 * 1024 * 1024) throw new Error('S3 fixture object too large'); chunks.push(Buffer.from(chunk)); } + let bytes = Buffer.concat(chunks); + if (req.headers['content-encoding'] === 'aws-chunked') bytes = decodeMaintenanceAwsChunks(bytes); + else if (req.headers['content-encoding']) throw new Error('Unknown S3 fixture content framing'); + const decodedLength = req.headers['x-amz-decoded-content-length']; + if (decodedLength !== undefined && Number(decodedLength) !== bytes.length) throw new Error('S3 decoded length mismatch'); + const previous = objects.get(key); + if ((req.headers['if-none-match'] !== undefined && req.headers['if-none-match'] !== '*') + || (req.headers['if-match'] !== undefined && typeof req.headers['if-match'] !== 'string')) throw new Error('Unknown conditional S3 request'); + if ((req.headers['if-none-match'] === '*' && previous) || (req.headers['if-match'] && previous?.etag !== req.headers['if-match'])) { + res.setHeader('content-type', 'application/xml'); reply(412, 'PreconditionFailed'); return; + } + const total = [...objects.values()].reduce((sum, object) => sum + object.bytes.length, 0) - (previous?.bytes.length ?? 0) + bytes.length; + if (bytes.length > 16 * 1024 * 1024 || total > 256 * 1024 * 1024 || objects.size >= 2048) throw new Error('S3 fixture storage bound exceeded'); + const object = { bytes, etag: `"${createHash('md5').update(bytes).digest('hex')}"`, contentType: String(req.headers['content-type'] ?? 'application/octet-stream') }; + objects.set(key, object); res.setHeader('ETag', object.etag); reply(200); return; + } + if (req.method === 'DELETE') { objects.delete(key); reply(204); return; } + if (req.method !== 'GET' && req.method !== 'HEAD') throw new Error('Unknown S3 fixture method'); + const object = objects.get(key); + if (!object) { res.setHeader('content-type', 'application/xml'); reply(404, req.method === 'GET' ? 'NoSuchKey' : undefined); return; } + if (req.headers.range) throw new Error('Range is outside this small-object fixture contract'); + res.setHeader('ETag', object.etag); res.setHeader('content-type', object.contentType); res.setHeader('content-length', object.bytes.length); + reply(200, req.method === 'GET' ? object.bytes : undefined); + } catch (error) { + errors.push(`${req.method} ${url.pathname + url.search}: ${error instanceof Error ? error.message : String(error)}`); + if (!res.headersSent) { res.setHeader('content-type', 'application/xml'); reply(501, 'NotImplemented'); } else res.destroy(); + } + }); + return { server, snapshot: () => ({ boundary: 'Actual AWS SDK HTTP; bounded in-memory S3 protocol fixture, no IAM/SigV4/provider durability proof', requests: requests.map(request => ({ ...request })), errors: [...errors], + objects: [...objects].map(([key, object]) => ({ key, bytes: object.bytes.length, sha256: createHash('sha256').update(object.bytes).digest('hex') })) }) }; +} + +/** Decode SDK checksum framing without pretending to validate provider signatures. */ +export function decodeMaintenanceAwsChunks(wire: Buffer) { + const chunks: Buffer[] = []; let offset = 0; + while (offset < wire.length) { + const end = wire.indexOf('\r\n', offset); if (end < 0) throw new Error('Malformed AWS chunk header'); + const header = wire.subarray(offset, end).toString('ascii'); + if (!/^[0-9a-f]+$/i.test(header)) throw new Error('Unknown AWS chunk extension'); + const length = Number.parseInt(header, 16); offset = end + 2; + if (!Number.isSafeInteger(length) || length < 0 || offset + length + 2 > wire.length) throw new Error('Truncated AWS chunk'); + if (length === 0) { + const trailers = wire.subarray(offset).toString('ascii'); + if (!/^(?:x-amz-checksum-[a-z0-9]+:[A-Za-z0-9+/=]+\r\n)*\r\n$/.test(trailers)) throw new Error('Unknown AWS checksum trailer'); + return Buffer.concat(chunks); + } + chunks.push(wire.subarray(offset, offset + length)); offset += length; + if (wire.subarray(offset, offset + 2).toString('ascii') !== '\r\n') throw new Error('Malformed AWS chunk terminator'); offset += 2; + } + throw new Error('Missing AWS final chunk'); +} diff --git a/server/src/__tests__/helpers/maintenance/server-child.ts b/server/src/__tests__/helpers/maintenance/server-child.ts index 5569b29d2e9..833726d930e 100644 --- a/server/src/__tests__/helpers/maintenance/server-child.ts +++ b/server/src/__tests__/helpers/maintenance/server-child.ts @@ -2,6 +2,7 @@ import { channel } from "node:diagnostics_channel"; import { mock } from "node:test"; import type { Db } from "@paperclipai/db"; import type { Socket } from "node:net"; +import { observeMaintenanceIncomingHttp } from "./incoming-http-observation.js"; const realTimeout = globalThis.setTimeout, realInterval = globalThis.setInterval, clearRealInterval = globalThis.clearInterval, realNow = Date.now; const fakeClock = process.env.PAPERCLIP_MAINTENANCE_FAKE_CLOCK === "true"; @@ -40,14 +41,42 @@ const observeRequest = () => { outgoingHttpRequests++; }; channel("undici:request:create").subscribe(observeRequest); channel("http.client.request.start").subscribe(observeRequest); +let workCapture: Awaited> | undefined; +let databaseCapture: ReturnType; +let incomingHttp: ReturnType | undefined; try { + // Must precede the announcement wrapper's first import of the real app graph. + if (process.env.PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE === '1') { + workCapture = await (await import('./idle-work-capture.js')).installMaintenanceIdleWorkCapture(); + } + const announcement = process.env.PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE === '1' + ? await (await import('./announcement-boundary.js')).installMaintenanceAnnouncementBoundary() : undefined; const { startServer } = await import("../../../index.js"); const { idleWorkSnapshot, startTaskDrain, stopTaskDrain } = await import("../../../services/task-admission.js"); const { getStartupRecoveryState } = await import("../../../startup-recovery-state.js"); const server = await startServer(); + if (workCapture) databaseCapture = (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv(); + const managedObservation = process.env.PAPERCLIP_MAINTENANCE_MANAGED_TENANT === '1' ? await (async () => { + const { loadConfig } = await import('../../../config.js'); + const { isCloudManagedInstance } = await import('../../../services/cloud-instance.js'); + const { idleApiActivity } = await import('../../../services/idle-api-activity.js'); + const config = loadConfig(); + return () => ({ isCloudManaged: isCloudManagedInstance(), deploymentMode: config.deploymentMode, + storageProvider: config.storageProvider, apiActivityActive: idleApiActivity.active() }); + })() : undefined; const app = server.server.listeners("request")[0] as unknown as { locals: { paperclipDb: Db } }; + try { incomingHttp = observeMaintenanceIncomingHttp(server.server); } + catch (error) { + try { await server.shutdown(); } + catch (cleanupError) { throw new AggregateError([error, cleanupError], "Incoming HTTP observation and owned startup cleanup failed"); } + throw error; + } + async function shutdownServer() { + try { await server.shutdown(); } finally { incomingHttp?.dispose(); } + } + try { announcement?.assertIntercepted(); } catch (error) { await shutdownServer(); throw error; } process.send?.({ kind: "ready", port: server.listenPort }); - process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } }) => { + process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } }) => { try { if (message.command === "snapshot" || message.command === "advance" || message.command === "drain" || message.command === "resume" || message.command === "freeze" || message.command === "realtime") { if (message.command === "freeze" && bootClock) { clearRealInterval(bootClock); bootClock = realInterval(() => mock.timers.tick(0), 10); } @@ -59,7 +88,23 @@ try { if (message.command === "drain") startTaskDrain({ purpose: "idle" }); if (message.command === "resume") stopTaskDrain(); if (fakeClock) mock.timers.tick(0); - process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), now: Date.now() }); + process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), incomingHttp: incomingHttp?.snapshot(), now: Date.now(), announcement: announcement?.snapshot(), managed: managedObservation?.(), workCapture: workCapture?.snapshot() }); + } else if (message.command === "prepareGitHub" || message.command === "settleGitHub") { + const { prepareGitHubConfiguration, settleGitHubConfiguration } = await import("./github-configuration-producer.js"); + const input = message.github; + if (!input) throw new Error("Missing GitHub fixture identity"); + const result = message.command === "prepareGitHub" + ? await prepareGitHubConfiguration(app.locals.paperclipDb, input.companyId!) + : await settleGitHubConfiguration(app.locals.paperclipDb, { connectionId: input.connectionId!, grantId: input.grantId! }); + process.send?.({ id: message.id, kind: "githubConfiguration", result }); + } else if (message.command === "produceChatHistory") { + const { produceChatHistory } = await import("./chat-history-producer.js"); + process.send?.({ id: message.id, kind: "chatHistory", result: await produceChatHistory(app.locals.paperclipDb) }); + } else if (message.command === "trace") { + const { getStartupTracer } = await import("../../../instrumentation.js"); + const span = getStartupTracer("paperclip.sleep-qualification").startSpan("sleep-qualification-stimulus"); + span.end(); + process.send?.({ id: message.id, kind: "traceEnded" }); } else if (message.command === "createOrphanLease") { if (!message.orphanLease) throw new Error("Missing orphan lease fixture"); const { environmentLeases } = await import("@paperclipai/db"); @@ -83,7 +128,20 @@ try { process.send?.({ id: message.id, kind: "published" }); } else if (message.command === "shutdown") { if (fakeClock) runClock(); - await server.shutdown(); + if (workCapture) { + const failures: unknown[] = []; + let before: ReturnType | undefined; + try { before = workCapture.summary(); } catch (error) { failures.push(error); } + try { await shutdownServer(); } catch (error) { failures.unshift(error); } + // A passive flush failure must not replace the original shutdown error. + try { databaseCapture?.flush(); } catch (error) { failures.push(error); } + try { + await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-cleanup.json'), JSON.stringify({ before, after: workCapture.summary() }, null, 2)); + } catch (error) { failures.push(error); } + finally { try { workCapture.dispose(); } catch (error) { failures.push(error); } } + if (failures.length > 1) throw new AggregateError(failures, 'Owned shutdown and passive evidence failed'); + if (failures.length) throw failures[0]; + } else await shutdownServer(); if (bootClock) { clearRealInterval(bootClock); bootClock = undefined; } process.send?.({ id: message.id, kind: "stopped", outgoingHttpRequests }); if (fakeClock) mock.timers.reset(); @@ -93,6 +151,16 @@ try { } catch (error) { process.send?.({ id: message.id, kind: "error", error: error instanceof Error ? error.stack : String(error) }); } }); } catch (error) { + incomingHttp?.dispose(); + if (workCapture) { + try { + // Existing bounded DB capture performs no SQL while flushing metadata. + databaseCapture ??= (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv(); + databaseCapture?.flush(); + await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-startup-failure.json'), JSON.stringify(workCapture.summary(), null, 2)); + } catch { console.warn('Passive startup evidence could not be persisted; original error preserved'); } + finally { try { workCapture.dispose(); } catch { console.warn('Passive capture cleanup failed; original startup error preserved'); } } + } if (bootClock) clearRealInterval(bootClock); process.send?.({ kind: "error", error: error instanceof Error ? error.stack : String(error) }); process.exitCode = 1; diff --git a/server/src/__tests__/helpers/maintenance/server-harness.ts b/server/src/__tests__/helpers/maintenance/server-harness.ts index 922f90292a8..15e1c6ca04e 100644 --- a/server/src/__tests__/helpers/maintenance/server-harness.ts +++ b/server/src/__tests__/helpers/maintenance/server-harness.ts @@ -1,3 +1,5 @@ +import type { IdleWorkCaptureSnapshot } from './idle-work-capture.js'; +import type { IncomingHttpSnapshot } from '../../../../../scripts/sleep-qualification/evidence.mjs'; import { fork } from "node:child_process"; import type { Server as HttpServer } from "node:http"; import { randomBytes, randomUUID } from "node:crypto"; @@ -42,9 +44,54 @@ export async function withMaintenanceProvider(provider: HttpServer, run: (ori } } -type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number }; +type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number; + incomingHttp: IncomingHttpSnapshot; + managed?: { isCloudManaged: boolean; deploymentMode: string; storageProvider: string; apiActivityActive: boolean }; + workCapture?: IdleWorkCaptureSnapshot; + announcement?: { intercepted: boolean; requests: string[]; unmatched: string[]; boundary: string } }; -export async function startMaintenanceServer(databaseUrl: string, options: { artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string }; cloud?: { jwks: string; s3Origin: string; standby: boolean } } = {}) { +export interface MaintenanceConfiguration { + automaticBackups?: { intervalMinutes: number; idleCheckpoint: boolean }; + announcements?: boolean; + telemetryOrigin?: string; + otelOrigin?: string; + serveUi?: boolean; + managedInstance?: boolean; + deploymentMode?: "local_trusted" | "authenticated"; + storageProvider?: "local_disk" | "s3"; +} +export interface MaintenanceServerOptions { + artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; + connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string; instanceId?: string }; + cloud?: { jwks: string; s3Origin: string; standby: boolean }; + configuration?: MaintenanceConfiguration; + controlJwks?: string; + captureIdleWork?: boolean; + tenantActor?: { userId: string; email: string; name: string }; +} +export async function startMaintenanceServer(databaseUrl: string, options: MaintenanceServerOptions = {}) { + const configuration = options.configuration; + if (options.captureIdleWork && options.clock) throw new Error("Passive setup diagnostics require the real clock"); + if (options.tenantActor && (!options.cloud || !configuration?.managedInstance || configuration.deploymentMode !== 'authenticated' || configuration.storageProvider !== 's3')) { + throw new Error('Managed tenant fixture requires the supported authenticated S3 profile'); + } + const tenantHeaders: Record = options.tenantActor ? { + 'x-paperclip-cloud-tenant-token': 'maintenance-tenant-token', 'x-paperclip-cloud-stack-id': 'maintenance-stack', + 'x-paperclip-cloud-stack-role': 'owner', 'x-paperclip-cloud-user-id': options.tenantActor.userId, + 'x-paperclip-cloud-user-email': options.tenantActor.email, 'x-paperclip-cloud-user-name': options.tenantActor.name, + 'x-paperclip-cloud-paperclip-company-name': 'Configured sleep acceptance', + } : {}; + for (const origin of [configuration?.telemetryOrigin, configuration?.otelOrigin]) { + if (!origin) continue; + const endpoint = new URL(origin); + if (endpoint.protocol !== "http:" || endpoint.hostname !== "127.0.0.1" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) { + throw new Error("Maintenance collectors must be explicit loopback HTTP endpoints"); + } + } + if (configuration?.automaticBackups && (!Number.isInteger(configuration.automaticBackups.intervalMinutes) || configuration.automaticBackups.intervalMinutes < 1)) { + throw new Error("Maintenance backup interval must be a positive integer"); + } + if (configuration && options.clock) throw new Error("Configured acceptance requires the real clock"); const artifacts = await createMaintenanceArtifacts(options.artifactRoot); const { directory, retainedDirectory } = artifacts; const reservation = createServer(); @@ -72,43 +119,57 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art const configPath = path.join(directory, "config.json"); await writeFile(configPath, JSON.stringify({ $meta: { version: 1, updatedAt: new Date().toISOString(), source: "onboard" }, - database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: false, dir: path.join(directory, "backups") } }, + database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: Boolean(configuration?.automaticBackups), intervalMinutes: configuration?.automaticBackups?.intervalMinutes ?? 60, dir: path.join(directory, "backups") } }, logging: { mode: "file", logDir: path.join(directory, "logs") }, - server: { deploymentMode: "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: false }, + server: { deploymentMode: options.tenantActor ? "authenticated" : "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: configuration?.serveUi ?? false }, storage: options.cloud ? { provider: "s3", s3: { bucket: "maintenance-test", region: "us-east-1", endpoint: options.cloud.s3Origin, forcePathStyle: true } } : { provider: "local_disk", localDisk: { baseDir: path.join(directory, "storage") } }, secrets: { provider: "local_encrypted", strictMode: false, localEncrypted: { keyFilePath: path.join(directory, "master.key") } }, - telemetry: { enabled: false }, updates: { checkEnabled: false }, + ...(options.tenantActor ? { auth: { baseUrlMode: "explicit", publicBaseUrl: `http://127.0.0.1:${address.port}`, disableSignUp: true } } : {}), + telemetry: { enabled: Boolean(configuration?.telemetryOrigin) }, updates: { checkEnabled: false }, }), { mode: 0o600 }); await new Promise(resolve => reservation.close(() => resolve())); + const captureStartedAt = options.captureIdleWork ? Date.now() : undefined; + const captureSources = options.captureIdleWork ? new Map() : undefined; + let settledDiagnosticId = 0; const child = fork(fileURLToPath(new URL("./server-child.ts", import.meta.url)), [], { cwd: fileURLToPath(new URL("../../../../..", import.meta.url)), - execArgv: ["--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], + execArgv: [...((configuration?.announcements || options.captureIdleWork) ? ['--experimental-test-module-mocks'] : []), "--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], env: { + ...(captureStartedAt === undefined ? {} : { + PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE: '1', + PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW: `${new Date(captureStartedAt).toISOString()}/${new Date(captureStartedAt + 60_000).toISOString()}`, + }), PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: "production", PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test", PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"), - DATABASE_URL: proxy.url, DATABASE_POOL_MAX: "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", + DATABASE_URL: proxy.url, DATABASE_POOL_MAX: configuration ? "4" : "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), - CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false", - PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_UI_DEV_MIDDLEWARE: "false", + CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: String(configuration?.announcements ?? false), + ...(configuration?.announcements ? { PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE: '1' } : {}), + PAPERCLIP_DB_BACKUP_ENABLED: String(Boolean(configuration?.automaticBackups)), PAPERCLIP_UI_DEV_MIDDLEWARE: "false", + ...(configuration?.automaticBackups ? { PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED: configuration.automaticBackups.idleCheckpoint ? "1" : "0" } : {}), + ...(configuration?.telemetryOrigin ? { PAPERCLIP_TELEMETRY_ENDPOINT: configuration.telemetryOrigin } : {}), + ...(configuration?.otelOrigin ? { OTEL_EXPORTER_OTLP_ENDPOINT: configuration.otelOrigin, OTEL_EXPORTER_OTLP_PROTOCOL: "http/json" } : {}), HEARTBEAT_SCHEDULER_ENABLED: String(options.enabled ?? true), PAPERCLIP_MAINTENANCE_FAKE_CLOCK: String(options.clock ?? false), ...(options.suppressed ? { PAPERCLIP_IN_WORKTREE: "true" } : {}), ...(options.githubOrigin ? { PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN: options.githubOrigin } : {}), ...(options.connector ? { - PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", + PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: options.connector.instanceId ?? "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY: options.connector.signPrivateKey, PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY: options.connector.sealPrivateKey, PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: options.connector.origin, } : {}), + ...(options.controlJwks ? { PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.controlJwks } : {}), ...(options.cloud ? { PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } }), PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"), PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "maintenance-tenant-token", PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.cloud.jwks, PAPERCLIP_CLOUD_WARM_STANDBY: options.cloud.standby ? "1" : "0", - PAPERCLIP_PUBLIC_URL: "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: "pool-maintenance.staging.paperclip.app", + PAPERCLIP_PUBLIC_URL: options.tenantActor ? `http://127.0.0.1:${address.port}` : "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: options.tenantActor ? "127.0.0.1" : "pool-maintenance.staging.paperclip.app", + ...(options.tenantActor ? { PAPERCLIP_MAINTENANCE_MANAGED_TENANT: "1" } : {}), AWS_ACCESS_KEY_ID: "maintenance-fixture", AWS_SECRET_ACCESS_KEY: "maintenance-fixture", AWS_EC2_METADATA_DISABLED: "true", } : {}), PAPERCLIP_MIGRATION_PROMPT: "never", PAPERCLIP_MIGRATION_AUTO_APPLY: "true", PAPERCLIP_MANAGED_RUNTIME_HTTPS: "off", @@ -139,12 +200,18 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art for (const request of requests.values()) request.reject(error); requests.clear(); }); - async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } } = {}): Promise { + async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } } = {}): Promise { const id = ++requestId; const result = new Promise((resolve, reject) => requests.set(id, { resolve, reject })); child.send({ id, command, ...payload }); - try { return await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); } - finally { requests.delete(id); } + try { + const value = await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); + if (captureSources) { + const capture = (value as { workCapture?: IdleWorkCaptureSnapshot }).workCapture; + for (const source of capture?.newSources ?? []) captureSources.set(source.id, source.callers); + } + return value; + } finally { requests.delete(id); } } stopChild = async () => { try { @@ -165,7 +232,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art if (options.clock) await request("realtime"); try { const response = await fetch(`http://127.0.0.1:${port}/api${route}`, { method, signal: AbortSignal.timeout(30_000), - headers: { "Content-Type": "application/json", ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); + headers: { "Content-Type": "application/json", ...(Object.keys(headers ?? {}).some(key => ["authorization", "x-paperclip-cloud-control"].includes(key.toLowerCase())) ? {} : tenantHeaders), ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); const value = await response.json(); if (!response.ok) throw new Error(`Maintenance API ${method} ${route}: ${response.status} ${JSON.stringify(value)}`); return value as T; @@ -182,6 +249,16 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art finally { if (options.clock) await request("freeze"); } }, publishBackup: (publication: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }) => request("publishBackup", { publication }), + async prepareGitHub(companyId: string) { + const result = await request<{ result: { connectionId: string; grantId: string } }>("prepareGitHub", { github: { companyId } }); + return result.result; + }, + async settleGitHub(input: { connectionId: string; grantId: string }) { + return request("settleGitHub", { github: input }); + }, + tenantHeaders, + produceChatHistory: () => request("produceChatHistory"), + trace: () => request("trace"), snapshot: () => request("snapshot"), drain: () => request("drain"), resume: () => request("resume"), realtime: () => request("realtime"), @@ -197,17 +274,38 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art }, freeze: () => request("freeze"), get output() { return output; }, - async settled(quietMs = 500, deferredStartupHolds = 0) { + async settled(quietMs = 500, deferredStartupHolds = 0, settleOptions: { managedSetupRecovery?: true } = {}) { + // A successful managed health check self-notifies, then reconciles its + // future deadline on the next existing 30s pass. Observe both passes + + // 500ms quiet + 500ms margin without changing worker timing/admission. + const settleTimeoutMs = settleOptions.managedSetupRecovery ? 61_000 : 30_000; + const diagnostic = options.captureIdleWork ? (await import('./idle-work-capture.js')).createSettledWorkSamples( + () => [...captureSources!].map(([id, callers]) => ({ id, callers }))) : undefined; + const diagnosticId = diagnostic ? ++settledDiagnosticId : undefined; + let failure: { error: unknown } | undefined; let previous = -1, quietSince = Date.now(); - for (const deadline = Date.now() + 30_000; Date.now() < deadline;) { + try { + for (const deadline = Date.now() + settleTimeoutMs; Date.now() < deadline;) { + const snapshot = await request("snapshot"); + diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending); + if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed"); + if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now(); + previous = proxy.queries.length; + if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) { + diagnostic?.assertComplete(); return snapshot; + } + await delay(25); + } const snapshot = await request("snapshot"); - if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed"); - if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now(); - previous = proxy.queries.length; - if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) return snapshot; - await delay(25); + diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending); + throw new Error(`Maintenance did not settle: ${JSON.stringify({ timeoutMs: settleTimeoutMs, quietMs, queries: proxy.queries.length, pending: proxy.pending, snapshot })}\n${output.slice(-12_000)}`); + } catch (error) { failure = { error }; throw error; } + finally { + if (diagnostic) { + try { await writeFile(path.join(directory, `idle-work-settled-samples-${diagnosticId}.json`), JSON.stringify({ ...diagnostic.report(), settleDeadlineMs: settleTimeoutMs }, null, 2)); } + catch (error) { if (failure) throw new AggregateError([failure.error, error], 'Original settling and passive evidence failed'); throw error; } + } } - throw new Error(`Maintenance did not settle: ${JSON.stringify({ queries: proxy.queries.length, pending: proxy.pending, snapshot: await request("snapshot") })}\n${output.slice(-12_000)}`); }, }; } catch (error) { diff --git a/server/src/__tests__/instrumentation-signals.test.ts b/server/src/__tests__/instrumentation-signals.test.ts new file mode 100644 index 00000000000..44e5638b83a --- /dev/null +++ b/server/src/__tests__/instrumentation-signals.test.ts @@ -0,0 +1,30 @@ +import { afterEach, expect, it, vi } from 'vitest'; + +const sdk = vi.hoisted(() => ({ options: [] as Array>, start: vi.fn(), shutdown: vi.fn(async () => {}) })); +vi.mock('../peer-version-check.js', () => ({ checkExactPeerVersions: () => ({ ok: true }) })); +vi.mock('@opentelemetry/sdk-node', () => ({ NodeSDK: class { + constructor(options: Record) { sdk.options.push(options); } + start = sdk.start; + shutdown = sdk.shutdown; +} })); +vi.mock('@opentelemetry/auto-instrumentations-node', () => ({ getNodeAutoInstrumentations: () => [] })); +vi.mock('@opentelemetry/resources', () => ({ resourceFromAttributes: (attributes: unknown) => attributes })); +vi.mock('@opentelemetry/semantic-conventions', () => ({ ATTR_SERVICE_NAME: 'service.name', ATTR_SERVICE_VERSION: 'service.version' })); +vi.mock('@opentelemetry/exporter-trace-otlp-http', () => ({ OTLPTraceExporter: class {} })); + +afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); }); +it('configures current plural SDK signal options so an opted-in trace exporter cannot enable default metrics or logs', async () => { + vi.stubEnv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector.invalid:4318'); + vi.stubEnv('OTEL_EXPORTER_OTLP_PROTOCOL', 'http/json'); + vi.spyOn(process, 'once').mockReturnValue(process); + vi.resetModules(); + const instrumentation = await import('../instrumentation.js'); + await instrumentation.instrumentationReady; + expect(sdk.options).toHaveLength(1); + expect(sdk.options[0]).toMatchObject({ metricReaders: [], logRecordProcessors: [], traceExporter: expect.any(Object) }); + expect(sdk.options[0]).not.toHaveProperty('metricReader'); + expect(sdk.options[0]).not.toHaveProperty('logRecordProcessor'); + expect(sdk.start).toHaveBeenCalledOnce(); + await instrumentation.shutdownInstrumentation(); + expect(sdk.shutdown).toHaveBeenCalledOnce(); +}); diff --git a/server/src/__tests__/maintenance-harness-resources.test.ts b/server/src/__tests__/maintenance-harness-resources.test.ts index 179bb9e20eb..6ab58089901 100644 --- a/server/src/__tests__/maintenance-harness-resources.test.ts +++ b/server/src/__tests__/maintenance-harness-resources.test.ts @@ -125,3 +125,11 @@ describe("owned maintenance providers", () => { } }); }); + +describe("configured acceptance boundaries", () => { + it("rejects remote collectors and fake-clock configuration before opening PostgreSQL", async () => { + await expect(startMaintenanceServer("not-a-database-url", { configuration: { telemetryOrigin: "https://telemetry.example.test" } })).rejects.toThrow("loopback"); + await expect(startMaintenanceServer("not-a-database-url", { clock: true, configuration: { announcements: true } })).rejects.toThrow("real clock"); + await expect(startMaintenanceServer("not-a-database-url", { configuration: { automaticBackups: { intervalMinutes: 0, idleCheckpoint: true } } })).rejects.toThrow("positive integer"); + }); +}); diff --git a/server/src/__tests__/sleep-backup-observation.test.ts b/server/src/__tests__/sleep-backup-observation.test.ts new file mode 100644 index 00000000000..34bac7bbce2 --- /dev/null +++ b/server/src/__tests__/sleep-backup-observation.test.ts @@ -0,0 +1,61 @@ +import { expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, open, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js'; + +it('observes complete owned gzip bytes without retaining SQL and refuses missing, truncated, foreign, historical or failed checkpoints', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-observer-')); + const id = '12345678-1234-1234-1234-123456789012'; + const startedAt = new Date(Date.now() - 1000).toISOString(); + const backupDir = path.join(directory, 'backups'); await mkdir(backupDir); + try { + const observe = () => observeIdleBackupCheckpoint(directory, [], startedAt, id); + await expect(observe()).rejects.toThrow(); + await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker'); + await expect(observe()).rejects.toThrow('no new archive'); + const archive = path.join(backupDir, 'owned.sql.gz'), bytes = gzipSync(`COPY fixture;\n${id}\n`); + await writeFile(archive, bytes); + const receipt = await observe(); + expect(receipt.archives[0]).toMatchObject({ bytes: bytes.length, gzipVerified: true, fixtureIdentifierFound: true }); + expect(receipt.archives[0].sha256).toMatch(/^[a-f0-9]{64}$/); + expect(JSON.stringify(receipt)).not.toContain(id); + await expect(observeIdleBackupCheckpoint(directory, ['owned.sql.gz'], startedAt, id)).rejects.toThrow('no new archive'); + await expect(observeIdleBackupCheckpoint(directory, [], new Date(Date.now() + 1000).toISOString(), id)).rejects.toThrow('predates'); + await writeFile(archive, bytes.subarray(0, bytes.length - 4)); await expect(observe()).rejects.toThrow(); + await writeFile(archive, gzipSync('foreign fixture')); await expect(observe()).rejects.toThrow('owned fixture'); + await writeFile(archive, bytes); await writeFile(path.join(backupDir, 'db-backup-to-s3.failure'), 'failed'); + await expect(observe()).rejects.toThrow('failure marker'); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it('uses opened-file mtime when birthtime is zero and still rejects old or malformed provenance', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-no-birthtime-')); + const id = '12345678-1234-1234-1234-123456789012', startedAt = new Date(Date.now() - 1000).toISOString(); + const backupDir = path.join(directory, 'backups'); await mkdir(backupDir); + const archive = path.join(backupDir, 'owned.sql.gz'); + await writeFile(archive, gzipSync(`COPY fixture;\n${id}\n`)); + await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker'); + const handle = await open(archive, 'r'), prototype = Object.getPrototypeOf(handle), originalStat = prototype.stat; + await handle.close(); + let override: { birthtimeMs: number; mtimeMs?: number } = { birthtimeMs: 0 }; + const spy = vi.spyOn(prototype, 'stat').mockImplementation(async function (this: unknown, ...args: unknown[]) { + return Object.assign(await originalStat.apply(this, args), override); + }); + try { + const observed = await observeIdleBackupCheckpoint(directory, [], startedAt, id); + expect(observed.archives[0]).toMatchObject({ createdAtSource: 'mtime', gzipVerified: true, fixtureIdentifierFound: true }); + expect(Date.parse(observed.archives[0]!.createdAt)).toBeGreaterThanOrEqual(Date.parse(startedAt)); + for (const invalid of [ + { birthtimeMs: 0, mtimeMs: Date.parse(startedAt) - 1 }, { birthtimeMs: 0, mtimeMs: 0 }, + { birthtimeMs: 0, mtimeMs: NaN }, { birthtimeMs: NaN }, { birthtimeMs: -1 }, + { birthtimeMs: 0, mtimeMs: Date.now() + 60_000 }, + { birthtimeMs: Date.parse(startedAt) - 1, mtimeMs: Date.now() }, + ]) { + override = invalid; + await expect(observeIdleBackupCheckpoint(directory, [], startedAt, id)).rejects.toThrow('predates'); + } + await expect(observeIdleBackupCheckpoint(directory, [], 'invalid', id)).rejects.toThrow('checkpoint timestamp'); + } finally { spy.mockRestore(); await rm(directory, { recursive: true, force: true }); } +}); diff --git a/server/src/__tests__/sleep-configuration-acceptance.test.ts b/server/src/__tests__/sleep-configuration-acceptance.test.ts new file mode 100644 index 00000000000..a6d6a7a7ede --- /dev/null +++ b/server/src/__tests__/sleep-configuration-acceptance.test.ts @@ -0,0 +1,312 @@ +import { createHash, generateKeyPairSync, randomUUID, sign } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; +import { createServer } from 'node:http'; +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { configuredSleepFeatures, finalizeLocalSleepReceipt, observeConfiguredSignals, observeIncomingHttpWindow, observedNoIncomingHttp, sleepConfigurationDigest } from '../../../scripts/sleep-qualification/evidence.mjs'; +import { idleBackupArchiveNames, observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js'; +import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { managedSleepConfiguration, managedSleepServerOptions, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js'; +import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js'; +import { produceCompletedToolReview } from './helpers/maintenance/configuration-producers.js'; +import { observeMaintenanceBrowser } from './helpers/maintenance/browser-observation.js'; +import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js'; + +const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL'); +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !databaseUrl) throw new Error('Configured acceptance requires an explicit dedicated PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL; embedded allocation is forbidden'); +const cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA; +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !/^[a-f0-9]{40}$/.test(cloudSha ?? '')) throw new Error('Configured acceptance requires the exact PAPERCLIP_SLEEP_CLOUD_SHA'); +const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT; +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !cloudCheckout) throw new Error('Configured acceptance requires a reviewed PAPERCLIP_SLEEP_CLOUD_CHECKOUT with its built qualification broker adapter'); +const enabled = process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && Boolean(databaseUrl); +const quietMs = 90_000, apiCooldownMs = 600_000; +if (enabled && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Configured qualification requires retained raw evidence'); + +describe.skipIf(!enabled)('configured assembled sleep qualification (explicit owned external PostgreSQL only)', () => { + beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000); + it('retains normal credentials, completed work and a future routine through a real cooldown and quiet window', async () => { + const phase = (name: string) => console.info(JSON.stringify({ qualificationPhase: name, at: new Date().toISOString() })); + phase('starting-owned-fixtures'); + const collectorRequests: Array<{ path: string; body: unknown }> = []; + const collector = createServer(async (req, res) => { + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + collectorRequests.push({ path: req.url!, body: JSON.parse(Buffer.concat(chunks).toString('utf8')) }); + res.setHeader('Content-Type', 'application/json'); res.end('{}'); + }); + if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Qualification Cloud checkout does not match the exact requested SHA'); + const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href); + const broker = await startQualificationBroker(); + const s3 = createMaintenanceS3Boundary(); + try { + await withMaintenanceProvider(s3.server, async s3Origin => { + await withMaintenanceProvider(collector, async origin => { + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'sleep-qualification', alg: 'EdDSA', use: 'sig' }; + const configuration = managedSleepConfiguration(origin, process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1'); + // Independent intended logical contract; collector ports are observed separately. + const expected = { target: 'owned-local-assembled', appSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), cloudSha: cloudSha!, + configDigest: createHash('sha256').update(JSON.stringify({ automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, + telemetry: true, otel: { enabled: true, protocol: 'http/json' }, serveUi: process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1', + managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' })).digest('hex'), + collectorOrigins: { telemetry: `${origin}/telemetry`, otel: origin } }; + expect(sleepConfigurationDigest(configuration)).toBe(expected.configDigest); + const startedAt = new Date().toISOString(); + const server = await startMaintenanceServer(databaseUrl!, managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker)); + phase('actual-server-ready'); + let fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const control = (action: string, idleCapabilities?: unknown) => { + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url'); + const now = Math.floor(Date.now() / 1000); + const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid }); + const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) }); + return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` }; + }; + let failure: { error: unknown } | undefined; + try { + await writeFile(path.join(server.directory, 'sleep-managed-startup.json'), JSON.stringify(await server.snapshot(), null, 2)); + const company = await provisionManagedSleepCompany(server, fixture); + expect(await server.api('GET', '/announcements/current')).toMatchObject({ id: 'new-projects' }); + const announcement = (await server.snapshot()).announcement; + expect(announcement?.intercepted).toBe(true); + expect(announcement?.requests).toEqual(['https://pages.paperclip.ing/announcements/v1/current.json']); + expect(announcement?.unmatched).toEqual([]); + const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Owned process worker', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', "console.log('sleep-acceptance-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const chatHistory = await server.produceChatHistory(); + const toolHistory = await produceCompletedToolReview(server, fixture, company.id); + const github = await server.prepareGitHub(company.id); + await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true }); + const githubContinuity = await server.settleGitHub(github); + await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1); + await broker.enqueue(); + await broker.deliver(`http://127.0.0.1:${server.port}`, () => control('connector-events:wake')[CLOUD_CONTROL_HEADER]); + // The normally enabled connection owner leases and acknowledges the real sealed event. + await expect.poll(async () => { + const [delivery] = await fixture.$client`select status from connection_event_deliveries where provider_delivery_id='qualification-delivery'`; + return delivery?.status; + }, { timeout: 30_000 }).toBe('processed'); + expect(await broker.pending()).toEqual([]); + const brokerBinding = await broker.binding(); + expect(brokerBinding?.bindings).toHaveLength(1); + const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Retained qualification credential' }); + const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'Owned completed history', status: 'todo', assigneeAgentId: agent.id }); + await expect.poll(async () => { + const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; + return run?.status; + }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + const routine = await server.api<{ id: string }>('POST', `/companies/${company.id}/routines`, { title: 'Harmless future obligation', assigneeAgentId: agent.id }); + await server.api('POST', `/routines/${routine.id}/triggers`, { kind: 'schedule', cronExpression: (() => { const next = new Date(Date.now() + 24 * 60 * 60_000); return `0 ${next.getUTCHours()} ${next.getUTCDate()} ${next.getUTCMonth() + 1} *`; })(), timezone: 'UTC' }); + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); + const lastAuthenticatedApiAt = performance.now(); + await writeFile(path.join(server.directory, 'sleep-setup-receipt.json'), JSON.stringify({ ...expected, configuration, company, + agentId: agent.id, keyId: key.id, announcement, chatHistory, toolHistory, githubContinuity, brokerBinding, + brokerBoundary: broker.boundary, s3: s3.snapshot(), snapshot: await server.snapshot(), + boundary: 'Actual managed startup/tenant provisioning/producers; Cloud front door, GitHub HTTP and S3 service are owned simulated transports' }, null, 2)); + expect(s3.snapshot().errors).toEqual([]); + let browser; + try { + browser = process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1' ? await observeMaintenanceBrowser(`http://127.0.0.1:${server.port}`, 'Organize your work.', server.tenantHeaders) : undefined; + await writeFile(path.join(server.directory, 'sleep-browser-receipt.json'), JSON.stringify({ browser, observedAt: new Date().toISOString() }, null, 2)); + } catch (error) { + await writeFile(path.join(server.directory, 'sleep-browser-error.json'), JSON.stringify({ observedAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) }, null, 2)); + throw error; + } + // Setup-only owner settling never substitutes for the later full quiet window. + await server.settled(500, 0, { managedSetupRecovery: true }); + const earlyDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + try { + const earlyCapabilities = { incomingWake: await broker.capability(earlyDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const earlyRead = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:read', earlyCapabilities)); + const early = await stabilizeEarlyDiagnostics(server, { ownerId: earlyDrain.ownerId, apiSucceededAt: lastAuthenticatedApiAt, read: earlyRead, + persist: observations => writeFile(path.join(server.directory, 'sleep-early-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) }); + await writeFile(path.join(server.directory, 'sleep-early-diagnostics.json'), JSON.stringify(early, null, 2)); + } finally { await server.api('DELETE', `/instance/task-drain?ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:stop')); } + // Anchor the full cooldown to a genuine successful opaque-key request. + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); + const lastApiActivity = performance.now(); + await fixture.$client.end({ timeout: 1 }); + await server.trace(); + phase('first-real-ten-minute-api-cooldown'); + // The real authenticated-use cooldown is part of ordinary behavior. + // No fake clock, production timing override, tenant health poll or SQL observer. + await delay(apiCooldownMs + 100); + await server.settled(); + const activeSignals = observeConfiguredSignals(collectorRequests); + expect(activeSignals.qualificationSpans).toBeGreaterThan(0); + expect(activeSignals.telemetryRequests).toBeGreaterThan(0); + const before = await server.snapshot(), queries = server.proxy.queries.length, transport = server.proxy.transport; + const wallStart = performance.now(), quietStartedAt = new Date().toISOString(); + phase('first-real-ninety-second-quiet-window'); + await delay(quietMs); + const elapsedMs = performance.now() - wallStart, quietFinishedAt = new Date().toISOString(); + const after = await server.snapshot(), finalTransport = server.proxy.transport; + const quiet = { startedAt: quietStartedAt, finishedAt: quietFinishedAt, elapsedMs, clock: 'real', + sqlQueries: server.proxy.queries.length - queries, + connectionsOpened: finalTransport.connectionsOpened - transport.connectionsOpened, + connectionsRemaining: finalTransport.connectionsOpened - finalTransport.connectionsClosed, + nonDatabaseRequests: after.outgoingHttpRequests - before.outgoingHttpRequests, + nonDatabaseBytes: after.network.nonDatabaseBytesWritten - before.network.nonDatabaseBytesWritten, + observerRequests: observeIncomingHttpWindow(before.incomingHttp, after.incomingHttp) }; + await writeFile(path.join(server.directory, 'sleep-first-frozen-quiet.json'), JSON.stringify({ quiet, announcementBefore: before.announcement, announcementAfter: after.announcement }, null, 2)); + const signalObservation = observeConfiguredSignals(collectorRequests); + await writeFile(path.join(server.directory, 'sleep-first-signal-observation.json'), JSON.stringify(signalObservation, null, 2)); + expect(signalObservation.qualificationSpans).toBeGreaterThan(0); + expect(signalObservation.telemetryRequests).toBeGreaterThan(0); + expect(signalObservation.metricRequests).toBe(0); expect(signalObservation.logRequests).toBe(0); + expect(signalObservation.malformedTraceRequests).toBe(0); expect(signalObservation.unexpectedRequests).toBe(0); + expect(quiet.elapsedMs).toBeGreaterThanOrEqual(quietMs); + expect(quiet.sqlQueries).toBe(0); + expect(quiet.connectionsOpened).toBe(0); + expect(quiet.connectionsRemaining).toBe(0); + expect(quiet.nonDatabaseRequests).toBe(0); + expect(quiet.nonDatabaseBytes).toBe(0); + // The installed listener must have observed the actual setup HTTP traffic. + expect(quiet.observerRequests.before.requests).toBeGreaterThan(0); + expect(observedNoIncomingHttp(quiet.observerRequests, quiet)).toBe(true); + expect(after.announcement).toEqual(before.announcement); + const backupNamesBefore = await idleBackupArchiveNames(server.directory); + const checkpointStartedAt = new Date().toISOString(); + const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const read = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + const initial = await read(); + await writeFile(path.join(server.directory, 'sleep-initial-safety.json'), JSON.stringify(initial, null, 2)); + const final = await read(); + phase('first-owned-eligibility-reports-read'); + // Persist raw refusal evidence before any assertion can terminate the case. + await writeFile(path.join(server.directory, 'sleep-initial-final-safety.json'), JSON.stringify({ initial, final }, null, 2)); + if (initial.idleSleepSafety.backgroundWork !== 'none' || final.idleSleepSafety.backgroundWork !== 'none') { + const diagnostic = await server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + await writeFile(path.join(server.directory, 'sleep-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2)); + } + const { parseApiWakeAdmission, authorizeApiWakeAdmission } = await import('../services/idle-api-wake-admission.js'); + const assertIncomingProof = (report: any, incomingWake: any, ownerId: string) => { + expect(report.idleSleepSafety.incomingWake).toEqual(incomingWake); + const admission = parseApiWakeAdmission(report.idleSleepSafety.apiWakeAdmission); + expect(admission).toMatchObject({ stackId: 'maintenance-stack', ownerId, + capabilityGeneration: incomingWake.generation, challenge: incomingWake.apiKeyProof.challenge }); + expect(authorizeApiWakeAdmission(admission, { stackId: 'maintenance-stack', token: key.token, now: Date.now() })).toBeDefined(); + }; + assertIncomingProof(initial, capabilities.incomingWake, drain.ownerId); + assertIncomingProof(final, capabilities.incomingWake, drain.ownerId); + expect(final.idleSleepSafety.apiWakeAdmission).toEqual(initial.idleSleepSafety.apiWakeAdmission); + const proof = final.idleSleepSafety; + const backupCheckpoint = await observeIdleBackupCheckpoint(server.directory, backupNamesBefore, checkpointStartedAt, company.id); + const observations = { version: 1, ...expected, startedAt, + configuration, browser, s3: s3.snapshot(), announcement: after.announcement, chatHistory, toolHistory, githubContinuity, brokerBinding, brokerBoundary: broker.boundary, backupCheckpoint, + signalObservation, features: configuredSleepFeatures({ announcement: after.announcement, signalObservation, backupCheckpoint }), + quiet, + eligibility: { observedAt: new Date().toISOString(), backgroundWork: proof.backgroundWork, ownedHold: final.ownerId === drain.ownerId, apiCooldownElapsedMs: performance.now() - lastApiActivity, + unchangedGeneration: /^[a-f0-9]{64}$/.test(proof.deadlineProof?.generation ?? '') && initial.idleSleepSafety.deadlineProof?.generation === proof.deadlineProof?.generation }, initial, final, + live: { databaseSuspension: 'INCONCLUSIVE', hostSuspension: 'INCONCLUSIVE', backupPersistence: 'INCONCLUSIVE', reason: 'Local actual server with simulated Cloud signer; no deployed provider target authorized' }, + coverage: { announcements: 'Actual route, validated feed/cache and UI with one named injected provider transport fetch; DNS/HTTPS guard covered separately, not exercised here.', browser: browser ? 'Actual Chromium foreground/background measured, then closed before server quiet window' : 'Closed; set PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE=1 with built UI and headed Chromium for background/active evidence', broker: 'Retained enabled connection, actual encrypted OAuth vault, actual continuity/binding owners and signed/sealed production broker delivery; host capability from actual ConnectorStore', history: 'Normal task/credential/routine, actual tool review rejection/outbox retirement, actual signed chat ingress completion and endpoint archival' } }; + await writeFile(path.join(server.directory, 'sleep-configuration-observations.json'), JSON.stringify(observations, null, 2)); + const { receipt, report } = finalizeLocalSleepReceipt(observations, expected); + await writeFile(path.join(server.directory, 'sleep-configuration-receipt.json'), JSON.stringify(receipt, null, 2)); + await writeFile(path.join(server.directory, 'sleep-configuration-evaluation.json'), JSON.stringify(report, null, 2)); + expect(proof.version).toBe(3); expect(proof.backgroundWork).toBe('none'); + expect(proof.deadlineProof).toMatchObject({ catalog: 'domain_deadlines_v1', generation: expect.stringMatching(/^[a-f0-9]{64}$/) }); + fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const [retained] = await fixture.$client`select (select count(*)::int from agent_api_keys where id=${key.id}) as keys, (select count(*)::int from issues where id=${issue.id}) as issues, (select count(*)::int from routines where id=${routine.id}) as routines, (select enabled from tool_connections where id=${github.connectionId}) as github_enabled`; + expect(retained).toEqual({ keys: 1, issues: 1, routines: 1, github_enabled: true }); + await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop')); + const nextIssue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'First ordinary producer after quiet', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` }); + const reactivationApiAt = performance.now(); + await expect.poll(async () => { + const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${nextIssue.id} order by created_at desc limit 1`; + return run?.status; + }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${nextIssue.id}`, { status: 'done' }); + const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${nextIssue.id}`; + expect(executions.count).toBe(1); + await fixture.$client.end({ timeout: 1 }); + phase('reactivation-real-ten-minute-api-cooldown'); + await delay(apiCooldownMs + 100); + await server.settled(); + const resumed = await server.snapshot(), resumedQueries = server.proxy.queries.length, resumedTransport = server.proxy.transport; + const resumedStart = performance.now(), resumedStartedAt = new Date().toISOString(); + phase('reactivation-real-ninety-second-quiet-window'); + await delay(quietMs); + const resumedElapsedMs = performance.now() - resumedStart, resumedFinishedAt = new Date().toISOString(); + const resumedEnd = await server.snapshot(), resumedFinalTransport = server.proxy.transport; + // Freeze every measurement before signed drain/proof requests do SQL. + const resumedQuiet = { startedAt: resumedStartedAt, finishedAt: resumedFinishedAt, elapsedMs: resumedElapsedMs, clock: 'real', + sqlQueries: server.proxy.queries.length - resumedQueries, + connectionsOpened: resumedFinalTransport.connectionsOpened - resumedTransport.connectionsOpened, + connectionsRemaining: resumedFinalTransport.connectionsOpened - resumedFinalTransport.connectionsClosed, + nonDatabaseRequests: resumedEnd.outgoingHttpRequests - resumed.outgoingHttpRequests, + nonDatabaseBytes: resumedEnd.network.nonDatabaseBytesWritten - resumed.network.nonDatabaseBytesWritten, + observerRequests: observeIncomingHttpWindow(resumed.incomingHttp, resumedEnd.incomingHttp) }; + await writeFile(path.join(server.directory, 'sleep-resumed-frozen-quiet.json'), JSON.stringify({ quiet: resumedQuiet, announcementBefore: resumed.announcement, announcementAfter: resumedEnd.announcement }, null, 2)); + const resumedSignals = observeConfiguredSignals(collectorRequests); + await writeFile(path.join(server.directory, 'sleep-resumed-signal-observation.json'), JSON.stringify(resumedSignals, null, 2)); + expect(resumedSignals.qualificationSpans).toBeGreaterThan(0); + expect(resumedSignals.telemetryRequests).toBeGreaterThan(0); + expect(resumedSignals.metricRequests).toBe(0); expect(resumedSignals.logRequests).toBe(0); + expect(resumedSignals.malformedTraceRequests).toBe(0); expect(resumedSignals.unexpectedRequests).toBe(0); + expect(resumedQuiet.elapsedMs).toBeGreaterThanOrEqual(quietMs); + expect(resumedQuiet.sqlQueries).toBe(0); + expect(resumedQuiet.connectionsOpened).toBe(0); + expect(resumedQuiet.connectionsRemaining).toBe(0); + expect(resumedQuiet.nonDatabaseRequests).toBe(0); + expect(resumedQuiet.nonDatabaseBytes).toBe(0); + expect(resumedQuiet.observerRequests.before.requests).toBeGreaterThan(0); + expect(observedNoIncomingHttp(resumedQuiet.observerRequests, resumedQuiet)).toBe(true); + expect(resumedEnd.announcement).toEqual(resumed.announcement); + expect(resumedEnd.announcement?.unmatched).toEqual([]); + const resumedDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + const resumedCapabilities = { incomingWake: await broker.capability(resumedDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const readResumed = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities)); + const resumedInitial = await readResumed(); + await writeFile(path.join(server.directory, 'sleep-resumed-initial-safety.json'), JSON.stringify(resumedInitial, null, 2)); + const resumedProof = await readResumed(); + await writeFile(path.join(server.directory, 'sleep-resumed-initial-final-safety.json'), JSON.stringify({ initial: resumedInitial, final: resumedProof }, null, 2)); + if (resumedInitial.idleSleepSafety.backgroundWork !== 'none' || resumedProof.idleSleepSafety.backgroundWork !== 'none') { + const diagnostic = await server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities)); + await writeFile(path.join(server.directory, 'sleep-resumed-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2)); + } + assertIncomingProof(resumedInitial, resumedCapabilities.incomingWake, resumedDrain.ownerId); + assertIncomingProof(resumedProof, resumedCapabilities.incomingWake, resumedDrain.ownerId); + expect(resumedProof.idleSleepSafety.apiWakeAdmission).toEqual(resumedInitial.idleSleepSafety.apiWakeAdmission); + expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toMatch(/^[a-f0-9]{64}$/); + expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toBe(resumedInitial.idleSleepSafety.deadlineProof?.generation); + expect(resumedProof.idleSleepSafety.backgroundWork).toBe('none'); + await writeFile(path.join(server.directory, 'sleep-local-reactivation-receipt.json'), JSON.stringify({ + version: 1, actualProducer: 'POST company issues with process assignee using the retained opaque key after owned drain release', executions: executions.count, + realApiCooldownElapsedMs: performance.now() - reactivationApiAt, + quiet: resumedQuiet, + signalObservation: resumedSignals, + realElapsedMs: resumedQuiet.elapsedMs, sqlQueries: resumedQuiet.sqlQueries, + networkRequests: resumedQuiet.nonDatabaseRequests, + initial: resumedInitial, + eligibility: resumedProof.idleSleepSafety, + boundary: 'Actual local application reactivation, real cooldown and fresh eligibility; no provider suspension or deployed inbound-wake claim', + }, null, 2)); + const s3Evidence = s3.snapshot(); + await writeFile(path.join(server.directory, 'sleep-s3-boundary.json'), JSON.stringify(s3Evidence, null, 2)); + expect(s3Evidence.errors).toEqual([]); + phase('configured-qualification-receipts-complete'); + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + try { await writeFile(path.join(server.directory, 'sleep-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + try { await server.stop(); } catch (error) { errors.push(error); } + try { await fixture.$client.end(); } catch (error) { errors.push(error); } + // Shutdown can flush storage; capture and validate only after child exit. + try { await writeFile(path.join(server.directory, 'sleep-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Configured work or owned cleanup failed'); + } + }); + }); + } finally { await broker.stop(); } + }, 1_800_000); +}); diff --git a/server/src/__tests__/sleep-early-diagnostics.test.ts b/server/src/__tests__/sleep-early-diagnostics.test.ts new file mode 100644 index 00000000000..b9db5979378 --- /dev/null +++ b/server/src/__tests__/sleep-early-diagnostics.test.ts @@ -0,0 +1,15 @@ +import { expect, it } from 'vitest'; +import { assertEarlySleepDiagnostics } from './helpers/maintenance/early-sleep-diagnostics.js'; +const fixture = () => ({ idleSleepSafety: { diagnostics: { complete: true, checks: [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog', 'issues'].map(reason => ({ reason, state: 'none' })), +] } } }); +it('retains only explicit early cooldown/diagnostic-backup reasons and refuses every unexpected or unknown blocker', () => { + expect(() => assertEarlySleepDiagnostics(fixture(), 90_000)).not.toThrow(); + for (const [reason, state] of [['issues', 'present'], ['deadlines.catalog', 'unknown'], ['inspection.hold', 'not_checked'], ['local.registered_probes', 'unknown']]) { + const f = fixture(); f.idleSleepSafety.diagnostics.checks.find(check => check.reason === reason)!.state = state; + expect(() => assertEarlySleepDiagnostics(f, 90_000)).toThrow('Unexpected'); + } + const missing = fixture(); missing.idleSleepSafety.diagnostics.checks.shift(); expect(() => assertEarlySleepDiagnostics(missing, 90_000)).toThrow('Missing'); + for (const duration of [NaN, Infinity, -1, 600_000]) expect(() => assertEarlySleepDiagnostics(fixture(), duration)).toThrow('cooldown'); +}); diff --git a/server/src/__tests__/sleep-idle-work-capture.test.ts b/server/src/__tests__/sleep-idle-work-capture.test.ts new file mode 100644 index 00000000000..114b04057bb --- /dev/null +++ b/server/src/__tests__/sleep-idle-work-capture.test.ts @@ -0,0 +1,90 @@ +import { afterEach, expect, it } from 'vitest'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { createMaintenanceIdleWorkCapture as createCapture, createSettledWorkSamples } from './helpers/maintenance/idle-work-capture.js'; + +const captures: ReturnType[] = []; +function createMaintenanceIdleWorkCapture(...args: Parameters) { + const capture = createCapture(...args); captures.push(capture); return capture; +} +afterEach(() => { for (const capture of captures.splice(0)) capture.dispose(); }); + +const stack = () => 'Error\n at begin (/checkout/server/src/services/owned-work.ts:12:3)\n at owner (/checkout/server/src/index.ts:100:5)'; +function original() { + let active = 0, generation = 0, finishes = 0, tracked: Promise | undefined; + const beginIdleTrackedWork = () => { + active++; generation++; let done = false; + return () => { finishes++; if (!done) { done = true; active--; generation++; } }; + }; + const trackIdleWork = (work: Promise) => { + const finish = beginIdleTrackedWork(); const result = work.finally(finish); tracked = result; return result; + }; + return { beginIdleTrackedWork, trackIdleWork, idleWorkSnapshot: () => ({ active, generation }), + finishes: () => finishes, lastTracked: () => tracked }; +} +it('delegates original finish calls/idempotence and returns the exact original tracked promise and rejection', async () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack }); + const finish = capture.beginIdleTrackedWork(); expect(actual.idleWorkSnapshot().active).toBe(1); + finish(); finish(); expect(actual.finishes()).toBe(2); expect(actual.idleWorkSnapshot().active).toBe(0); + let resolve!: (value: string) => void; + const tracked = capture.trackIdleWork(new Promise(done => { resolve = done; })); + expect(tracked).toBe(actual.lastTracked()); expect(capture.snapshot().observedActive).toBe(1); + resolve('same'); await expect(tracked).resolves.toBe('same'); expect(capture.snapshot().observedActive).toBe(0); + const failure = new Error('original rejection'), rejected = capture.trackIdleWork(Promise.reject(failure)); + expect(rejected).toBe(actual.lastTracked()); await expect(rejected).rejects.toBe(failure); + expect(capture.snapshot().observedActive).toBe(0); expect(actual.idleWorkSnapshot().active).toBe(0); + capture.dispose(); capture.dispose(); expect(capture.snapshot().disposed).toBe(true); + await expect(capture.trackIdleWork(Promise.resolve('after-dispose'))).resolves.toBe('after-dispose'); + expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().observedActive).toBe(0); +}); +it('cannot throw from passive metadata into actual work and makes missing observation explicit', () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack, now: () => { throw new Error('private observer detail'); } }); + let finish!: () => void; expect(() => { finish = capture.beginIdleTrackedWork(); }).not.toThrow(); + expect(actual.idleWorkSnapshot().active).toBe(1); expect(capture.snapshot().missingPaths).toBeGreaterThan(0); + expect(() => finish()).not.toThrow(); expect(actual.idleWorkSnapshot().active).toBe(0); + expect(capture.snapshot().observerErrors).toBeGreaterThan(0); expect(JSON.stringify(capture.summary())).not.toContain('private observer detail'); +}); +it('bounds active entries, completion tail and filtered callsites without clearing original work', () => { + const actual = original(); + const callers = () => 'private outside value\n' + Array.from({ length: 8 }, (_, i) => ` at work (/root/server/src/services/owned-${i}.ts:1:2)`).join('\n'); + const capture = createMaintenanceIdleWorkCapture(actual, { stack: callers }); + const finish = Array.from({ length: 513 }, () => capture.beginIdleTrackedWork()); + const busy = capture.snapshot(); expect(busy.active).toHaveLength(512); expect(busy.overflow).toBe(1); + expect(busy.newSources[0]!.callers).toHaveLength(4); expect(JSON.stringify(busy)).not.toContain('private outside value'); + expect(actual.idleWorkSnapshot().active).toBe(513); expect(busy.missingPaths).toBeGreaterThan(0); + finish.forEach(done => done()); expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().completionTail).toHaveLength(64); +}); +it('bounds unique source metadata and preserves original finisher failures', () => { + const actual = original(); let source = 0; + const capture = createMaintenanceIdleWorkCapture(actual, { stack: () => `Error\n at owner (/checkout/server/src/services/source-${source++}.ts:1:1)` }); + for (let i = 0; i < 513; i++) capture.beginIdleTrackedWork()(); + const state = capture.snapshot(); expect(capture.summary().sourceCatalog).toHaveLength(512); expect(state.sourceOverflow).toBe(1); + expect(actual.idleWorkSnapshot().active).toBe(0); + const failure = new Error('original finisher'), begin = actual.beginIdleTrackedWork; + actual.beginIdleTrackedWork = () => { const finish = begin(); return () => { finish(); throw failure; }; }; + const observed = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = observed.beginIdleTrackedWork(); + expect(() => finish()).toThrow(failure); expect(actual.idleWorkSnapshot().active).toBe(0); expect(observed.snapshot().observedActive).toBe(0); +}); +it('bounds settle samples and stores callsites once, while missing paths forbid diagnostic completion', () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = capture.beginIdleTrackedWork(); + const state = capture.snapshot(), catalog = state.newSources; + const samples = createSettledWorkSamples(() => catalog); + for (let i = 0; i < 2442; i++) samples.sample({ now: 1000 + i, idle: actual.idleWorkSnapshot(), workCapture: state }, 20 + i, 0); + const report = samples.report(); expect(report.samples).toHaveLength(2441); expect(report.sampleLimit).toBe(2441); expect(report.overflowSamples).toBe(1); + expect(report.samples[1]!.queryDelta).toBe(1); expect(report.sourceCatalog).toHaveLength(1); + expect(JSON.stringify(report.samples)).not.toContain('owned-work'); expect(() => samples.assertComplete()).toThrow('incomplete'); + finish(); + const missing = createSettledWorkSamples(() => []); missing.sample({ now: 0, idle: actual.idleWorkSnapshot() }, 0, 0); + expect(() => missing.assertComplete()).toThrow('incomplete'); +}); +it('intercepts the actual Node24 TS module and preserves every other original export and existing state', async () => { + const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url))], { timeout: 15_000, maxBuffer: 128 * 1024 }); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ identity: true, originalStatePreserved: true, finalActive: 0 }); +}); +it('preserves actual Node unhandledRejection with the same original tracked promise and reason', async () => { + const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url)), 'unhandled'], { timeout: 15_000, maxBuffer: 128 * 1024 }); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 }); +}); diff --git a/server/src/__tests__/sleep-incoming-http-observation.test.ts b/server/src/__tests__/sleep-incoming-http-observation.test.ts new file mode 100644 index 00000000000..e42d215eb54 --- /dev/null +++ b/server/src/__tests__/sleep-incoming-http-observation.test.ts @@ -0,0 +1,100 @@ +import { createServer, request } from "node:http"; +import { setImmediate } from "node:timers/promises"; +import { expect, it } from "vitest"; +import { observeIncomingHttpWindow, observedNoIncomingHttp } from "../../../scripts/sleep-qualification/evidence.mjs"; +import { observeMaintenanceIncomingHttp } from "./helpers/maintenance/incoming-http-observation.js"; +import { withMaintenanceProvider } from "./helpers/maintenance/server-harness.js"; + +it("counts a direct SQL-free health fetch without retaining credentials or changing its response", async () => { + // This fixture has no database or outgoing client; its only work is answering HTTP. + const server = createServer((_req, res) => { res.setHeader("Content-Type", "application/json"); res.end('{"status":"ok"}'); }); + server.on("upgrade", (_req, socket) => socket.destroy()); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const before = capture.snapshot(), startedAt = new Date().toISOString(); + expect(before).toMatchObject({ installed: true, valid: true, requests: 0 }); + expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, before), { startedAt: before.observedAt, finishedAt: before.observedAt })).toBe(true); + const response = await fetch(`${origin}/api/health?fixture=private-query`, { signal: AbortSignal.timeout(5_000), headers: { Authorization: "Bearer fixture-private-token" } }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ status: "ok" }); + const finishedAt = new Date().toISOString(), after = capture.snapshot(); + expect(after).toMatchObject({ installed: true, valid: true, captureId: before.captureId }); + const observed = observeIncomingHttpWindow(before, after); + expect(observed.requests).toBe(1); + expect(observedNoIncomingHttp(observed, { startedAt, finishedAt })).toBe(false); + expect(JSON.stringify(observed)).not.toMatch(/private-query|fixture-private-token|Authorization|\/api\/health/); + }); + } finally { capture.dispose(); } +}); + +it("observes a genuinely request-free interval without making snapshot reads into HTTP", async () => { + const server = createServer((_req, res) => res.end()); + server.on("upgrade", (_req, socket) => socket.destroy()); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async () => { + const before = capture.snapshot(), startedAt = new Date().toISOString(); + capture.snapshot(); await setImmediate(); capture.snapshot(); + const finishedAt = new Date().toISOString(), after = capture.snapshot(); + expect(observeIncomingHttpWindow(before, after).requests).toBe(0); + expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, after), { startedAt, finishedAt })).toBe(true); + // This short boundary test does not claim the configured case's 90-second quiet proof. + }); + } finally { capture.dispose(); } +}); + +it("counts an actual HTTP upgrade as incoming activity", async () => { + const server = createServer((_req, res) => res.end()); + const upgrade = (_req: unknown, socket: import("node:stream").Duplex) => { + socket.end("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n"); + }; + server.on("upgrade", upgrade); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const before = capture.snapshot(); + const client = request(origin, { signal: AbortSignal.timeout(5_000), headers: { Connection: "Upgrade", Upgrade: "fixture" } }); + try { + await new Promise((resolve, reject) => { + client.once("error", reject); + client.once("upgrade", (_response, socket) => { socket.destroy(); resolve(); }); + client.end(); + }); + } finally { client.destroy(); } + expect(observeIncomingHttpWindow(before, capture.snapshot()).requests).toBe(1); + }); + } finally { capture.dispose(); server.off("upgrade", upgrade); } +}); + +it("invalidates a detached listener and disposes only its own listeners", async () => { + const app = (_req: unknown, res: import("node:http").ServerResponse) => res.end("app"); + const server = createServer(app); + const unrelatedUpgrade = () => {}; + server.on("upgrade", unrelatedUpgrade); + const originalRequest = server.listeners("request"), originalUpgrade = server.listeners("upgrade"); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const counter = server.listeners("request")[0]! as (...args: unknown[]) => void; + server.off("request", counter); + expect(capture.snapshot()).toMatchObject({ installed: true, valid: false }); + // Reattaching cannot erase an unobserved interval. + server.prependListener("request", counter); + expect(capture.snapshot().valid).toBe(false); + capture.dispose(); capture.dispose(); + expect(capture.snapshot()).toMatchObject({ installed: false, valid: false }); + expect(server.listeners("request")).toEqual(originalRequest); + expect(server.listeners("upgrade")).toEqual(originalUpgrade); + expect(await (await fetch(origin, { signal: AbortSignal.timeout(5_000) })).text()).toBe("app"); + }); + } finally { capture.dispose(); } +}); + +it("leaves Node's unsupported-upgrade behavior alone when the app has no upgrade handler", () => { + const server = createServer((_req, res) => res.end()); + const requestHandlers = server.listeners("request"), upgradeHandlers = server.listeners("upgrade"); + expect(() => observeMaintenanceIncomingHttp(server)).toThrow("requires existing app request and upgrade handlers"); + expect(server.listeners("request")).toEqual(requestHandlers); + expect(server.listeners("upgrade")).toEqual(upgradeHandlers); +}); diff --git a/server/src/__tests__/sleep-managed-fixture.test.ts b/server/src/__tests__/sleep-managed-fixture.test.ts new file mode 100644 index 00000000000..0f7a3ada9f5 --- /dev/null +++ b/server/src/__tests__/sleep-managed-fixture.test.ts @@ -0,0 +1,69 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { createMaintenanceS3Boundary, decodeMaintenanceAwsChunks } from './helpers/maintenance/s3-boundary.js'; +import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { createS3StorageProvider } from '../storage/s3-provider.js'; +import { classifyEarlyDiagnostic, stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; + +afterEach(() => vi.unstubAllEnvs()); +it('uses real small-object SDK HTTP with conditional bytes, reads and deletion; unknown operations fail closed', async () => { + vi.stubEnv('AWS_ACCESS_KEY_ID', 'owned-fixture'); vi.stubEnv('AWS_SECRET_ACCESS_KEY', 'owned-fixture'); vi.stubEnv('AWS_SESSION_TOKEN', undefined); + const fixture = createMaintenanceS3Boundary(); + await withMaintenanceProvider(fixture.server, async endpoint => { + const provider = createS3StorageProvider({ bucket: 'maintenance-test', region: 'us-east-1', endpoint, forcePathStyle: true }); + const input = { objectKey: 'owned/value', contentType: 'text/plain', contentLength: 3, body: Buffer.from('one') }; + const first = await provider.putObjectConditional!({ ...input, ifNoneMatch: '*' }); + expect(first.etag).toBeTruthy(); + await expect(provider.putObjectConditional!({ ...input, ifNoneMatch: '*' })).rejects.toMatchObject({ name: 'PreconditionFailed' }); + await expect(provider.putObjectConditional!({ ...input, ifMatch: 'wrong' })).rejects.toMatchObject({ name: 'PreconditionFailed' }); + await provider.putObjectConditional!({ ...input, body: Buffer.from('two'), ifMatch: first.etag }); + const object = await provider.getObject({ objectKey: input.objectKey }); const chunks: Buffer[] = []; + for await (const chunk of object.stream) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe('two'); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(true); + expect((await provider.listObjects!({ prefix: 'owned/' })).keys).toEqual([input.objectKey]); + await provider.deleteObject({ objectKey: input.objectKey }); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(false); + expect(fixture.snapshot().errors).toEqual([]); + const unknown = await fetch(`${endpoint}/maintenance-test/key?uploads`, { method: 'POST' }); expect(unknown.status).toBe(501); + expect(fixture.snapshot().errors).toHaveLength(1); + }); +}); +it('decodes only complete bounded AWS checksum chunks and refuses malformed framing', () => { + expect(decodeMaintenanceAwsChunks(Buffer.from('3\r\none\r\n3\r\ntwo\r\n0\r\nx-amz-checksum-crc32:YWJj\r\n\r\n')).toString()).toBe('onetwo'); + for (const wire of ['3\r\none', '3;signature=unknown\r\none\r\n0\r\n\r\n', '0\r\nunknown:secret\r\n\r\n']) expect(() => decodeMaintenanceAwsChunks(Buffer.from(wire))).toThrow(); +}); +it('retries only observed transient inspection states and rejects durable blockers even alongside them', () => { + const checks = [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, { reason: 'inspection.hold', state: 'none' }, + { reason: 'inspection.generation', state: 'unknown' }, { reason: 'local.tracked_work', state: 'present' }, + { reason: 'plugins.runtime_drain', state: 'not_checked' }, { reason: 'durable.snapshot', state: 'none' }, + { reason: 'deadlines.catalog', state: 'none' }, + ]; + const report = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: false, checks } } }; + expect(classifyEarlyDiagnostic(report, 90_000)).toBe('transient'); + const durable = structuredClone(report); durable.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' }); + expect(() => classifyEarlyDiagnostic(durable, 90_000)).toThrow('chat_endpoints'); + const unknown = structuredClone(report); unknown.idleSleepSafety.diagnostics.checks.push({ reason: 'deadlines.reader', state: 'unknown' }); + expect(() => classifyEarlyDiagnostic(unknown, 90_000)).toThrow('deadlines.reader'); + for (const check of report.idleSleepSafety.diagnostics.checks) if (['inspection.generation', 'local.tracked_work', 'plugins.runtime_drain'].includes(check.reason)) check.state = 'none'; + report.idleSleepSafety.diagnostics.complete = true; expect(classifyEarlyDiagnostic(report, 90_000)).toBe('strict'); +}); + +it('retains observer generation changes and still rejects a second durable report after a transient first', async () => { + const checks = [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog'].map(reason => ({ reason, state: 'none' })), + ]; + const strict = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: true, checks } } }; + const observations: unknown[][] = []; + const server = { settled: async () => ({ idle: { active: 0, generation: 10 }, managed: { apiActivityActive: true } }), + snapshot: async () => ({ idle: { active: 0, generation: 14 }, managed: { apiActivityActive: true } }) } as unknown as Parameters[0]; + const accepted = await stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => structuredClone(strict), persist: async values => { observations.push(structuredClone(values)); } }); + expect(accepted.rounds).toBe(1); expect(observations.at(-1)).toHaveLength(4); + const transient = structuredClone(strict); transient.idleSleepSafety.diagnostics.complete = false; + transient.idleSleepSafety.diagnostics.checks.find(check => check.reason === 'inspection.generation')!.state = 'unknown'; + const blocked = structuredClone(transient); blocked.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' }); + let reads = 0; + await expect(stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => reads++ ? blocked : transient, + persist: async values => { observations.push(structuredClone(values)); } })).rejects.toThrow('chat_endpoints'); + expect(reads).toBe(2); +}); diff --git a/server/src/__tests__/sleep-managed-preflight.test.ts b/server/src/__tests__/sleep-managed-preflight.test.ts new file mode 100644 index 00000000000..eef54f4fa15 --- /dev/null +++ b/server/src/__tests__/sleep-managed-preflight.test.ts @@ -0,0 +1,99 @@ +import { execFileSync } from 'node:child_process'; +import { generateKeyPairSync, randomUUID, sign } from 'node:crypto'; +import { createServer } from 'node:http'; +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js'; +import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { managedSleepConfiguration, managedSleepServerOptions, managedTenantActor, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; +import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js'; + +const requested = process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT === '1'; +const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_TEST_DATABASE_URL'); +if (requested && !databaseUrl) throw new Error('Managed preflight requires an explicit dedicated external *_test URL; no embedded allocation'); +if (requested && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Managed preflight requires retained raw evidence'); +const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT, cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA; +if (requested && (!cloudCheckout || !/^[a-f0-9]{40}$/.test(cloudSha ?? ''))) throw new Error('Managed preflight requires the exact reviewed private Cloud checkout/SHA'); + +describe.skipIf(!requested)('actual managed fixture preflight (no browser, cooldown or quiet claim)', () => { + beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000); + it('provisions through real managed auth and completes a process run before strict activity diagnostics', async () => { + if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Cloud source does not match the reviewed preflight SHA'); + const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href); + const broker = await startQualificationBroker(); const s3 = createMaintenanceS3Boundary(); + const collectorRequests: string[] = []; + const collector = createServer(async (req, res) => { for await (const _chunk of req) { /* Fully consume actual exports. */ } collectorRequests.push(req.url!); res.setHeader('content-type', 'application/json'); res.end('{}'); }); + try { + await withMaintenanceProvider(s3.server, async s3Origin => withMaintenanceProvider(collector, async origin => { + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'managed-preflight', alg: 'EdDSA', use: 'sig' }; + const configuration = managedSleepConfiguration(origin, true); + const server = await startMaintenanceServer(databaseUrl!, { ...managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker), + captureIdleWork: process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_DIAGNOSTICS === '1' }); + const fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const control = (action: string, idleCapabilities?: unknown) => { + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url'), now = Math.floor(Date.now() / 1000); + const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid }); + const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) }); + return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` }; + }; + let failure: { error: unknown } | undefined; + try { + await writeFile(path.join(server.directory, 'managed-preflight-startup.json'), JSON.stringify({ configuration, snapshot: await server.snapshot(), s3: s3.snapshot() }, null, 2)); + expect(s3.snapshot().errors).toEqual([]); + const company = await provisionManagedSleepCompany(server, fixture); + await expect(server.api('POST', '/companies', { name: 'Must remain platform managed' })).rejects.toThrow('cloud_managed'); + // Reuse the combined case's real vault/continuity/enrollment owners. + // The strict broker capability is unavailable before event-wake enrollment. + const github = await server.prepareGitHub(company.id); + await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true }); + const githubContinuity = await server.settleGitHub(github); + await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding: await broker.binding(), boundary: broker.boundary }, null, 2)); + await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1); + const brokerBinding = await broker.binding(); + await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding, boundary: broker.boundary }, null, 2)); + expect(brokerBinding?.instanceId).toBe(broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID); + expect(brokerBinding?.bindings).toEqual([[`${github.grantId}_101`, company.id, managedTenantActor.userId, + 'github.code', '101', github.connectionId, github.grantId]]); + const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Managed preflight process', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', "console.log('managed-preflight-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Actual managed preflight key' }); + const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'One actual managed process run', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` }); + await expect.poll(async () => { const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; return run?.status; }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${issue.id}`; expect(executions.count).toBe(1); + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); const apiSucceededAt = performance.now(); + const activity = await server.snapshot(); + await writeFile(path.join(server.directory, 'managed-preflight-setup.json'), JSON.stringify({ company, githubContinuity, brokerBinding, + agentId: agent.id, keyId: key.id, issueId: issue.id, executions: executions.count, activity, s3: s3.snapshot() }, null, 2)); + expect(activity.managed?.apiActivityActive).toBe(true); + await server.settled(500, 0, { managedSetupRecovery: true }); + const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + try { + const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const read = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + const proof = await stabilizeEarlyDiagnostics(server, { ownerId: drain.ownerId, apiSucceededAt, read, + persist: observations => writeFile(path.join(server.directory, 'managed-preflight-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) }); + await writeFile(path.join(server.directory, 'managed-preflight-receipt.json'), JSON.stringify({ companyId: company.id, proof, s3: s3.snapshot(), collectorRequests, + boundary: 'Actual managed startup/provisioning/key activity/process outcome; no browser, real cooldown/quiet, eligibility, provider suspension or durable S3 claim' }, null, 2)); + } finally { await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop')); } + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + try { await writeFile(path.join(server.directory, 'managed-preflight-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + try { await server.stop(); } catch (error) { errors.push(error); } + try { await fixture.$client.end({ timeout: 1 }); } catch (error) { errors.push(error); } + try { await writeFile(path.join(server.directory, 'managed-preflight-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Managed preflight or owned cleanup failed'); + } + })); + } finally { await broker.stop(); } + }, 180_000); +}); diff --git a/server/src/__tests__/sleep-otel-export.test.ts b/server/src/__tests__/sleep-otel-export.test.ts new file mode 100644 index 00000000000..f8276760850 --- /dev/null +++ b/server/src/__tests__/sleep-otel-export.test.ts @@ -0,0 +1,43 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { createServer } from 'node:http'; +import { it, expect } from 'vitest'; +import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; + +// Opt-in qualification must fail when the SDK falls back to no-op. The ordinary +// bootstrap tests separately retain coverage for the supported missing-peer mode. +it.skipIf(process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE !== '1')('exports an actual named span with no metric or log traffic over 90 real seconds', async () => { + const spans: string[] = [], paths: string[] = []; + const collector = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + paths.push(req.url ?? 'missing-path'); + const envelope = JSON.parse(Buffer.concat(chunks).toString('utf8')); + for (const resource of envelope.resourceSpans ?? []) for (const scope of resource.scopeSpans ?? []) for (const span of scope.spans ?? []) spans.push(span.name); + res.setHeader('Content-Type', 'application/json'); res.end('{}'); + }); + await withMaintenanceProvider(collector, async origin => { + const startedAt = performance.now(); + const { stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url)), '--observe-exporters'], { + env: { PATH: process.env.PATH, NODE_ENV: 'production', OTEL_EXPORTER_OTLP_ENDPOINT: origin, OTEL_EXPORTER_OTLP_PROTOCOL: 'http/json' }, + timeout: 110_000, maxBuffer: 2 * 1024 * 1024, + }); + expect(performance.now() - startedAt).toBeGreaterThanOrEqual(90_000); + expect(stderr).not.toContain('Continuing without tracing'); + expect(spans).toContain('sleep-qualification-stimulus'); + expect(paths).not.toContain('/v1/metrics'); + expect(paths).not.toContain('/v1/logs'); + }); +}, 120_000); + + +it('keeps the optional SDK and exporters inactive without an endpoint', async () => { + const { stdout, stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url))], { + env: { PATH: process.env.PATH, NODE_ENV: 'production' }, timeout: 15_000, maxBuffer: 2 * 1024 * 1024, + }); + expect(stdout).not.toContain('OpenTelemetry service.version='); + expect(stderr).not.toContain('Continuing without tracing'); +}); diff --git a/server/src/__tests__/voice-session-store.integration.test.ts b/server/src/__tests__/voice-session-store.integration.test.ts index 70903f3b786..8c0bb1f1d6c 100644 --- a/server/src/__tests__/voice-session-store.integration.test.ts +++ b/server/src/__tests__/voice-session-store.integration.test.ts @@ -1892,8 +1892,28 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }); it("retains one safe report per call and rejects history after access revocation", async () => { const f = await nativeServiceFixture(), first = await f.service.start(f.input); + const foreign = await nativeServiceFixture(); + const [foreignSession] = await db.select().from(chatVoiceSessions).where(eq(chatVoiceSessions.id, foreign.sessionId)); + await db.insert(chatVoiceReports).values({ companyId: foreign.companyId, sessionId: foreign.sessionId, nextCheckAt: new Date(0) }); + const foreignReports = await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId)); + const reportProviderIds: string[] = []; + const fetchReport = f.transport.callReport as (providerSessionId: string) => ReturnType; + const reports = voiceCallReports(db, session => { + reportProviderIds.push(session.providerSessionId!); + return fetchReport(session.providerSessionId!); + }); + expect(await reports.load({ kind: "company", id: foreign.companyId })).toEqual([ + expect.objectContaining({ id: foreign.sessionId, companyId: foreign.companyId, dueAt: 0 }), + ]); + // This case owns report/history behavior for its company. Global recovery + // also visits every predecessor fixture's live calls, replies and reports. + const reconcileReports = async () => { + const due = (await reports.load({ kind: "company", id: f.companyId })) + .filter(row => row.dueAt <= Date.now()).sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id)); + for (const row of due) await reports.check(row.id); + }; await f.service.end(f.companyId, first.session.id, f.input.caller); - await f.service.reconcile(1000); + await reconcileReports(); const detail = await f.service.callDetail(f.companyId, first.session.id, f.input.caller); expect(detail.report).toMatchObject({ status: "available", costMicroUsd: "120000", transcript: [{ id: "turn1", text: "A test request" }] }); expect(await f.service.history(f.companyId, f.endpointId, f.input.caller)).toEqual(expect.arrayContaining([detail])); @@ -1908,12 +1928,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const baseline = await f.transport.callReport(), latest = new Date(Date.now() + 10000); let response = { ...baseline, costMicroUsd: "240000", providerUpdatedAt: latest }; f.transport.callReport.mockImplementation(async (providerSessionId?: string) => providerSessionId === stored.providerSessionId ? response : baseline); - await f.service.reconcile(1000); + await reconcileReports(); expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000"); expect((await f.service.callDetail(f.companyId, other.session.id, f.input.caller)).report.costMicroUsd).toBe("120000"); await db.update(chatVoiceReports).set({ nextCheckAt: new Date(0) }).where(eq(chatVoiceReports.sessionId, first.session.id)); response = { ...baseline, costMicroUsd: "1", providerUpdatedAt: new Date(0) }; - await f.service.reconcile(1000); + await reconcileReports(); expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000"); expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.sessionId, first.session.id))).toHaveLength(1); await expect(f.service.callDetail(f.companyId, first.session.id, { id: randomUUID(), authority: "member" })).rejects.toMatchObject({ status: 403 }); @@ -1922,6 +1942,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await expect(f.service.history(f.companyId, f.endpointId, f.input.caller)).rejects.toMatchObject({ status: 403 }); await expect(f.service.callDetail(f.companyId, first.session.id, f.input.caller)).rejects.toMatchObject({ status: 403 }); expect(f.transport.createPhoneSession).not.toHaveBeenCalled(); + expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId))).toEqual(foreignReports); + expect(reportProviderIds).toContain(stored.providerSessionId); + expect(reportProviderIds).not.toContain(foreignSession.providerSessionId); }); it("does not create provider media when membership is revoked during credential resolution", async () => { const f = await nativeServiceFixture(); diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index b5aba0a7f0e..4e4ca127631 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -14,6 +14,7 @@ import { agents, companies, createDb, + closeRegisteredClients, executionWorkspaces, heartbeatRuns, issueComments, @@ -87,6 +88,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; const RUNTIME_OWNED_GIT_BRANCH_METADATA = { createdByRuntime: true, @@ -134,8 +136,12 @@ function workspaceBranchIncoherenceFingerprintForTest(input: { } const leasedRunIds = new Set(); -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); -const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const externalRuntimeControlDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_RUNTIME_CONTROL_TEST_DATABASE_URL"); +const embeddedPostgresSupport: Awaited> = externalRuntimeControlDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +// The explicit fixture belongs only to the service-control suite. Other DB +// suites must not allocate an embedded or admin-owned database in this mode. +const describeEmbeddedPostgres = !externalRuntimeControlDatabaseUrl && embeddedPostgresSupport.supported ? describe : describe.skip; +const describeRuntimeControl = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( @@ -6672,16 +6678,18 @@ describeEmbeddedPostgres("workspace dirty quarantine branch repair", () => { }, 20_000); }); -describeEmbeddedPostgres("workspace runtime service control persistence", () => { +describeRuntimeControl("workspace runtime service control persistence", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-"); - db = createDb(tempDb.connectionString); + if (!externalRuntimeControlDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-"); + db = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString, + externalRuntimeControlDatabaseUrl ? { maxConnections: 4 } : undefined); }, 20_000); afterAll(async () => { + if (externalRuntimeControlDatabaseUrl) await closeRegisteredClients(externalRuntimeControlDatabaseUrl, { scope: "database" }); await tempDb?.cleanup(); }); @@ -7284,14 +7292,14 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => return result[0]!; })); - // The two lanes claim ports concurrently in-process. The allocator guarantees - // distinct ports, but not which lane wins the base port. It depends on the - // scheduling order. So assert order-independent invariants, not array index. + // A free-port probe is not a reservation. Both siblings must own distinct + // live ports in the allocator's bounded range even if the preferred port + // becomes unavailable before allocation. const lowerPort = Math.min(first.port!, second.port!); const upperPort = Math.max(first.port!, second.port!); const maxAllocatablePort = basePort + WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS - 1; expect(first.port).not.toBe(second.port); - expect(lowerPort).toBe(basePort); + expect(lowerPort).toBeGreaterThanOrEqual(basePort); expect(upperPort).toBeGreaterThan(basePort); expect(upperPort).toBeLessThanOrEqual(maxAllocatablePort); expect(first.url).toBe(`http://127.0.0.1:${first.port}`); @@ -7333,7 +7341,7 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => it("commits fixed-port reservations across the configured database idle timeout", async () => { const fixture = await createRuntimeFixture(); - const readinessDb = createDb(tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 }); + const readinessDb = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 }); const [initialConnection] = await readinessDb.$client<{ pid: number }[]>`select pg_backend_pid() as pid`; const cleanupRuntimeHome = await createRuntimeHome(); const workspace = fixture.workspaces[0]!; diff --git a/server/src/instrumentation.ts b/server/src/instrumentation.ts index ac632a57e15..d26b09ceae4 100644 --- a/server/src/instrumentation.ts +++ b/server/src/instrumentation.ts @@ -499,6 +499,10 @@ async function bootstrapOtel(endpoint: string): Promise { console.log(`[paperclip] OpenTelemetry service.version=${serviceVersion}`); const sdk = new NodeSDK({ + // This integration exports traces only. NodeSDK otherwise creates + // default OTLP metric/log exporters even when none were configured. + metricReaders: [], + logRecordProcessors: [], resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME || "paperclip", [ATTR_SERVICE_VERSION]: serviceVersion, diff --git a/tests/e2e/disconnect-write-recovery.spec.ts b/tests/e2e/disconnect-write-recovery.spec.ts index 1d7cc5d2fe1..1450aec855d 100644 --- a/tests/e2e/disconnect-write-recovery.spec.ts +++ b/tests/e2e/disconnect-write-recovery.spec.ts @@ -178,8 +178,9 @@ test("replaying an accepted comment does not reopen completed work", async ({ pa const activity = await json(await request.get(`/api/issues/${issue.id}/activity`)); expect(activity.filter((entry: { action: string; details?: { commentId?: string } }) => entry.action === "issue.comment_added" && entry.details?.commentId === first.id)).toHaveLength(1); - await page.goto(`/${company.issuePrefix}/issues/${issue.id}`); - await expect(page.getByRole("heading", { name: issue.title, exact: true })).toBeVisible(); + await page.goto(`/${company.issuePrefix}/issues/${issue.identifier}`); + await expect(page.getByTestId("task-chat-history-content")).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId("issue-detail-header").getByRole("heading", { name: issue.title, exact: true })).toBeVisible(); await expect(page.getByText(body, { exact: true })).toBeVisible(); } finally { await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } }); diff --git a/tests/e2e/task-loading-frame.spec.ts b/tests/e2e/task-loading-frame.spec.ts index f607b245059..e784e61dfc1 100644 --- a/tests/e2e/task-loading-frame.spec.ts +++ b/tests/e2e/task-loading-frame.spec.ts @@ -155,13 +155,14 @@ for (const coordinator of [false, true]) test(`mobile chrome stays fixed when pe let release!: () => void; const preferenceReady = new Promise(resolve => { release = resolve; }); await page.addInitScript(() => { - const frames: number[][] = []; - Object.assign(window, { mobileShellFrames: frames }); + const frames: number[][] = [], frameTimes: number[] = []; + Object.assign(window, { mobileShellFrames: frames, mobileShellFrameTimes: frameTimes }); const sample = () => { const shell = document.querySelector(".paperclip-app-shell"); if (shell && shell.getBoundingClientRect().width > 0) { const rects = [".app-breadcrumb", ".mobile-bottom-nav", "#main-content"].map(selector => document.querySelector(selector)!.getBoundingClientRect()); frames.push([innerWidth, innerHeight, scrollX, scrollY, ...rects.flatMap((rect, index) => index === 2 ? [rect.x, rect.y, rect.width] : [rect.x, rect.y, rect.width, rect.height])]); + frameTimes.push(performance.now()); } requestAnimationFrame(sample); }; @@ -184,9 +185,27 @@ for (const coordinator of [false, true]) test(`mobile chrome stays fixed when pe release(); if (coordinator) await expect(page.getByTestId("task-chat-thread").locator(':scope > [aria-busy="false"]')).toBeVisible(); else await expect(page.getByTestId("issue-detail-header").getByRole("heading", { name: fixture.title, exact: true })).toBeVisible(); - await page.waitForTimeout(300); - const frames = await page.evaluate(() => (window as unknown as { mobileShellFrames: number[][] }).mobileShellFrames); - await info.attach("mobile-preference-chrome", { body: JSON.stringify(frames), contentType: "application/json" }); + const ready = await page.evaluate(() => ({ + count: (window as unknown as { mobileShellFrames: number[][] }).mobileShellFrames.length, + at: performance.now(), + })); + let frames: number[][] = []; + try { + // Observe six real post-ready frames and preserve the original 300ms + // interval. Wall time alone does not establish that rAF sampled it. + await expect.poll(() => page.evaluate(({ count, at }) => { + const state = window as unknown as { mobileShellFrames: number[][]; mobileShellFrameTimes: number[] }; + return state.mobileShellFrames.length >= count + 6 && (state.mobileShellFrameTimes.at(-1) ?? 0) >= at + 300; + }, ready)).toBe(true); + } finally { + const captured = await page.evaluate(() => { + const state = window as unknown as { mobileShellFrames: number[][]; mobileShellFrameTimes: number[] }; + return { frames: state.mobileShellFrames, times: state.mobileShellFrameTimes }; + }); + frames = captured.frames; + await info.attach("mobile-preference-chrome", { body: JSON.stringify(frames), contentType: "application/json" }); + await info.attach("mobile-preference-frame-times", { body: JSON.stringify({ ready, times: captured.times }), contentType: "application/json" }); + } expect(frames.length).toBeGreaterThan(5); for (let dimension = 0; dimension < frames[0].length; dimension++) { const values = frames.map(frame => frame[dimension]); From 8771606fd87f69ed4057779e3dce3a9bca8088e6 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:47:13 -0600 Subject: [PATCH 02/28] test(e2e): cover browser task editing and reopening (#18782) ## Thinking Path > - Paperclip lets people manage tasks and agent work. > - Daily QA uses reusable Product E2E journeys against a named target. > - Existing task journeys cover creation and assignment. > - They lack saved title and label edits across completion and reopening. > - This pull request adds one browser-only lifecycle journey with independent saved-state checks. > - It also puts shared registry entries on separate lines so independent case PRs can use distinct insertion points. > - The registry formatting preserves every token, value, entry order and case body. ## Linked Issues or Issue Description Refs: #18655. **What existing behavior does this improve?** Product E2E coverage of ordinary task editing and completion. **Current behavior** The reusable catalog covers task creation and assignment. It lacks one journey that edits a task, reloads it, completes it and reopens it while retaining its saved context. Several shared registries put all entries on one line, so independent case additions conflict. **Proposed behavior** Create a task assigned to the signed-in person. Edit its title and create one label through the visible UI. Change its status to In Progress, reload it, mark it Done, reload it, reopen it to Todo and reload it again. Require the exact saved task and label throughout. Preserve its description, assignee, project and priority. Allow no agent run or model usage. Put the existing shared registry entries on separate lines without changing their contents or order. ## What Changed - Add `task-edit-lifecycle` and six required checks for browser creation, saved edits, reload, completion and reopening, preserved context and no agent work. - Add valid, wrong and missing evidence controls. Retain allowlisted observations and bounded GET failure receipts in private evidence. - Register the case, feature associations, catalog expectation, source-hash inputs and browser-only result validation. - Extend target ownership for run-owned labels and one exact predeclared title change. Require compatible scoped cleanup before selecting the case. - Open the real side panel or Properties control when needed. Exclude inert and aria-hidden Labels rows and require visible readback. Document inline title editing and the case prerequisites. - Keep priority editing disabled. Verify that the saved priority remains unchanged. - Keep Cursor test observations outside its existing Stop deadline, preserve the full rejection/Stop overlap matrix and add pre-release no-provider-call controls. Production behavior and deadline values stay unchanged. - Format five shared registry files with one entry per line: the JourneyId and AgentJourney unions, definition filenames, expected catalog IDs and agent run counts. No registry entry moves in this PR. ## Verification - Current head is `d1af3aae3712455c90db937fa7a5141555710506`. Its only change from integrated head d40f28 is the Cursor driver test-fixture ordering repair described below. The prior normal merge consumes actual master `d59791828482f213713ca55000c40e6c9c6c03e5`, including reviewed UI repair #18848 and the existing idle-history fixture repair from #18853. It has both real parents, no conflicts and the required trailer. All seven lifecycle case/oracle/catalog/documentation blobs and six selected registry constructs remain exact 10625b. The idle-history fixture is byte-identical to master. The PR diff has 13 paths, including the one Cursor test file, and no feature-map or production-driver changes. - 167 focused tests pass across catalog, target contracts, agent evidence, lifecycle evidence and observation reads. E2E TypeScript and credential-free discovery of21 Product E2E cases pass. The real merged idle-history and exposure-reservation suites pass all 109 tests in132 seconds, including the owned HTTP-process adoption assertion and explicit unknown-listener denial/drift controls. Canonical shared/SDK, database and Runner TypeScript dependencies were rebuilt for the changed master inputs. All4,220 tracked package/script/root inputs and 4,954 dependency outputs are recorded; the outputs remain unchanged after the tests and browser journey. The first database build stopped because child PATH selected a different global pnpm and its version switch could not fetch signed metadata. The preserved attempt made no package or source change. Using the already-installed verified pnpm9 on child PATH passed. No install, Rust or provider-build global setup ran. - One fresh canonical managed-local browser journey passes on exact d40f28: all six checks and authentication, setup, journey, collection and cleanup pass. Cleanup releases the disposable instance. The journey takes 12.059 seconds and has explicit zero-run readback at all five saved states. Its runtime commit isd40f28. Definition hash is `f262707de2e544cecdb9c5c4c5222d61f04a122575b438658dec069e1a93f9cb`. The prior10625b local6-check/5-phase pass remains separately recorded with its own source and definition hash. - A real closed-panel control previously reproduced the original label-interaction failure using only the presentation preference `paperclip:panel-visible=false`. The corrected journey passed all six unchanged checks through an ordinary Show properties click. A separate canonical743 local run passed all six checks and five phases with released cleanup. No domain outcome was supplied by that control. - A separate QA staging qualification passed all six checks and five phases on exact743, with closed cleanup and verified zero agent work. That original pass remains bound to743. It is not regraded or relabeled as a staging run of10625b. Daily admission remains a separate coordinator action. - Earlier material failures remain part of the verification record. The first managed-local setup failed before authentication with an embedded PostgreSQL exit code of null; its original cause remains unresolved. A later loader diagnostic found missing library aliases in this isolated checkout. Existing bundled libraries were hash-verified and the aliases were restored only there. Subsequent real runs exposed an overlong label name, a status picker name mismatch and the closed-panel issue. Each failed result and cleanup record is retained. The fixes preserve the full campaign name, validate the production48-character label limit before mutations, and use the actual visible controls. - Original full CI [41652807327](https://github.com/paperclipai/paperclip/actions/runs/41652807327) failed an unchanged Discord fixture assertion, attention-retention fixture limits and the CLI import/export startup limit. One bounded recheck recovered Discord but repeated the other failures. The reviewed merged retention-fixture repair was consumed normally through #18770. Full CI [41667811850](https://github.com/paperclipai/paperclip/actions/runs/41667811850) passed all E2E shards but exposed inbox/account-fixture failures. This branch consumed the actual reviewed repairs from #18798 and #18799. Original logs and failed assertions remain retained; no timeout or oracle was weakened. - Original743 CI [41682757346](https://github.com/paperclipai/paperclip/actions/runs/41682757346) failed the unchanged CLI suite's30-second child-server startup hook. A bounded local phase probe passed the same startup oracle but did not prove the original CI cause. One coordinator-approved official recheck followed. Timestamp comparison shows46 non-skipped jobs retained their original start and end times; only the failed CLI job and final aggregate have new start and end times. The skipped selection job remains unclassified. New GitHub job IDs alone did not prove new executions. The original startup failure remains preserved and its cause remains unproven. - Formatting-head CI [41726104217](https://github.com/paperclipai/paperclip/actions/runs/41726104217) failed the upstream idle-history process-adoption test and dependent verify job:46 jobs passed, one was skipped and two failed. Its actual synthetic checkout had parents masterf187 and10625b; the failed test is absent from 10625b itself. The assertion expected one adopted process and received zero. A pure production-function control proves a conditional reserved-port drift path, but the actual failed CI port and full reconciliation counters were not logged. The actual CI cause remains unproven. This branch consumes the already-reviewed bounded fixture-port repair in #18853 through its normal actual-master merge. The original failure is retained. No unchanged-code retry, timeout increase, skipped assertion or ownership-policy change was used. - Integrated-head CI [41730231620](https://github.com/paperclipai/paperclip/actions/runs/41730231620) passed the formerly failing server1/12 lane, Typecheck, Build and E2E. It failed the Cursor package fixture below and the dependent verify job. A separate server4/12 job lost runner communication; the exact runner instance and Spot-request readback confirm a capacity interruption. Its unavailable log is not treated as a test pass. The original failures remain retained. No old-head retry was made. - The Cursor fixture sets a 300 ms cancellation budget, holds a rejected-create checkpoint and originally performed directory/file observations while that clock ran. Controlled400ms observation latency on the original fixture reproduces the same fenced Stop error. The same diagnostic passes after moving file observations before the cancellation clock, or while the initial Stop-checkpoint gate still prevents it from starting. The driver,300ms configuration and timeout negatives are unchanged. Every original assertion AST remains, and three new assertions forbid provider observation/cancel calls before releasing the checkpoint. All81 driver tests,12 client tests and43 checkpoint-archive tests pass; Runner TypeScript build and E2E TypeScript pass. The actual CI inner error and observation duration were not logged, so its precise original cause remains unproven. - Final d1af3a full CI [41733938939](https://github.com/paperclipai/paperclip/actions/runs/41733938939) attempt3 is successful. Attempt1 lost four exact-bound Spot runners; attempt2 recovered the Runner and both server lanes but lost the replacement E2E runner to another confirmed Spot capacity interruption. Two separate coordinator-authorized official failed-job rechecks were recorded. Attempt3 passed the E2E path and final aggregate. Successful executions retain their original timestamps; new job IDs alone were not used as proof of re-execution. All original failures remain retained. - Fresh Greptile check126258142754 completed successfully on exact d1af3a with 5/5 and zero unresolved threads. Full recursive workspace typecheck, all workspace tests and the full build were not repeated locally; the corresponding hosted full CI gates pass. ## Risks The case requires target support for `label` ownership and scoped `taskEdit.revisedTitle` cleanup. Older target adapters must not select it. Cleanup must reconcile lost creation responses and exact predeclared title saves without touching other resources. Registry formatting changes the harness definition hash because that hash includes source bytes. Historical runs must keep their original source and hash. The formatting does not change runtime values, order, dispatch, case checks or product behavior. Later case PRs must verify their actual merge resolutions and final-head checks before admission. ## Model Used OpenAI GPT-6 through Codex with reasoning, tool use and code execution. The runtime does not expose a more specific model ID or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../cursor-cloud/cursor-cloud-driver.test.ts | 17 +- tests/runner-e2e/PRODUCT-JOURNEYS.md | 36 ++++ tests/runner-e2e/catalog.test.ts | 24 ++- tests/runner-e2e/journey-features.ts | 1 + .../product-journeys/agent-flows.ts | 8 +- tests/runner-e2e/product-journeys/catalog.ts | 1 + tests/runner-e2e/product-journeys/launch.ts | 71 ++++++- .../product-journeys/success-contract.ts | 17 +- .../task-lifecycle-evidence.test.ts | 146 +++++++++++++++ .../task-lifecycle-evidence.ts | 98 ++++++++++ .../product-journeys/task-lifecycle.ts | 173 ++++++++++++++++++ tests/runner-e2e/product-journeys/types.ts | 27 ++- tests/runner-e2e/result-validation.ts | 4 +- 13 files changed, 610 insertions(+), 13 deletions(-) create mode 100644 tests/runner-e2e/product-journeys/task-lifecycle-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/task-lifecycle-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/task-lifecycle.ts diff --git a/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts b/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts index 532faaf17b6..219f45cd95d 100644 --- a/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts @@ -994,21 +994,32 @@ describe("Cursor Cloud native driver", () => { try { const turn = await f.session.startTurn({ message: { role: "user", text: rejection === "unsent_payload" ? "\u0001".repeat(400_000) : "Rejected task" } }); await entered.promise; + const observeHeldCheckpoint = async () => { + const checkpointPath = join(f.root, (await readdir(f.root)).find(name => /^checkpoint-.*\.json$/.test(name))!); + expect(JSON.parse(await readFile(checkpointPath, "utf8")).phase).not.toBe("idle"); + return checkpointPath; + }; + let checkpointPath: string; if (overlap === "stop_write") { stop(); await stopHeld.promise; rejectCreate.resolve(); await new Promise(accept => setImmediate(accept)); + // The Stop checkpoint gate still holds its cancellation clock here. + checkpointPath = await observeHeldCheckpoint(); releaseStop.resolve(); - await waitFor(() => pausedRejection || stopDone); + await Promise.race([rejectionHeld.promise, stopped!]); expect(pausedRejection, "the definite rejection must reach durable settlement even while stopping").toBe(true); } else { rejectCreate.resolve(); await rejectionHeld.promise; + // File observations must not consume Stop's 300ms settlement budget. + checkpointPath = await observeHeldCheckpoint(); stop(); await new Promise(accept => setImmediate(accept)); } expect(stopDone, "Stop must wait for the rejected-create checkpoint").toBe(false); expect(await f.session.read()).toMatchObject({ phase: "stopping" }); await expect(f.session.startTurn({ message: { role: "user", text: "Too early" } })).rejects.toThrow("unresolved remote work"); - const checkpointPath = join(f.root, (await readdir(f.root)).find(name => /^checkpoint-.*\.json$/.test(name))!); - expect(JSON.parse(await readFile(checkpointPath, "utf8")).phase).not.toBe("idle"); + expect(peer.getAgent).toHaveBeenCalledTimes(before.getAgent); + expect(peer.getRun).toHaveBeenCalledTimes(before.getRun); + expect(peer.cancelRun).toHaveBeenCalledTimes(before.cancel); releaseRejection.resolve(); expect(await stopped).toEqual({ error: null }); await waitFor(() => f.events.some(event => event.turnId === turn.turnId && event.eventType === "turn.failed" && event.payload.remoteWorkAccepted === false)); diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 42d8c6cde70..9bc8ca779ab 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -33,6 +33,7 @@ never start a server or depend on a tenant filesystem. | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | | `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | +| `task-edit-lifecycle` | Company, signed-in board user, task-write capability; target support for run-owned labels and exact predeclared `taskEdit.revisedTitle` cleanup; zero model runs | Create a self-assigned task, edit its inline header title and label, reload, complete and reopen it; independently verify exact saved state, unchanged context and zero runs. Open a closed Properties panel through its shipped toggle | | `task-document-persistence` | Company, signed-in board user, task-write capability, prepared run-owned backlog task; no model allowance or runs | Create a general document through Artifacts, edit its title and body, reopen and reload it; independently verify exact canonical content, preserved original and latest revisions, board-user authorship, and no agent work | | `task-search-filter` | Company, signed-in board user, task-write capability; zero model runs | Search three owned target/control tasks through the task list, require exact matching identities, filter Backlog to exclude the completed control, reload and return from the exact saved task with search/filter context intact | | `plan-revise-accept` | Native agent; three work runs | Reuse the classic Plan story: canonical draft, browser revision request, new revision-bound confirmation, approval and exact saved final response | @@ -488,3 +489,38 @@ within the original journey deadline. Binary attachment and browser downloads remain single attempts; transport failures and digest mismatches fail visibly. Git inspection consumes the exact digest-verified download buffer; browser downloads remain an independent result-access check. The repository journey shares the observation reader and browser helpers above. + + +## Task editing and lifecycle + +`task-edit-lifecycle` creates a self-assigned task through the browser with an +explicit title. It edits the title inline in the task header and adds one +run-owned label through the Properties panel. If the panel starts closed, the +journey opens it through the ordinary Toggle side panel or Show properties +control. It changes the status to In Progress, reloads, marks +the task Done, reloads, reopens it to Todo and reloads again. Independent API +reads bind every observed state to the same company, task and human owner. +The exact description, assignee, project and priority remain unchanged. +Priority editing is not part of this journey because the production UI hides +that control. No agent or model work is allowed. + +Labels use `Journey QA label ${campaignId}`, retaining the complete campaign +identity within the production 48-character limit for ordinary generated +campaigns. The production schema rejects an overlong name before any browser +mutation or ownership intent; names are never truncated or hashed. +The target must journal `label` resources by campaign name before browser label +creation and by exact ID after its 201 response. Before title editing, the owned +issue receipt retains its original name and ID and declares +`taskEdit: { revisedTitle }` for this case. Cleanup may match only that one +predeclared title or the original title on the same company and issue ID. +The private target must reconcile a lost label-creation response, remove only +its proven run-owned label, and verify absence. Targets without these ownership +and cleanup semantics must not select this journey. + +The case uses no private fixture writes. Its limits require zero model calls and +zero task runs. `private-stories/task-edit-lifecycle/task-lifecycle-observations.json` +retains the five allowlisted task observations. The adjacent +`observation-read-failures.json` retains bounded read-transport failures. +Valid, wrong and missing evidence +controls calibrate the independent oracle. Registration and local checks do not +claim a Cloud staging pass or admission to a daily plan. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index f5c240c0df4..5f08f93b832 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -126,7 +126,29 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["company-access-isolation", "repository-continuation", "routine-scheduled-run", "inbox-triage", "file-input-answer", "task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual([ + "task-edit-lifecycle", + "company-access-isolation", + "repository-continuation", + "routine-scheduled-run", + "inbox-triage", + "file-input-answer", + "task-create", + "project-task-context", + "ask-question", + "question-answer-resume", + "task-properties", + "task-document-persistence", + "task-search-filter", + "plan-revise-accept", + "build-revise", + "protected-actions", + "create-skill-studio", + "stop-redirect", + "followup-while-running", + "hire-delegate-reuse", + "cloud-onboarding" + ]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 962117f2ab1..18cf44b1f33 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -28,6 +28,7 @@ associate(["product-journeys"], { "followup-while-running": ["steering/queue"], "hire-delegate-reuse": ["agents/hire", "documents-artifacts/documents"], "project-task-context": ["projects/lifecycle", "projects/task-context", "tasks/create"], + "task-edit-lifecycle": ["tasks/properties", "tasks/completion"], "task-properties": ["tasks/properties"], "task-document-persistence": ["documents-artifacts/documents"], "task-search-filter": ["tasks/organize"], "cloud-onboarding": ["onboarding/first-task"] }); associate(["antigravity-agent-journey"], { "saved-subscription-two-turn": ["agents/hire", "agents/instructions", "runs-adapters/validation", "runs-adapters/session", diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index 96d0fd8d713..5986c9ca143 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -20,7 +20,13 @@ import { agentJourneyRuns } from "./catalog.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 1e1f2cabf36..7de9ae41786 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -2,6 +2,7 @@ import { agentJourneyRuns } from "./success-contract.js"; export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ + { id: "task-edit-lifecycle", label: "Edit task properties, complete and reopen with saved context", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/properties", "tasks/completion"], paid: false }, { id: "company-access-isolation", label: "Switch QA workspaces and verify restricted agent isolation", requires: ["company", "board-user", "task-write", "company-access"], capabilities: ["companies/selection", "access/resources"], paid: false }, { id: "repository-continuation", label: "Continue repository work with the same saved workspace", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["projects/task-context", "documents-artifacts/attachments"], paid: true }, { id: "routine-scheduled-run", label: "Run a scheduled routine once and verify pause and saved history", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["routines/definition", "routines/triggers", "routines/history", "routines/management"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 6482fc9580a..705946b8503 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -9,6 +9,7 @@ import { runnerSuites } from "../catalog.js"; import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney } from "./catalog.js"; import { agentJourneyRunCountValid } from "./success-contract.js"; import { journeyFixtures } from "./fixtures.js"; +import { taskEditLifecycle } from "./task-lifecycle.js"; import { taskProperties } from "./task-properties.js"; import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; @@ -113,7 +114,73 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["observation-read.ts", "../connection-config.ts", "catalog.ts", "company-access-flow.ts", "company-access-contract.ts", "repository-continuation-flow.ts", "repository-continuation-contract.ts", "repository-continuation-evidence.ts", "../registered-artifact.ts", "../user-actions.ts", "../connection-target.ts", "../../../packages/shared/src/project-mentions.ts", "routine-schedule-flow.ts", "routine-schedule-evidence.ts", "input-file-flow.ts", "input-file-evidence.ts", "inbox-triage-flow.ts", "inbox-triage-evidence.ts", "inbox-triage-fixture.ts", "inbox-author-contract.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of [ + "task-lifecycle.ts", + "task-lifecycle-evidence.ts", + "../../../packages/shared/src/validators/issue.ts", + "observation-read.ts", + "../connection-config.ts", + "catalog.ts", + "company-access-flow.ts", + "company-access-contract.ts", + "repository-continuation-flow.ts", + "repository-continuation-contract.ts", + "repository-continuation-evidence.ts", + "../registered-artifact.ts", + "../user-actions.ts", + "../connection-target.ts", + "../../../packages/shared/src/project-mentions.ts", + "routine-schedule-flow.ts", + "routine-schedule-evidence.ts", + "input-file-flow.ts", + "input-file-evidence.ts", + "inbox-triage-flow.ts", + "inbox-triage-evidence.ts", + "inbox-triage-fixture.ts", + "inbox-author-contract.ts", + "task-properties.ts", + "task-search-filter.ts", + "task-search-evidence.ts", + "basic-flows.ts", + "basic-evidence.ts", + "task-document-flow.ts", + "task-document-evidence.ts", + "project-context-flow.ts", + "project-context-readback.ts", + "project-context-evidence.ts", + "agent-flows.ts", + "hire-delegate-flow.ts", + "hire-delegate-composer.ts", + "hire-delegate-evidence.ts", + "active-followup-flow.ts", + "active-followup-evidence.ts", + "stop-redirect-evidence.ts", + "clarification-flow.ts", + "clarification-evidence.ts", + "../question-resume-ui.ts", + "../question-resume-scoring.ts", + "../runtime-question-readiness.ts", + "../chat-flow.ts", + "../continuation-readiness.ts", + "../continuation-cases.ts", + "success-contract.ts", + "product-screenshot.ts", + "cloud-onboarding.ts", + "onboarding-connect.ts", + "onboarding-first-task.ts", + "workspace-deferral.ts", + "onboarding-pre-acceptance.ts", + "artifact-evidence.ts", + "fixtures.ts", + "launch.ts", + "types.ts", + "collection.ts", + "local-target.ts", + "../../../ui/src/lib/markdown-reply-content.ts", + "../../../ui/src/lib/issue-reference.ts", + "../../../ui/src/lib/company-routes.ts", + "../../../ui/package.json" + ]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "chat-hardening.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -234,6 +301,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri } }); } else if (id === "company-access-isolation") await companyAccessIsolation(ready, session!, campaignId, evidence); else if (id === "inbox-triage") await inboxTriage(ready, session!, campaignId, evidence); + else if (id === "task-edit-lifecycle") await taskEditLifecycle(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); else if (id === "task-document-persistence") await taskDocumentPersistence(ready, session!, campaignId, evidence); else if (id === "project-task-context") await projectTaskContext(ready, session!, campaignId, evidence); diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 32c2eca9b47..91b14865b2e 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,6 +1,18 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ -export const agentJourneyRuns = { "repository-continuation": 2, "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3, "hire-delegate-reuse": 7 } as const; +export const agentJourneyRuns = { + "repository-continuation": 2, + "routine-scheduled-run": 1, + "file-input-answer": 1, + "ask-question": 1, + "plan-revise-accept": 3, + "build-revise": 2, + "followup-while-running": 2, + "protected-actions": 4, + "create-skill-studio": 1, + "stop-redirect": 3, + "question-answer-resume": 3, + "hire-delegate-reuse": 7 +} as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; @@ -9,6 +21,7 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { + "task-edit-lifecycle": ["task-lifecycle-browser-created", "task-lifecycle-visible-edits-persisted", "task-lifecycle-edits-reloaded", "task-lifecycle-completed-and-reopened", "task-lifecycle-unedited-context-preserved", "no-agent-work"], "company-access-isolation": ["isolation-owner-resource-controls", "cloud-workspace-switch-and-return", "workspace-data-scoped", "restricted-agent-positive-controls", "cross-workspace-agent-read-write-denied", "isolation-credential-revoked", "no-agent-work"], "repository-continuation": ["repository-configured-source-bound", "repository-first-commit-verified", "repository-original-files-saved", "repository-followup-stored-once", "repository-same-workspace-retained", "repository-second-commit-verified", "repository-original-and-result-reopened", "configured-model-and-environment"], "routine-scheduled-run": ["routine-browser-created", "routine-scheduler-created-execution", "routine-result-attributed", "routine-history-preserved", "routine-pause-prevented-next-run", "routine-result-reopened", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/task-lifecycle-evidence.test.ts b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.test.ts new file mode 100644 index 00000000000..ab1cfd3176f --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { createIssueLabelSchema } from "../../../packages/shared/src/validators/issue.js"; +import { taskLifecycleEvidence, taskLifecycleLabelName, type LifecycleObservation, type LifecycleTask } from "./task-lifecycle-evidence.js"; +import { parseTargetConfig, preparationPreview } from "./catalog.js"; +import { journeyRequiredChecks } from "./success-contract.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import type { RunnerE2EResult } from "../types.js"; + +function fixture(): Parameters[0] { + const expected = { taskId: "task-1", companyId: "company-1", userId: "user-1", originalTitle: "Original task", + revisedTitle: "Revised task", description: "Exact persisted context", labelName: "Owned label" }; + const initial: LifecycleTask = { id: expected.taskId, companyId: expected.companyId, identifier: "QA-1", title: expected.originalTitle, + description: expected.description, status: "todo", priority: "medium", projectId: "project-1", assigneeUserId: expected.userId, + assigneeAgentId: null, createdByUserId: expected.userId, createdByAgentId: null, labelIds: [], completedAt: null }; + const observe = (task: LifecycleTask, labelVisible: boolean): LifecycleObservation => ({ task: structuredClone(task), runCount: 0, + titleVisible: true, descriptionVisible: true, statusVisible: true, labelVisible }); + const edited = { ...initial, title: expected.revisedTitle, status: "in_progress", labelIds: ["label-1"] }; + return { expected, label: { id: "label-1", name: expected.labelName, companyId: expected.companyId, color: "#0088ff" }, + created: observe(initial, false), edited: observe(edited, true), reloaded: observe(edited, true), + completed: observe({ ...edited, status: "done", completedAt: "2026-01-01T00:00:00.000Z" }, true), + reopened: observe({ ...edited, status: "todo" }, true) }; +} +const stages = ["created", "edited", "reloaded", "completed", "reopened"] as const; + +describe("task lifecycle saved-state oracle", () => { + it("keeps a full generated campaign identity within the production label-name limit", () => { + const campaigns = ["journey-1793720364049-f3055878", "journey-9999999999999-ffffffff"]; + const labels = campaigns.map(taskLifecycleLabelName); + for (const [index, name] of labels.entries()) { + expect(name).toContain(campaigns[index]); + expect(createIssueLabelSchema.shape.name.safeParse(name).success).toBe(true); + } + expect(new Set(labels).size).toBe(campaigns.length); + }); + it("rejects an overlong campaign before it can produce a browser label", () => { + expect(() => taskLifecycleLabelName("c".repeat(32))).toThrow("task_lifecycle_campaign_label_name_invalid"); + }); + it("accepts the complete browser-edit/reload/complete/reopen sequence", () => { + expect(taskLifecycleEvidence(fixture())).toEqual({ created: true, edited: true, reloaded: true, + completedAndReopened: true, uneditedFieldsPreserved: true, noAgentWork: true }); + }); + it("accepts a fresh backlog task and an absent project without changing either contract", () => { + const input = fixture(); input.created.task!.status = "backlog"; + for (const stage of stages) input[stage].task!.projectId = null; + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(true); + }); + it.each(stages)("rejects a missing saved task at %s", stage => { + const input = fixture(); input[stage].task = null; + const result = taskLifecycleEvidence(input); + expect(result.uneditedFieldsPreserved).toBe(false); + expect(Object.values(result).every(Boolean)).toBe(false); + }); + it.each(["id", "companyId", "assigneeUserId", "assigneeAgentId", "createdByUserId", "createdByAgentId", "description"] as const)( + "rejects a wrong owner or saved context field: %s", key => { + const input = fixture(); Object.assign(input.reopened.task!, { [key]: "foreign" }); + expect(taskLifecycleEvidence(input).completedAndReopened).toBe(false); + expect(taskLifecycleEvidence(input).uneditedFieldsPreserved).toBe(false); + }); + it.each(["identifier", "priority", "projectId"] as const)("requires %s to survive every edit", key => { + for (const stage of stages.slice(1)) { + const input = fixture(); Object.assign(input[stage].task!, { [key]: "changed" }); + expect(taskLifecycleEvidence(input).uneditedFieldsPreserved).toBe(false); + } + }); + it.each(["edited", "reloaded", "completed", "reopened"] as const)("rejects a lost title or label at %s", stage => { + for (const patch of [{ title: "Original task" }, { labelIds: [] }, { labelIds: ["foreign"] }, { labelIds: ["label-1", "label-1"] }]) { + const input = fixture(); Object.assign(input[stage].task!, patch); + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(false); + } + }); + it.each(["titleVisible", "descriptionVisible", "statusVisible", "labelVisible"] as const)("requires visible reloaded %s", key => { + for (const stage of ["reloaded", "completed", "reopened"] as const) { + const input = fixture(); input[stage][key] = false; + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(false); + } + }); + it("does not accept a completed-looking UI without saved Done or completion time", () => { + const wrongStatus = fixture(); wrongStatus.completed.task!.status = "in_review"; + expect(taskLifecycleEvidence(wrongStatus).completedAndReopened).toBe(false); + for (const completedAt of [null, "", "not a timestamp"]) { + const input = fixture(); input.completed.task!.completedAt = completedAt; + expect(taskLifecycleEvidence(input).completedAndReopened).toBe(false); + } + }); + it("requires reopened Todo with its completion time cleared", () => { + for (const patch of [{ status: "done" }, { completedAt: "2026-01-01T00:00:00.000Z" }]) { + const input = fixture(); Object.assign(input.reopened.task!, patch); + expect(taskLifecycleEvidence(input).completedAndReopened).toBe(false); + } + }); + it.each(stages)("requires an explicit zero-run observation at %s", stage => { + for (const runCount of [null, 1, -1]) { + const input = fixture(); input[stage].runCount = runCount; + expect(taskLifecycleEvidence(input).noAgentWork).toBe(false); + } + }); + it("rejects stale starting data, a wrong label and an unchanged expected title", () => { + for (const change of [ + (input: ReturnType) => { input.created.task!.labelIds = ["existing"]; }, + (input: ReturnType) => { input.created.task!.status = "done"; }, + (input: ReturnType) => { input.label = null; }, + (input: ReturnType) => { input.label!.companyId = "foreign"; }, + (input: ReturnType) => { input.label!.name = "Other label"; }, + (input: ReturnType) => { input.label!.color = "invalid"; }, + (input: ReturnType) => { input.expected.revisedTitle = input.expected.originalTitle; }, + ]) { + const input = fixture(); change(input); + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(false); + } + }); +}); + +function result(): RunnerE2EResult { + return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "test", + executionId: "product-journeys.qa.browser.daytona.task-edit-lifecycle", profileId: "browser", environmentId: "daytona", + caseId: "task-edit-lifecycle", provider: null, model: null, runtimeMode: "none", status: "passed", cleanup: "passed", + attempt: 1, startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, + productJourney: { outcome: "passed", startingState: "recipe", phases: [{ phase: "journey", outcome: "passed" }], + checks: journeyRequiredChecks["task-edit-lifecycle"]!.map(id => ({ id, passed: true })), privacy: "private", resourceDisposition: "released", + deployment: { kind: "cloud", targetId: "qa", origin: "https://qa.example.test", commit: "a".repeat(40), + cloudRelease: "test", runtimeImage: null, configurationHash: "test" } } }; +} + +describe("task lifecycle discovery and report contract", () => { + it("previews a browser-only existing target with no provider allowance", () => { + const config = parseTargetConfig({ version: 1, id: "qa", mode: "existing-cloud", adapterModule: "/private/adapter.mjs", + journeys: ["task-edit-lifecycle"], executionEnvironment: "daytona", baseline: { features: {}, connections: [] }, + limits: { durationMs: 180000, maxRuns: 0, modelBudgetCents: 0 } }); + expect(preparationPreview(config).journeys).toMatchObject([{ id: "task-edit-lifecycle", paid: false }]); + expect(config.limits.maxRuns).toBe(0); + }); + it("rejects every missing required check and unexpected native work", () => { + expect(() => validateRetainedRunnerResult(result())).not.toThrow(); + for (const id of journeyRequiredChecks["task-edit-lifecycle"]!) { + const missing = result(); missing.productJourney!.checks = missing.productJourney!.checks.filter(check => check.id !== id); + expect(() => validateRetainedRunnerResult(missing)).toThrow(); + } + for (const patch of [{ provider: "openai" }, { model: "wrong" }, { runtimeMode: "native" }, { runIds: ["unexpected"] }]) { + expect(() => validateRetainedRunnerResult({ ...result(), ...patch })).toThrow(); + } + const usage = result(); usage.usage = { runs: [{ runId: "unexpected", usage: { inputTokens: 1 } }] }; + expect(() => validateRetainedRunnerResult(usage)).toThrow("result.browserOnly"); + const emptyUsage = result(); emptyUsage.usage = { runs: [] }; + expect(() => validateRetainedRunnerResult(emptyUsage)).not.toThrow(); + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-lifecycle-evidence.ts b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.ts new file mode 100644 index 00000000000..a20b1bab0d4 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.ts @@ -0,0 +1,98 @@ +import { createIssueLabelSchema } from "../../../packages/shared/src/validators/issue.js"; +import { JourneyError } from "./types.js"; + +export function taskLifecycleLabelName(campaignId: string): string { + const name = `Journey QA label ${campaignId}`; + if (!createIssueLabelSchema.shape.name.safeParse(name).success) { + throw new JourneyError("unsupported", "task_lifecycle_campaign_label_name_invalid"); + } + return name; +} + +export interface LifecycleTask { + id: string; + companyId: string; + identifier: string; + title: string; + description: string; + status: string; + priority: string; + projectId: string | null; + assigneeUserId: string; + assigneeAgentId: null; + createdByUserId: string; + createdByAgentId: null; + labelIds: string[]; + completedAt: string | null; +} +export interface LifecycleLabel { id: string; companyId: string; name: string; color: string } +export interface LifecycleObservation { + task: LifecycleTask | null; + runCount: number | null; + titleVisible: boolean; + descriptionVisible: boolean; + statusVisible: boolean; + labelVisible: boolean; +} +export interface LifecycleExpectation { + taskId: string; + companyId: string; + userId: string; + originalTitle: string; + revisedTitle: string; + description: string; + labelName: string; +} + +function ownedTask(task: LifecycleTask | null, expected: LifecycleExpectation) { + return Boolean(task && task.id === expected.taskId && task.companyId === expected.companyId && task.identifier + && task.description === expected.description && task.assigneeUserId === expected.userId && task.assigneeAgentId === null + && task.createdByUserId === expected.userId && task.createdByAgentId === null); +} +function singleton(ids: unknown, id: string) { return Array.isArray(ids) && ids.length === 1 && ids[0] === id; } +function sameUneditedFields(task: LifecycleTask | null, initial: LifecycleTask | null) { + return Boolean(task && initial && task.identifier === initial.identifier && task.priority === initial.priority && task.projectId === initial.projectId); +} + +/** Independent saved states accompany visible UI readback. Empty or missing observations cannot pass. */ +export function taskLifecycleEvidence(input: { + expected: LifecycleExpectation; + label: LifecycleLabel | null; + created: LifecycleObservation; + edited: LifecycleObservation; + reloaded: LifecycleObservation; + completed: LifecycleObservation; + reopened: LifecycleObservation; +}) { + const { expected, label, created, edited, reloaded, completed, reopened } = input; + const validExpected = Boolean(expected.taskId && expected.companyId && expected.userId && expected.originalTitle + && expected.revisedTitle && expected.originalTitle !== expected.revisedTitle && expected.description && expected.labelName); + const fresh = validExpected && ownedTask(created.task, expected) && created.task!.title === expected.originalTitle + && ["backlog", "todo"].includes(created.task!.status) && created.task!.completedAt === null + && ["critical", "high", "medium", "low"].includes(created.task!.priority) + && (created.task!.projectId === null || typeof created.task!.projectId === "string" && created.task!.projectId.length > 0) + && Array.isArray(created.task!.labelIds) && created.task!.labelIds.length === 0 + && created.titleVisible && created.descriptionVisible && created.statusVisible; + const labelValid = Boolean(label && label.id && label.companyId === expected.companyId && label.name === expected.labelName + && /^#[a-fA-F0-9]{6}$/.test(label.color)); + const saved = (observation: LifecycleObservation, status: string) => fresh && labelValid && ownedTask(observation.task, expected) + && sameUneditedFields(observation.task, created.task) && observation.task!.title === expected.revisedTitle + && observation.task!.status === status && singleton(observation.task!.labelIds, label!.id); + const shown = (observation: LifecycleObservation) => observation.titleVisible && observation.descriptionVisible + && observation.statusVisible && observation.labelVisible; + const editedMatches = saved(edited, "in_progress") && shown(edited) && edited.task!.completedAt === null; + const reloadedMatches = saved(reloaded, "in_progress") && shown(reloaded) && reloaded.task!.completedAt === null; + const completedMatches = saved(completed, "done") && shown(completed) + && typeof completed.task!.completedAt === "string" && Number.isFinite(Date.parse(completed.task!.completedAt)); + const reopenedMatches = saved(reopened, "todo") && shown(reopened) && reopened.task!.completedAt === null; + const observations = [created, edited, reloaded, completed, reopened]; + return { + created: fresh, + edited: editedMatches, + reloaded: reloadedMatches, + completedAndReopened: completedMatches && reopenedMatches, + uneditedFieldsPreserved: fresh && observations.every(observation => ownedTask(observation.task, expected) + && sameUneditedFields(observation.task, created.task)), + noAgentWork: fresh && observations.every(observation => observation.runCount === 0), + }; +} diff --git a/tests/runner-e2e/product-journeys/task-lifecycle.ts b/tests/runner-e2e/product-journeys/task-lifecycle.ts new file mode 100644 index 00000000000..b2086f890cb --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-lifecycle.ts @@ -0,0 +1,173 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi, withBrowserEvent } from "../user-actions.js"; +import { createObservedReader } from "./observation-read.js"; +import { taskLifecycleEvidence, taskLifecycleLabelName, type LifecycleExpectation, type LifecycleLabel, type LifecycleObservation, type LifecycleTask } from "./task-lifecycle-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Stage = "create" | "initial_read" | "rename" | "label" | "in_progress" | "reload" | "complete" | "reopen"; +function taskSnapshot(task: LifecycleTask): LifecycleTask { + return { id: task.id, companyId: task.companyId, identifier: task.identifier, title: task.title, + description: task.description, status: task.status, priority: task.priority, projectId: task.projectId, + assigneeUserId: task.assigneeUserId, assigneeAgentId: task.assigneeAgentId, createdByUserId: task.createdByUserId, + createdByAgentId: task.createdByAgentId, labelIds: task.labelIds, completedAt: task.completedAt }; +} +const emptyObservation = (): LifecycleObservation => ({ task: null, runCount: null, + titleVisible: false, descriptionVisible: false, statusVisible: false, labelVisible: false }); + +async function properties(context: JourneyContext) { + const { page } = context; + // Closed panes keep their rows mounted. Their geometry alone does not make + // an inert or accessibility-hidden panel available for interaction. + const row = page.locator('[data-property-row]:not([inert] *):not([aria-hidden="true"] *)') + .filter({ has: page.locator('[data-property-label="Labels"]') }).filter({ visible: true }); + const toggle = page.getByRole("button", { name: "Toggle side panel", exact: true, pressed: false }).filter({ visible: true }); + const propertiesButton = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).filter({ visible: true }); + await expect.poll(async () => await row.isVisible() || await toggle.isVisible() || await propertiesButton.isVisible()).toBe(true); + if (!await row.isVisible()) { + if (await toggle.isVisible()) await toggle.click(); + else await propertiesButton.click(); + } + await expect(row).toBeVisible(); + await row.scrollIntoViewIfNeeded(); + await expect(row).toBeInViewport(); + return row; +} + +async function setStatus(context: JourneyContext, label: string) { + const { page } = context; + await page.getByTestId("issue-detail-header").filter({ visible: true }) + .getByRole("button", { name: /^Change status \(current:/ }).click(); + // The shipped picker includes both a labeled status glyph and visible text. + // Bind the choice to its exact accessible glyph inside the open popover. + await page.locator('[data-slot="popover-content"]').filter({ visible: true }) + .getByRole("button").filter({ has: page.getByRole("img", { name: label, exact: true }) }).click(); +} + +/** All measured edits use the shipped browser. Independent GETs verify saved state. */ +export async function taskEditLifecycle(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, + options: { deadlineAt: number; directory: string }) { + const { page, api, company, user } = context; + const read = createObservedReader(api, options.deadlineAt); + const journeyId = "task-edit-lifecycle" as const; + const originalTitle = `Journey QA task lifecycle ${campaignId}`; + const revisedTitle = `Journey QA task lifecycle revised ${campaignId}`; + const description = `Keep this task assigned to me. Saved task context ${campaignId} must survive every edit.`; + const labelName = taskLifecycleLabelName(campaignId); + const observed = { created: emptyObservation(), edited: emptyObservation(), reloaded: emptyObservation(), + completed: emptyObservation(), reopened: emptyObservation() }; + let expected: LifecycleExpectation | undefined; + let label: LifecycleLabel | null = null; + let stage: Stage = "create"; + const check = (id: string, passed: boolean) => { + evidence.checks.push({ id, passed }); + if (!passed) throw new JourneyError("failed", id.replaceAll("-", "_")); + }; + const preserve = async () => { + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + await writeFile(path.join(options.directory, "task-lifecycle-observations.json"), + JSON.stringify({ schema: "paperclip.task-lifecycle-observations/v1", stage, expected, label, ...observed }, null, 2), { mode: 0o600 }); + }; + const grade = () => taskLifecycleEvidence({ expected: expected!, label, ...observed }); + try { + for (const capability of ["company", "board-user", "task-write"]) { + if (!context.capabilities.includes(capability)) throw new JourneyError("unsupported", `missing_${capability}`); + } + if (!campaignId) throw new JourneyError("blocked", "task_lifecycle_campaign_required"); + const created = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: originalTitle, prompt: description, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: originalTitle, journeyId }) }); + await session.own({ kind: "issue", name: originalTitle, id: created.issueId, journeyId }); + const route = `/api/issues/${created.issueId}`; + expected = { taskId: created.issueId, companyId: company.id, userId: user.id, originalTitle, revisedTitle, description, labelName }; + const task = await read.get(route); + await page.goto(`${api.origin}/${company.issuePrefix}/issues/${task.identifier}`); + const capture = async (title: string, statusLabel: string, withLabel: boolean): Promise => { + await expect(page.getByRole("heading", { name: title, exact: true })).toBeVisible(); + await expect(page.getByText(description, { exact: true }).first()).toBeVisible(); + const status = page.getByRole("button", { name: `Change status (current: ${statusLabel})`, exact: true }).filter({ visible: true }).first(); + await expect(status).toBeVisible(); + const labelRow = await properties(context); + if (withLabel) await expect(labelRow.getByText(labelName, { exact: true })).toBeVisible(); + const saved = await read.get(route); + const runs = await read.get(`${route}/runs`); + return { task: structuredClone(taskSnapshot(saved)), runCount: Array.isArray(runs) ? runs.length : null, + titleVisible: await page.getByRole("heading", { name: title, exact: true }).isVisible(), + descriptionVisible: await page.getByText(description, { exact: true }).first().isVisible(), + statusVisible: await status.isVisible(), labelVisible: withLabel && await labelRow.getByText(labelName, { exact: true }).isVisible() }; + }; + stage = "initial_read"; + const initialStatus = task.status === "backlog" ? "Backlog" : "Todo"; + observed.created = await capture(originalTitle, initialStatus, false); + await preserve(); + check("task-lifecycle-browser-created", grade().created); + stage = "rename"; + // Keep the creation receipt stable. Cleanup may accept this one exact, + // predeclared title if the browser save commits but its response is lost. + await session.own({ kind: "issue", name: originalTitle, id: created.issueId, journeyId, taskEdit: { revisedTitle } }); + await page.getByRole("heading", { name: originalTitle, exact: true }).click(); + const titleEditor = page.getByTestId("issue-detail-header").locator("textarea").first(); + await titleEditor.fill(revisedTitle); + await titleEditor.press("Enter"); + await expect.poll(async () => (await read.get(route)).title).toBe(revisedTitle); + stage = "label"; + const labelsRoute = `/api/companies/${company.id}/labels`; + const beforeLabels = await read.get(labelsRoute); + if (beforeLabels.some(row => row.name === labelName)) throw new JourneyError("failed", "task_lifecycle_label_not_fresh"); + const labels = await properties(context); + await labels.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByPlaceholder("New label", { exact: true }).fill(labelName); + await session.own({ kind: "label", name: labelName, journeyId }); + const response = await withBrowserEvent(() => page.waitForResponse(response => response.request().method() === "POST" + && new URL(response.url()).pathname === labelsRoute), + () => page.getByRole("button", { name: "Create label", exact: true }).click()); + if (response.status() !== 201) throw new JourneyError("failed", "task_lifecycle_label_create_failed"); + const createdLabel = await response.json() as LifecycleLabel; + if (typeof createdLabel.id !== "string" || !createdLabel.id || createdLabel.name !== labelName || createdLabel.companyId !== company.id) { + throw new JourneyError("failed", "task_lifecycle_label_receipt_mismatch"); + } + await session.own({ kind: "label", name: labelName, id: createdLabel.id, journeyId }); + label = { id: createdLabel.id, companyId: createdLabel.companyId, name: createdLabel.name, color: createdLabel.color }; + await page.keyboard.press("Escape"); + const savedLabels = await read.get(labelsRoute); + if (savedLabels.filter(row => row.name === labelName).length !== 1 + || !savedLabels.some(row => row.id === label!.id && row.name === labelName && row.companyId === company.id && row.color === label!.color)) { + throw new JourneyError("failed", "task_lifecycle_label_saved_mismatch"); + } + stage = "in_progress"; + await setStatus(context, "In Progress"); + await expect.poll(async () => (await read.get(route)).status).toBe("in_progress"); + await expect.poll(async () => { + const task = await read.get(route); + return Array.isArray(task.labelIds) && task.labelIds.length === 1 && task.labelIds[0] === label!.id; + }).toBe(true); + observed.edited = await capture(revisedTitle, "In Progress", true); + await preserve(); + check("task-lifecycle-visible-edits-persisted", grade().edited); + stage = "reload"; + await page.reload(); + observed.reloaded = await capture(revisedTitle, "In Progress", true); + await preserve(); + check("task-lifecycle-edits-reloaded", grade().reloaded); + stage = "complete"; + await setStatus(context, "Done"); + await expect.poll(async () => (await read.get(route)).status).toBe("done"); + await page.reload(); + observed.completed = await capture(revisedTitle, "Done", true); + await preserve(); + stage = "reopen"; + await setStatus(context, "Todo"); + await expect.poll(async () => (await read.get(route)).status).toBe("todo"); + await page.reload(); + observed.reopened = await capture(revisedTitle, "Todo", true); + await preserve(); + const final = grade(); + check("task-lifecycle-completed-and-reopened", final.completedAndReopened); + check("task-lifecycle-unedited-context-preserved", final.uneditedFieldsPreserved); + check("no-agent-work", final.noAgentWork); + } catch (error) { + if (error instanceof JourneyError) throw error; + throw new JourneyError("failed", `task_lifecycle_${stage}_failed`); + } finally { await preserve(); await read.preserve(options.directory); } +} diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 4c235990f6c..34dbf8ff913 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -2,8 +2,28 @@ import type { BrowserContext, Page } from "@playwright/test"; import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; -export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "task-document-persistence" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" | "file-input-answer" | "inbox-triage" | "routine-scheduled-run" | "repository-continuation" | "company-access-isolation"; +export type JourneyId = + "task-edit-lifecycle" + | "task-properties" + | "task-create" + | "project-task-context" + | "task-document-persistence" + | "ask-question" + | "cloud-onboarding" + | "question-answer-resume" + | "task-search-filter" + | "followup-while-running" + | "plan-revise-accept" + | "build-revise" + | "protected-actions" + | "create-skill-studio" + | "stop-redirect" + | "hire-delegate-reuse" + | "file-input-answer" + | "inbox-triage" + | "routine-scheduled-run" + | "repository-continuation" + | "company-access-isolation"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; @@ -113,7 +133,8 @@ export interface TargetSession { /** Publish the observed finalization failure without releasing resources or leases. */ checkpointEvidence?(input?: TargetLifecycleInput): Promise; /** Durable intent before creation; receipt immediately after creation. */ - own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill" | "routine"; name: string; id?: string; journeyId?: JourneyId; + own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill" | "routine" | "label"; name: string; id?: string; journeyId?: JourneyId; + taskEdit?: { revisedTitle: string }; freshChat?: { companyId: string; userId: string; leadAgentId: string; absent: true } }): Promise; } export interface TargetProvider { diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 419e37a934d..9906e0d4869 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -434,8 +434,8 @@ export function validateRetainedRunnerResult( if (!evidence) return invalid("result.productJourney"); const required = journeyRequiredChecks[checked.caseId]; if (evidence.outcome !== "passed" || !evidence.deployment || !required || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.checks.some(check => !check.passed) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); - if (["inbox-triage", "task-properties", "task-create", "project-task-context", "task-document-persistence", "task-search-filter", "company-access-isolation"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); - if (checked.caseId === "company-access-isolation" && checked.usage != null && Object.entries(checked.usage).some(([key, value]) => key !== "runs" || !Array.isArray(value) || value.length > 0)) invalid("result.browserOnly"); + if (["task-edit-lifecycle", "inbox-triage", "task-properties", "task-create", "project-task-context", "task-document-persistence", "task-search-filter", "company-access-isolation"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); + if (["company-access-isolation", "task-edit-lifecycle"].includes(checked.caseId) && checked.usage != null && Object.entries(checked.usage).some(([key, value]) => key !== "runs" || !Array.isArray(value) || value.length > 0)) invalid("result.browserOnly"); if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (!agentJourneyRunCountValid(checked.caseId, checked.runIds?.length) || !checked.model || !checked.provider || checked.runtimeMode !== "native")) invalid("result.agentJourneyRun"); if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt"); From f77499283270d2e966aabd58493f9d35c6bbdf5f Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:01:07 -0600 Subject: [PATCH 03/28] perf(ci): admit release candidates and isolate native image capacity (#18869) Admit full source verification only for exact-source Docker or npm release work, while superseded pending sources complete metadata-only admission. Restart only strictly deferred runs; failed or cancelled checks require an explicit request. Route canonical master native images to dedicated 32-core Spot profiles and keep source checks on protected verification capacity. Use the four-slot GitHub-hosted publication pool for registry work and signing, with proof waits outside that pool and a fresh source-proof check before tagging. Trim historical checkout blobs and prepared-runner disk cleanup while preserving tag and fork publication behavior. Validation: release/source and source/artifact regressions, 665 workflow contracts, actionlint, complete latest-head PR CI, and Greptile 5/5 with all comments resolved. Existing CLI and large-file repository timing tests also passed independently; their CI shards received one scoped retry. Co-Authored-By: Paperclip --- .../tests/post-merge-runner-routing.test.mjs | 13 +- .../workflows/cloud-migrator-artifacts.yml | 30 +- .github/workflows/cloud-readiness.yml | 33 +- .github/workflows/docker.yml | 81 +++- .github/workflows/release-verify.yml | 12 +- .github/workflows/release.yml | 6 +- .github/workflows/runner-chaos-evals.yml | 4 +- doc/cloud-build-readiness.md | 88 ++++- .../release-verify-workflow.test.mjs | 39 +- scripts/cloud-release-admission.mjs | 123 ++++++ scripts/cloud-source-verification.mjs | 47 ++- scripts/cloud-source-verification.test.mjs | 365 +++++++++++++++++- scripts/watch-release-source.mjs | 108 ++++++ 13 files changed, 886 insertions(+), 63 deletions(-) create mode 100644 scripts/cloud-release-admission.mjs create mode 100644 scripts/watch-release-source.mjs diff --git a/.github/scripts/tests/post-merge-runner-routing.test.mjs b/.github/scripts/tests/post-merge-runner-routing.test.mjs index 7c46aa47931..0e905635214 100644 --- a/.github/scripts/tests/post-merge-runner-routing.test.mjs +++ b/.github/scripts/tests/post-merge-runner-routing.test.mjs @@ -10,7 +10,8 @@ const base = { ref: "refs/heads/master", event_name: "push", sha, run_attempt: 1, }; const expectedJobs = { - "cloud-readiness.yml": [], + "docker.yml": ["source_verification", "publication_verification"], + "cloud-readiness.yml": ["admission", "source_verified"], "cloud-migrator-artifacts.yml": [], "release-verify.yml": ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"], "runner-chaos-evals.yml": ["chaos_and_recovery"], @@ -59,9 +60,9 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { cases.push({ name: "stable release", inputs: { channel: "stable" } }); } cases.push({ name: "first attempt as a string", github: { run_attempt: "1" }, expected: fleet }); - cases.push({ name: "rerun after runner interruption", github: { run_attempt: 2 } }); - cases.push({ name: "later rerun", github: { run_attempt: 3 } }); - cases.push({ name: "rerun attempt as a string", github: { run_attempt: "2" } }); + cases.push({ name: "admitted retry", github: { run_attempt: 2 }, expected: release ? "ubuntu-latest" : fleet }); + cases.push({ name: "later retry", github: { run_attempt: 3 }, expected: release ? "ubuntu-latest" : fleet }); + cases.push({ name: "retry attempt as a string", github: { run_attempt: "2" }, expected: release ? "ubuntu-latest" : fleet }); cases.push({ name: "missing attempt", github: { run_attempt: undefined } }); for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) { test(`${file}/${job}: ${name}`, () => { @@ -88,7 +89,7 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { } -test("Cloud readiness bookkeeping never waits for the AWS verification fleet", () => { +test("Cloud source proof keeps source-only gating on protected verification capacity", () => { const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8"); const bodies = new Map(); for (const [name, needs] of [ @@ -97,7 +98,7 @@ test("Cloud readiness bookkeeping never waits for the AWS verification fleet", ( const body = workflow.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=^ [a-z_]+:|(?![\\s\\S]))`, "m"))?.[1]; assert.ok(body, `missing ${name} job`); bodies.set(name, body); - assert.match(body, /^ runs-on: ubuntu-latest$/m); + assert.match(body, /runs-on:.*paperclip-post-merge-x64/); assert.doesNotMatch(body, /^ +continue-on-error:|^ +if:.*always\(\)/m); assert.match(body, /^ if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'$/m); assert.match(body, /^ +SOURCE_SHA: \$\{\{ github.sha \}\}$/m); diff --git a/.github/workflows/cloud-migrator-artifacts.yml b/.github/workflows/cloud-migrator-artifacts.yml index 5c698230938..e60b7a54456 100644 --- a/.github/workflows/cloud-migrator-artifacts.yml +++ b/.github/workflows/cloud-migrator-artifacts.yml @@ -12,9 +12,33 @@ concurrency: cancel-in-progress: false jobs: - build: + admission: + name: Cloud image candidate admission v1 if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + admitted: ${{ steps.admission.outputs.admitted }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + sparse-checkout: scripts + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - id: admission + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/cloud-release-admission.mjs image-only + + build: + needs: admission + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true' + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} timeout-minutes: 10 permissions: contents: read @@ -52,7 +76,7 @@ jobs: publish: needs: build if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} timeout-minutes: 5 permissions: contents: read diff --git a/.github/workflows/cloud-readiness.yml b/.github/workflows/cloud-readiness.yml index 6c709ade0ed..b9ee017ddea 100644 --- a/.github/workflows/cloud-readiness.yml +++ b/.github/workflows/cloud-readiness.yml @@ -8,14 +8,38 @@ on: permissions: {} -# Source verification must start outside the full npm release's queue. +# Source verification stays independent of npm publication and image builds. concurrency: group: cloud-readiness-${{ github.sha }} cancel-in-progress: false jobs: + admission: + name: Cloud candidate admission v1 + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + admitted: ${{ steps.admission.outputs.admitted }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + sparse-checkout: scripts + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - id: admission + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/cloud-release-admission.mjs + verify: - if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + needs: admission + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true' permissions: contents: read uses: ./.github/workflows/release-verify.yml @@ -29,8 +53,9 @@ jobs: name: Cloud source verified v1 needs: [verify] if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' - # Bookkeeping must not wait for the AWS builders it observes. - runs-on: ubuntu-latest + # Keep the final proof off the shared hosted queue too. The protected + # fleet has a separate job ceiling and launches Spot workers on demand. + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 permissions: contents: read diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ec7c71000e5..6048c277e41 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,29 @@ concurrency: cancel-in-progress: false jobs: + source_verification: + name: Watch exact-source verification + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 44 + permissions: + contents: read + actions: write + outputs: + verified: ${{ steps.watch.outputs.verified }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Stop images when source checks fail and report runner waits + id: watch + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs + reuse_canary_image: if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v') runs-on: ubuntu-latest @@ -100,11 +123,10 @@ jobs: - platform: linux/arm64 runner: ubuntu-24.04-arm arch: arm64 - runs-on: ${{ matrix.runner }} - # Native builds land well inside this; it is a backstop, not a budget. - # (The interim fix while this PR landed raised the single QEMU job's cap - # to 120 minutes; native per-arch legs make that headroom unnecessary.) - timeout-minutes: 60 + runs-on: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && format('runs-on/fleet=paperclip-core-image-{0}/env=public-ci', matrix.arch == 'amd64' && 'x64' || 'arm64') || matrix.runner }} + # Stop before the AWS worker's 45-minute maximum lifetime. Hosted tag + # builds retain their existing limit. + timeout-minutes: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && 40 || 60 }} permissions: contents: read packages: write @@ -115,6 +137,9 @@ jobs: # Full history and tags so `git describe` below can compute the # release version to stamp into the image. fetch-depth: 0 + # Keep the version's full commit graph without downloading blobs + # from every historical revision of this large repository. + filter: blob:none # `.git` is dockerignored, so a running image cannot derive its own # version and otherwise reports the source package.json placeholder in @@ -183,6 +208,7 @@ jobs: echo "Using refreshed pnpm-lock.yaml in the Docker build context." - name: Free runner disk + if: ${{ !(vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master') }} run: | set -euo pipefail echo "Disk before cleanup:" @@ -304,16 +330,40 @@ jobs: if-no-files-found: error retention-days: 1 + # Wait for proof on ephemeral verification capacity. Publication runners do + # only registry/signing work. Each Spot observer stays below its 45m lifetime. + publication_verification: + name: Require exact-source verification + if: ${{ !cancelled() && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.build-and-push.result == 'success' && needs.source_verification.result == 'success' && needs.source_verification.outputs.verified != 'true' }} + needs: [build-and-push, source_verification] + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 44 + permissions: + contents: read + actions: write # restart only exact-source producers deferred before admission + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Require source proof and migrator admission before publication + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs --require-proof + # Names the per-architecture digests as one manifest list under the real # tags. Nothing is publicly tagged until this runs, so a half-published # multi-arch image is not a state anything can pull. merge-and-push: - if: ${{ !cancelled() && needs.build-and-push.result == 'success' }} - needs: build-and-push - runs-on: ubuntu-latest + if: ${{ !cancelled() && needs.build-and-push.result == 'success' && (needs.source_verification.result == 'success' || needs.source_verification.result == 'skipped') && (needs.publication_verification.result == 'success' || (needs.publication_verification.result == 'skipped' && (github.repository != 'paperclipai/paperclip' || github.repository_id != '1170821064' || github.ref != 'refs/heads/master' || needs.source_verification.outputs.verified == 'true'))) }} + needs: [build-and-push, source_verification, publication_verification] + runs-on: ${{ github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32') || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read + actions: read packages: write id-token: write attestations: write @@ -322,8 +372,6 @@ jobs: # script the verification step pipes in. - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - name: Download digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -375,6 +423,19 @@ jobs: io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }} io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }} + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + + # Revalidate the latest attempt after runner assignment without waiting + # here. A concurrent source retry must invalidate the earlier proof. + - name: Revalidate exact-source proof before publication + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs --check-proof + - name: Create manifest list and push working-directory: /tmp/digests run: | diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index e36bdbccbd2..a8a29e433b5 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -34,7 +34,7 @@ jobs: typecheck: if: ${{ !inputs.runner_binaries_only }} name: Typecheck - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -120,7 +120,7 @@ jobs: general_tests: if: ${{ !inputs.runner_binaries_only }} name: General tests (${{ matrix.group_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -232,7 +232,7 @@ jobs: serialized_tests: if: ${{ !inputs.runner_binaries_only }} name: Serialized tests (${{ matrix.shard_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -282,7 +282,7 @@ jobs: runner_workflow_evals: if: ${{ !inputs.runner_binaries_only }} name: Runner workflow eval scorer contract - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 10 permissions: contents: read @@ -313,7 +313,7 @@ jobs: verify_paperclip_runner: if: ${{ !inputs.runner_binaries_only }} name: Verify Paperclip Runner (${{ matrix.lane }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -523,7 +523,7 @@ jobs: build: if: ${{ !inputs.runner_binaries_only }} name: Build - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7b310f4ee1..d9bdc1824f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -320,12 +320,12 @@ jobs: verify_canary: name: Reuse exact-source verification - if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' runs-on: ubuntu-latest - timeout-minutes: 50 + timeout-minutes: 70 permissions: contents: read - actions: read + actions: write # restart only an exact-source push deferred before admission steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml index 810d28ec2e7..fbb60238de3 100644 --- a/.github/workflows/runner-chaos-evals.yml +++ b/.github/workflows/runner-chaos-evals.yml @@ -20,8 +20,8 @@ concurrency: jobs: chaos_and_recovery: name: Restart, replay, trace, and recovery faults - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} - timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} permissions: contents: read diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md index 18611b833b0..df1a595bf56 100644 --- a/doc/cloud-build-readiness.md +++ b/doc/cloud-build-readiness.md @@ -1,30 +1,42 @@ # Cloud build readiness -The `Cloud readiness` workflow starts for every master push and retains the -versioned `Cloud source verified v1` job. It calls the full `Release Verify` -workflow for that exact commit, including typecheck, builds, general and -serialized tests, and Runner verification. The source proof depends on every -source check and fails closed if verification fails, is cancelled, or is skipped. +The `Cloud readiness` workflow starts a small admission check for each master +push. Full verification starts only when the exact source has a running master +Docker or automatic npm release workflow. Superseded pending sources avoid the +full test matrix. The migrator producer similarly waits for image admission. + +A selected consumer restarts a previously deferred push run, preserving its +original source and push identity. This restarts only a successful admission +job followed entirely by skipped jobs. A failed check, cancelled job, incomplete +listing, or identity mismatch cannot trigger this automatic retry. Explicit +manual runs and whole-workflow retries request full verification directly. + +Admitted source runs retain the versioned `Cloud source verified v1` job. They +call the full `Release Verify` workflow for that exact commit, including +typecheck, builds, general and serialized tests, and Runner verification. The +proof depends on every source check and fails closed if verification fails, is +cancelled, or is skipped. Consumers accept only the latest attempt's proof. The recurring public `-cloud` publisher and its `Cloud deployable v1` gate are retired. The workflow no longer builds a legacy image or waits for one. Keep its filename and source-proof job name stable: npm canary publication and downstream image composers consume that exact contract. -Standard images still publish independently through `docker.yml`. The +Standard images build in parallel through `docker.yml`. Master image publication +waits for the exact-source proof; a failed source check cancels its image build +without waiting for the remaining test shards. The `cloud-migrator-artifacts.yml` workflow still publishes signed exact-source migrators independently. A downstream composer must verify those artifacts, build and test its own image, and record separate deployment readiness. -Verification runs outside the full npm release's concurrency group. Different -commits have independent groups. The npm canary release reuses the source proof +Verification runs outside the full npm release's concurrency group. Each source retains its independent verification run. The npm canary release reuses the source proof for its exact master push instead of starting another `Release Verify` run. Stable releases and candidate-branch betas still run full verification. The canary consumer requires the expected workflow ID and path, upstream source repository, master push event, full SHA, and a successful job in the latest run attempt. It checks the run again after reading the jobs to reject a concurrent -rerun. Missing proof waits for up to 45 minutes; failed, skipped, cancelled, +rerun. Missing proof waits for up to 60 minutes; failed, skipped, cancelled, ambiguous, or mismatched proof cannot authorize publication. API failures fail closed. If a source check fails, fix it and rerun Cloud readiness before retrying the release. Use **Re-run all jobs** when a later attempt did not rerun the source @@ -120,15 +132,16 @@ excluded or sleeping targets, retries, and failures with the fleet result. Recor runner queue conditions and cache state; one warm or cold run is a sample, not a latency guarantee. -## Reserved AWS verification capacity +## Dedicated AWS verification capacity `AWS_POST_MERGE_CI_ENABLED=true` routes cloud source verification and exact-master migrator preparation to the `paperclip-post-merge` runner group. The separate Fleet label is -`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. Its 36 reserved slots use -the same four-vCPU, 16-GiB machines as approved PR jobs. PR capacity is reduced -to 64; the separately provisioned image capacity is unchanged by this retirement. -This keeps PR bursts from consuming every post-merge verification slot. +`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. The protected fleet uses +separate 8-vCPU, 32-GiB Spot workers that launch only when jobs arrive. Its +job ceiling and public PR capacity are managed independently. There are no +hot or stopped pools. Quota ceilings do not reserve hardware; measure cold +launch and runner assignment times as well as job duration. Every selector checks the canonical repository name and ID, master ref, and a push or manual event. Reusable verification also requires `inputs.ref` to equal @@ -142,7 +155,8 @@ its trusted-publisher identity. Before enabling the switch, deploy the separate Fleet and restrict its GitHub runner group to repository ID `1170821064` and these workflows at `refs/heads/master`: `cloud-readiness.yml`, -`release-verify.yml`, `runner-chaos-evals.yml`, and `release.yml`. Do not authorize +`release-verify.yml`, `runner-chaos-evals.yml`, `release.yml`, and `docker.yml`. +The Docker workflow uses this group for its exact-source monitor and proof wait. Do not authorize PR-controlled workflow versions. The direct migrator producer always uses GitHub-hosted runners and needs no AWS runner-group authorization. PR placement retains its independent pinned workflow and six-account author/actor allowlist. @@ -158,6 +172,43 @@ route a public image job after this retirement. This source change does not delete runner groups, Fleets, credentials, registry images, or cache tags. Review shared infrastructure ownership separately before removing those resources. +## Protected standard-image builders + +`CORE_RELEASE_AWS_ENABLED=true` routes canonical master image jobs to native +`paperclip-core-image-x64` and `paperclip-core-image-arm64` fleets. Each has two +32-vCPU / 128-GiB Spot slots, for four large image jobs in total. Workers +launch for jobs and terminate afterward; no idle pool is maintained. The `paperclip-core-release` runner group permits only +Core's `docker.yml@refs/heads/master`. PRs, branches, and tag releases keep their +existing GitHub-hosted placement. Disable the switch and rerun the whole image +workflow to restore hosted placement. + +Image checkout retains the complete commit graph for version stamping but filters +out historical blobs. The manifest job needs only a shallow source checkout. +Prepared AWS image disks skip the standard hosted runner's disk cleanup. Native +architecture builds, digest publication, the orphan-reaping smoke test, and +attestation remain required. The source monitor has Actions write permission to cancel its own canonical +master Docker run and failed exact-source verification siblings. It never +automatically retries a failed source check. Its guarded retry permission can +restart only an exact-source producer deferred before full admission. It checks run, attempt, SHA, +repository, ref, and workflow identity before cancellation. Its bounded +observation window does not cancel healthy pending checks. A successful observer +records proof for publication immediately. If healthy checks remain pending after +40 minutes, a separate required proof job waits for up to another 40 minutes on +the verification fleet. Both jobs fit within the Spot worker lifetime. Neither +wait occupies the four-slot publication pool. The hosted manifest job requires +successful source proof and admitted migrator work before it starts; a queued +migrator job alone is not evidence of admission. It retains its 20-minute budget +for manifest verification and signing. It revalidates the latest source attempt +once before publishing; changed or missing proof fails immediately without +starting another wait on the publication runner. +Image jobs retain +their existing package permissions. + +The monitor emits an Actions warning and step summary when an image or source job waits +more than 60 seconds for a runner. This measures runner assignment, not image +execution. Compare automatic merge-to-ready and merge-to-healthy-target timings; +a manual dispatch alone does not prove deployment latency. + ### Typecheck Rust dependency cache Source verification's typecheck job builds the native Runner binary through the @@ -236,3 +287,10 @@ fi A subsequent master install can populate the missing entry. Check the saved archive size and package reuse in install logs; a cache hit alone does not prove that the entry contains dependencies. + +Image manifest verification and attestation, plus exact-source migrator build +and publication, use the protected four-slot GitHub-hosted `cloud-release-32` +pool. `CORE_PUBLICATION_RUNNER=ubuntu-latest` restores standard hosted placement. +The signing jobs remain GitHub-hosted because downstream provenance explicitly +requires that certificate claim. Their master-only group also serves the private +Cloud publisher and harness deployment; it excludes PRs and branch benchmarks. diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 5167a32193f..3ca0bc25840 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -61,6 +61,39 @@ test("Docker Hub mirrors apply to both image builders without widening publicati "the registry-only manifest merge does not need a Docker Hub mirror"); }); +test("source proof gates publication without occupying a publication runner", () => { + const workflow = readWorkflow("docker.yml"); + const observer = workflow.split(" source_verification:\n")[1].split(" reuse_canary_image:")[0]; + const gate = workflow.split(" publication_verification:\n")[1].split(" merge-and-push:")[0]; + const publisher = workflow.split(" merge-and-push:\n")[1].split(" promote_canary_channel:")[0]; + assert.match(observer, /verified: \$\{\{ steps.watch.outputs.verified \}\}/); + assert.match(gate, /run: node scripts\/watch-release-source.mjs --require-proof/); + assert.match(gate, /actions: write # restart only exact-source producers deferred before admission/); + assert.match(gate, /needs: \[build-and-push, source_verification\]/); + assert.match(publisher, /needs: \[build-and-push, source_verification, publication_verification\]/); + assert.doesNotMatch(publisher, /--require-proof|actions: write/); + assert.ok(publisher.indexOf("run: node scripts/watch-release-source.mjs --check-proof") < publisher.indexOf("- name: Create manifest list and push")); + assert.match(publisher, /actions: read/); + assert.match(publisher, /timeout-minutes: 20/); + const expression = publisher.match(/^ if: \$\{\{ (.+) \}\}$/m)[1].replaceAll("needs.build-and-push", "needs.build"); + for (const [ref, observed, fallback, expected, repository = "paperclipai/paperclip", repository_id = "1170821064"] of [ + ["refs/heads/master", "true", "skipped", true], + ["refs/heads/master", "false", "skipped", false], + ["refs/heads/master", undefined, "skipped", false], + ["refs/heads/master", "false", "success", true], + ["refs/heads/master", "false", "failure", false], + ["refs/tags/v1", undefined, "skipped", true], + ["refs/heads/master", undefined, "skipped", true, "fork/paperclip", "123"], + ["refs/heads/master", undefined, "skipped", true, "paperclipai/paperclip", "123"], + ]) { + const canonical = repository === "paperclipai/paperclip" && repository_id === "1170821064" && ref === "refs/heads/master"; + assert.equal(runInNewContext(expression, { cancelled: () => false, github: { ref, repository, repository_id }, needs: { + build: { result: "success" }, source_verification: { result: canonical ? "success" : "skipped", outputs: { verified: observed } }, + publication_verification: { result: fallback }, + } }), expected); + } +}); + test("chaos verification isolates callers that verify the same source commit", () => { const chaosWorkflow = readWorkflow("runner-chaos-evals.yml"); const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1]; @@ -82,11 +115,11 @@ test("chaos verification isolates callers that verify the same source commit", ( assert.match(chaosWorkflow, /cancel-in-progress: true/); }); -test("canary reuses exact-source proof while stable keeps full verification", () => { +test("canary can admit deferred exact-source checks while stable keeps full verification", () => { const releaseWorkflow = readWorkflow("release.yml"); const canary = releaseWorkflow.match(/ verify_canary:\n[\s\S]*?(?=\n [a-z_]+:)/)[0]; - assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); - assert.match(canary, /actions: read/); + assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.repository_id == '1170821064' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); + assert.match(canary, /actions: write # restart only an exact-source push deferred before admission/); assert.match(canary, /ref: \$\{\{ github\.sha \}\}/); assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/); diff --git a/scripts/cloud-release-admission.mjs b/scripts/cloud-release-admission.mjs new file mode 100644 index 00000000000..c7ed134ba4b --- /dev/null +++ b/scripts/cloud-release-admission.mjs @@ -0,0 +1,123 @@ +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +const repository = "paperclipai/paperclip"; +const repositoryId = 1170821064; +const workflows = new Set(["docker.yml", "release.yml", "cloud-readiness.yml", "cloud-migrator-artifacts.yml"]); +export const sourceAdmissionJob = "Cloud candidate admission v1"; +export const migratorAdmissionJob = "Cloud image candidate admission v1"; + +function assertSha(sha) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("Admission requires a full source SHA."); +} + +function trusted(run, sha, workflow) { + return run.workflow_id === workflow.id && run.path === workflow.path && run.head_sha === sha && + run.repository?.id === repositoryId && run.head_repository?.id === repositoryId && + run.repository?.full_name === repository && run.head_repository?.full_name === repository && + run.head_branch === "master" && run.event === "push" && + Number.isSafeInteger(run.id) && run.id > 0 && Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0; +} + +async function producer(file, sha, api, status) { + assertSha(sha); + if (!workflows.has(file)) throw new Error("Unknown release workflow."); + const workflow = await api(`/repos/${repository}/actions/workflows/${file}`); + if (workflow.path !== `.github/workflows/${file}` || !Number.isSafeInteger(workflow.id) || workflow.id < 1) throw new Error("Release workflow identity mismatch."); + const list = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?branch=master&event=push&head_sha=${sha}&per_page=100${status ? `&status=${status}` : ""}`); + if (!Array.isArray(list.workflow_runs) || !Number.isSafeInteger(list.total_count) || list.total_count < 0 || + list.total_count > 100 || list.workflow_runs.length !== list.total_count) throw new Error("Incomplete release workflow listing."); + const run = list.workflow_runs.filter(run => trusted(run, sha, workflow) && (!status || run.status === status)).sort((a, b) => b.id - a.id)[0]; + return { workflow, run }; +} + +// A retry is an explicit request for this exact immutable source. Normal push +// work starts only for a candidate admitted by an image build or npm canary. +export async function releaseAdmitted(sha, { api, event = "push", attempt = 1, imageOnly = false }) { + assertSha(sha); + if (!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release attempt."); + if (!["push", "workflow_dispatch"].includes(event)) return false; + if (event === "workflow_dispatch" || (Number.isSafeInteger(attempt) && attempt > 1)) return true; + for (const file of imageOnly ? ["docker.yml"] : ["docker.yml", "release.yml"]) { + const { workflow, run } = await producer(file, sha, api, "in_progress"); + if (!run) continue; + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (current.id === run.id && trusted(current, sha, workflow) && current.run_attempt === run.run_attempt && current.status === "in_progress") return true; + } + return false; +} + +export function deferredProducer(jobs, run, admissionJob) { + const admission = jobs.filter(job => job.name === admissionJob); + const remaining = jobs.filter(job => job.name !== admissionJob); + return run.status === "completed" && run.conclusion === "success" && admission.length === 1 && admission[0].status === "completed" && admission[0].conclusion === "success" && remaining.length > 0 && + remaining.every(job => job.status === "completed" && job.conclusion === "skipped") && + jobs.every(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === run.head_sha); +} + +// Only skipped admission is restartable here. A failed check or build never +// becomes an automatic retry, and every write revalidates canonical identity. +export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) { + if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract."); + const { workflow, run } = await producer(file, sha, api); + if (!run) return "pending"; + const key = `${run.id}/${run.run_attempt}`; + if (requested.has(key)) return "pending"; + const jobs = []; + let total; + for (let page = 1; page <= 10; page++) { + const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`); + if (!Array.isArray(batch.jobs) || !Number.isSafeInteger(batch.total_count) || batch.total_count < 0) throw new Error("Invalid release producer job list."); + if ((total !== undefined && total !== batch.total_count) || + batch.jobs.length !== Math.min(100, Math.max(0, batch.total_count - (page - 1) * 100))) throw new Error("Incomplete release producer job list."); + total = batch.total_count; + jobs.push(...batch.jobs); + if (page * 100 >= total) break; + if (page === 10) throw new Error("Incomplete release producer job list."); + } + if (run.status !== "completed") { + const current = jobs.filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha); + // Admission itself succeeds for both outcomes. Only an actual build is + // evidence that its output admitted work; queued jobs may still be skipped. + return current.some(job => job.name !== admissionJob && job.labels?.length > 0 && ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped") + ? "admitted" : "pending"; + } + if (!deferredProducer(jobs, run, admissionJob)) return "finished"; + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (current.id !== run.id || !trusted(current, sha, workflow) || current.run_attempt !== run.run_attempt || current.status !== "completed" || current.conclusion !== "success") return "pending"; + try { + await rerun(`/repos/${repository}/actions/runs/${run.id}/rerun`); + } catch (error) { + // Docker and npm may select the same source at the same time. A rejected + // duplicate write is harmless only when a fresh read proves a retry began. + const latest = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (latest.id !== run.id || !trusted(latest, sha, workflow)) throw error; + if (!(latest.run_attempt > run.run_attempt || (latest.run_attempt === run.run_attempt && + ["queued", "in_progress", "requested", "pending", "waiting"].includes(latest.status)))) { + // GitHub may reject a duplicate before its read API shows the retry. + // Leave it pending for the next poll; this is not proof of admission. + if ([409, 422].includes(error.status) && latest.run_attempt === run.run_attempt && latest.status === "completed" && latest.conclusion === "success") return "pending"; + throw error; + } + } + requested.add(key); + return "pending"; +} + +export function createReleaseRerunner(token, fetchImpl = fetch) { + return async path => { + if (!/^\/repos\/paperclipai\/paperclip\/actions\/runs\/[1-9]\d*\/rerun$/.test(path)) throw new Error("Invalid release retry path."); + const response = await fetchImpl(`https://api.github.com${path}`, { method: "POST", redirect: "error", + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw Object.assign(new Error(`Could not admit deferred release work: HTTP ${response.status}.`), { status: response.status }); + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master" || !process.env.GITHUB_TOKEN) throw new Error("Canonical master admission context required."); + const { createActionsReader } = await import("./cloud-source-verification.mjs"); + const admitted = await releaseAdmitted(process.env.GITHUB_SHA, { api: createActionsReader({ token: process.env.GITHUB_TOKEN }), + event: process.env.GITHUB_EVENT_NAME, attempt: Number(process.env.GITHUB_RUN_ATTEMPT), imageOnly: process.argv[2] === "image-only" }); + console.log(admitted ? "Release candidate admitted." : "Full build deferred until a release consumer admits this source."); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `admitted=${admitted}\n`); +} diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs index bd40e33cd3c..6a6f316da6f 100644 --- a/scripts/cloud-source-verification.mjs +++ b/scripts/cloud-source-verification.mjs @@ -1,9 +1,18 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; +import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; const repository = "paperclipai/paperclip"; const workflowPath = ".github/workflows/cloud-readiness.yml"; export const sourceVerificationJob = "Cloud source verified v1"; +export class SourceVerificationFailed extends Error { + constructor(message, run) { + super(message); + this.runId = run.id; + this.attempt = run.run_attempt; + } +} +export class SourceVerificationDeferred extends Error {} function assertSha(sha) { if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required."); @@ -19,7 +28,7 @@ function trustedRun(run, sha, workflowId) { // Consume one versioned job, independent of image/migrator availability. A // failed image build must not invalidate source checks that already passed. -export async function readSourceVerification(sha, api) { +export async function readSourceVerification(sha, api, { failFast = false, onJobs } = {}) { assertSha(sha); const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`); if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) { @@ -46,6 +55,17 @@ export async function readSourceVerification(sha, api) { } const matches = jobs.filter((job) => job.name === sourceVerificationJob); if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous."); + await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt)); + if (failFast) { + const failure = jobs.find((job) => job.status === "completed" && + ["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) && + job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt); + if (failure) { + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined; + throw new SourceVerificationFailed(`Cloud source verification failed: ${failure.name} (${failure.conclusion}).`, run); + } + } const job = matches[0]; if (job?.status === "completed" && job.conclusion === "success" && job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) { @@ -56,13 +76,17 @@ export async function readSourceVerification(sha, api) { return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id }; } if (job?.status === "completed" || run.status === "completed") { - throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`); + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined; + if (job?.conclusion === "skipped" && current.status !== "completed") return undefined; + if (deferredProducer(jobs, current, sourceAdmissionJob)) throw new SourceVerificationDeferred("Source checks are deferred until release admission."); + throw new SourceVerificationFailed(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`, run); } return undefined; } // Statuses that say "ask again", not "the answer is no". A release must not be -// blocked because GitHub returned a gateway error during a 45-minute poll. +// blocked because GitHub returned a gateway error during a 60-minute poll. const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); // A rate-limited read also says "ask again", and GitHub reports both primary @@ -137,14 +161,20 @@ export function createActionsReader({ export async function waitForSourceVerification(sha, { api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), - timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log, + timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun, } = {}) { assertSha(sha); const deadline = now() + timeoutMs; + const requested = new Set(); log(`Waiting for ${sourceVerificationJob} for ${sha}.`); while (now() < deadline) { - const result = await readSourceVerification(sha, api); - if (result) return result; + try { + const result = await readSourceVerification(sha, api); + if (result) return result; + } catch (error) { + if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error; + await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } const remaining = deadline - now(); if (remaining > 0) await sleep(Math.min(intervalMs, remaining)); } @@ -156,10 +186,11 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required."); // One deadline for both layers: the reader stops retrying when the poll it // serves is out of time, instead of extending the wait past its timeout. - const timeoutMs = 45 * 60_000; + const timeoutMs = 60 * 60_000; const deadline = Date.now() + timeoutMs; const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline }); - const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs }); + const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined; + const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun }); const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`; console.log(message); if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`); diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs index 4a47abfd164..8221bba4bd3 100644 --- a/scripts/cloud-source-verification.test.mjs +++ b/scripts/cloud-source-verification.test.mjs @@ -1,6 +1,10 @@ import assert from "node:assert/strict"; -import test from "node:test"; -import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs"; +import test, { describe } from "node:test"; +import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; + +import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; + +import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs"; const sha = "a".repeat(40); const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; @@ -62,7 +66,8 @@ test("job identities and terminal failures fail closed", async () => { { head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 }, { conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" }, ]) { - const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] }); + const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }], + current: overrides.conclusion === "skipped" ? { ...baseRun, status: "completed", conclusion: "success" } : baseRun }); await assert.rejects(readSourceVerification(sha, api), /did not pass/); } }); @@ -231,3 +236,357 @@ test("retries stop at the caller's deadline rather than outliving the poll", asy assert.equal(seen.length, 1); assert.deepEqual(waits, []); }); + + +describe("image build source monitor", () => { + const sha = "a".repeat(40); + const runId = 50; + const attempt = 1; + const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; + const sourceRun = { id: 99, workflow_id: 123, path: workflow.path, run_attempt: 1, + repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" }, + head_sha: sha, head_branch: "master", event: "push", status: "in_progress" }; + const imageRun = { id: runId, run_attempt: attempt, head_sha: sha, head_branch: "master", event: "push", + repository: { id: 1170821064 }, head_repository: { id: 1170821064 }, + path: ".github/workflows/docker.yml", status: "in_progress" }; + const proof = { id: 111, name: "Cloud source verified v1", run_id: 99, run_attempt: 1, + head_sha: sha, status: "completed", conclusion: "success" }; + + function fixture({ jobs = [proof], run = sourceRun, current = run, image = imageRun, imageJobs = [], missing = false } = {}) { + const cancellations = []; + const messages = []; + const api = async path => { + if (path.endsWith("/workflows/cloud-readiness.yml")) return workflow; + if (path.includes("/workflows/123/runs?")) return { workflow_runs: missing ? [] : [run], total_count: missing ? 0 : 1 }; + if (path.includes("/runs/99/attempts/1/jobs?")) return { jobs }; + if (path.endsWith("/runs/99")) return current; + if (path.includes(`/runs/${runId}/attempts/${attempt}/jobs?`)) return { jobs: imageJobs, total_count: imageJobs.length }; + if (path.endsWith(`/runs/${runId}`)) return image; + throw new Error(`Unexpected request: ${path}`); + }; + return { sha, runId, attempt, api, cancel: async path => cancellations.push(path), + cancellations, messages, log: message => messages.push(message), summary: async message => messages.push(message) }; + } + + test("successful exact-source proof permits publication without canceling image work", async () => { + const f = fixture(); + assert.equal((await watchReleaseSource(f)).jobId, proof.id); + assert.deepEqual(f.cancellations, []); + }); + + test("a required check failing cancels the same image attempt before siblings finish", async () => { + const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }] }); + await assert.rejects(watchReleaseSource(f), /General tests/); + assert.deepEqual(f.cancellations, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", `/repos/paperclipai/paperclip/actions/runs/${runId}/cancel`]); + }); + + test("a superseded readiness attempt cancels image work with no deployable future", async () => { + const f = fixture({ jobs: [], run: { ...sourceRun, status: "completed", conclusion: "cancelled" } }); + await assert.rejects(watchReleaseSource(f), /did not pass/); + assert.equal(f.cancellations.length, 1); + }); + + test("a rerun racing a failed job is retried rather than canceling the image", async () => { + const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }], + current: { ...sourceRun, run_attempt: 2 } }); + let time = 0; + const api = f.api; + f.api = async path => time > 0 ? fixture().api(path) : api(path); + const result = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; } }); + assert.equal(result.jobId, proof.id); + assert.deepEqual(f.cancellations, []); + }); + + test("a transport failure does not issue a destructive cancellation", async () => { + const f = fixture(); + await assert.rejects(watchReleaseSource({ ...f, api: async () => { throw new Error("GitHub unavailable"); } }), /unavailable/); + assert.deepEqual(f.cancellations, []); + }); + + test("the observation window ends without cancelling healthy pending verification", async () => { + const f = fixture({ missing: true }); + let time = 0; + const proof = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; }, timeoutMs: 20 }); + assert.equal(proof, undefined); + assert.equal(time, 20); + assert.deepEqual(f.cancellations, []); + assert.ok(f.messages.some(message => message.includes("final publication still requires exact-source verification"))); + }); + + test("final publication requires proof even after the early observation window ends", async () => { + const f = fixture({ missing: true }); + let time = 0; + await assert.rejects(watchReleaseSource({ ...f, requireProof: true, now: () => time, + sleep: async ms => { time += ms; }, timeoutMs: 20 }), /before the publication deadline/); + assert.deepEqual(f.cancellations, []); + assert.equal((await watchReleaseSource({ ...fixture(), requireProof: true })).jobId, proof.id); + }); + + test("cancellation rejects another SHA, repository, ref, workflow, or attempt", async () => { + for (const change of [{ head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "pull_request" }, + { path: ".github/workflows/release.yml" }, { run_attempt: 2 }, { id: 51 }, + { repository: { id: 42 } }, { head_repository: { id: 42 } }]) { + const f = fixture({ image: { ...imageRun, ...change } }); + await assert.rejects(cancelImageRun(f), /different image workflow identity/); + assert.deepEqual(f.cancellations, []); + } + }); + + test("queue measurement does not count running time or skipped dependency jobs", () => { + const created_at = "2020-01-01T00:00:00Z"; + const started_at = "2020-01-01T00:00:20Z"; + const job = { id: 1, name: "amd64", labels: ["release"], created_at, started_at, runner_id: 10, status: "in_progress" }; + const now = Date.parse(created_at) + 120_000; + assert.deepEqual(runnerWaits([job, { ...job, status: "completed", conclusion: "skipped" }], now), [{ id: 1, job: "amd64", seconds: 20 }]); + assert.equal(runnerWaits([{ ...job, status: "queued", runner_id: 0 }], now)[0].seconds, 120); + }); + + test("a runner wait over 60 seconds produces an Actions warning and summary", async () => { + const f = fixture({ imageJobs: [{ id: 1, name: "amd64", labels: ["release"], status: "queued", runner_id: 0, created_at: "2020-01-01T00:00:00Z" }] }); + await watchReleaseSource({ ...f, now: () => Date.parse("2020-01-01T00:02:00Z") }); + assert.ok(f.messages.some(message => message.startsWith("::warning") && message.includes("120s"))); + assert.ok(f.messages.some(message => message.includes("Release runner wait: amd64"))); + }); +}); + +describe("release candidate admission", () => { + const repository = { id: 1170821064, full_name: "paperclipai/paperclip" }; + const ids = { "docker.yml": 101, "release.yml": 102, "cloud-readiness.yml": 123, "cloud-migrator-artifacts.yml": 124 }; + function run(file, change = {}) { + return { id: ids[file] + 1000, workflow_id: ids[file], path: `.github/workflows/${file}`, + repository, head_repository: repository, head_sha: sha, head_branch: "master", event: "push", + run_attempt: 1, status: "in_progress", conclusion: null, ...change }; + } + function job(run, name, conclusion = "skipped") { + return { id: 789, run_id: run.id, run_attempt: run.run_attempt, head_sha: run.head_sha, + name, status: "completed", conclusion }; + } + function fixture(file = "cloud-readiness.yml", change = {}) { + const producer = run(file, { status: "completed", conclusion: "success", ...change }); + const admission = file === "cloud-migrator-artifacts.yml" ? migratorAdmissionJob : sourceAdmissionJob; + const f = { runs: { [file]: [producer] }, jobs: { [producer.id]: [job(producer, admission, "success"), job(producer, sourceVerificationJob)] }, + current: {}, writes: [], calls: [], requested: new Set(), file, sha, admissionJob: admission }; + f.api = async path => { + f.calls.push(path); + const url = new URL(`https://api.github.com${path}`); + const fileMatch = path.match(/\/workflows\/([^/?]+\.yml)$/); + if (fileMatch) return { id: ids[fileMatch[1]], path: `.github/workflows/${fileMatch[1]}` }; + const listMatch = path.match(/\/workflows\/(\d+)\/runs\?/); + if (listMatch) { + const name = Object.keys(ids).find(file => ids[file] === Number(listMatch[1])); + const runs = f.runs[name] ?? []; + return { workflow_runs: runs, total_count: f.total ?? runs.length }; + } + const jobMatch = path.match(/\/runs\/(\d+)\/attempts\/(\d+)\/jobs\?/); + if (jobMatch) { + const page = Number(url.searchParams.get("page") ?? "1"); + const jobs = f.jobs[Number(jobMatch[1])] ?? []; + return { total_count: f.jobTotal ?? jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100) }; + } + const currentMatch = path.match(/\/runs\/(\d+)$/); + if (currentMatch) return f.current[Number(currentMatch[1])] ?? Object.values(f.runs).flat().find(run => run.id === Number(currentMatch[1])); + throw new Error(`Unexpected admission API path: ${path}`); + }; + f.rerun = async path => f.writes.push(path); + return f; + } + + test("only a running image or npm release admits a normal push", async () => { + for (const file of ["docker.yml", "release.yml"]) { + const f = fixture(); + assert.equal(await releaseAdmitted(sha, f), false); + f.runs[file] = [run(file)]; + assert.equal(await releaseAdmitted(sha, f), true); + assert.equal(await releaseAdmitted(sha, { ...f, imageOnly: true }), file === "docker.yml"); + for (const status of ["queued", "pending", "completed"]) { + f.runs[file] = [run(file, { status })]; + assert.equal(await releaseAdmitted(sha, f), false, status); + } + } + }); + + test("manual and retry requests admit exact source without consumer discovery", async () => { + const api = () => assert.fail("explicit request needs no discovery"); + assert.equal(await releaseAdmitted(sha, { api, event: "workflow_dispatch" }), true); + assert.equal(await releaseAdmitted(sha, { api, attempt: 2 }), true); + assert.equal(await releaseAdmitted(sha, { api, event: "pull_request", attempt: 2 }), false); + for (const attempt of [0, "2"]) { + await assert.rejects(releaseAdmitted(sha, { api, attempt }), /Invalid release attempt/); + } + }); + + test("forks, stale identities, tags, and non-push consumers do not admit work", async () => { + for (const change of [{ repository: { ...repository, id: 2 } }, { head_repository: { ...repository, id: 2 } }, + { repository: { ...repository, full_name: "fork/paperclip" } }, { path: ".github/workflows/pr.yml" }, + { workflow_id: 999 }, { head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "workflow_dispatch" }, + { id: 0 }, { run_attempt: 0 }]) { + const f = fixture(); + f.runs["docker.yml"] = [run("docker.yml", change)]; + assert.equal(await releaseAdmitted(sha, f), false, JSON.stringify(change)); + } + const f = fixture(); + const active = run("docker.yml"); + f.runs["docker.yml"] = [active]; + f.current[active.id] = { ...active, run_attempt: 2 }; + assert.equal(await releaseAdmitted(sha, f), false); + }); + + test("incomplete discovery and transport failure fail closed", async () => { + for (const total of [1, 101, -1]) { + const f = fixture(); + f.total = total; + await assert.rejects(releaseAdmitted(sha, f), /Incomplete/); + assert.deepEqual(f.writes, []); + } + await assert.rejects(releaseAdmitted(sha, { api: async () => { throw new Error("API unavailable"); } }), /unavailable/); + await assert.rejects(releaseAdmitted("master", { api: () => assert.fail() }), /full source SHA/); + }); + + test("a selected consumer restarts only a strictly deferred producer, once", async () => { + for (const file of ["cloud-readiness.yml", "cloud-migrator-artifacts.yml"]) { + const f = fixture(file); + assert.equal(await requestDeferredProducer(f), "pending"); + assert.deepEqual(f.writes, [`/repos/paperclipai/paperclip/actions/runs/${ids[file] + 1000}/rerun`]); + await requestDeferredProducer(f); + assert.equal(f.writes.length, 1); + } + }); + + test("failed, incomplete, ambiguous, or mismatched work is never automatically retried", async () => { + for (const change of [jobs => { jobs[1].conclusion = "failure"; }, jobs => { jobs[0].conclusion = "failure"; }, + jobs => { jobs[1].run_attempt = 2; }, jobs => { jobs[1].head_sha = "b".repeat(40); }, + jobs => { jobs[1].run_id = 2; }, jobs => { jobs[1].status = "queued"; }, + jobs => { jobs.push({ ...jobs[0] }); }, jobs => { jobs.splice(1); }]) { + const f = fixture(); + change(f.jobs[1123]); + assert.equal(await requestDeferredProducer(f), "finished"); + assert.deepEqual(f.writes, []); + } + const f = fixture(); + for (const change of [{ run_attempt: 2 }, { status: "in_progress" }, { head_sha: "b".repeat(40) }, { id: 2 }]) { + f.current[1123] = { ...f.runs[f.file][0], ...change }; + assert.equal(await requestDeferredProducer(f), "pending"); + assert.deepEqual(f.writes, []); + } + }); + + test("an explicitly cancelled or failed producer cannot be restarted by admission", async () => { + for (const conclusion of ["cancelled", "failure", "timed_out"]) { + const f = fixture("cloud-readiness.yml", { conclusion }); + assert.equal(await requestDeferredProducer(f), "finished"); + assert.deepEqual(f.writes, []); + } + }); + + test("a truncated job page cannot hide a failed check and trigger a retry", async () => { + const f = fixture(); + f.jobTotal = 3; + await assert.rejects(requestDeferredProducer(f), /Incomplete release producer job list/); + assert.deepEqual(f.writes, []); + f.jobTotal = "2"; + await assert.rejects(requestDeferredProducer(f), /Invalid release producer job list/); + assert.deepEqual(f.writes, []); + }); + + test("producer retries reject arbitrary workflow and admission contracts", async () => { + const f = fixture(); + for (const change of [{ file: "docker.yml" }, { admissionJob: migratorAdmissionJob }]) { + await assert.rejects(requestDeferredProducer({ ...f, ...change }), /Unknown deferred/); + } + assert.deepEqual(f.calls, []); + }); + + test("a deferred completion racing the run listing does not become a source failure", async () => { + const f = fixture(); + const completed = f.runs[f.file][0]; + f.runs[f.file] = [{ ...completed, status: "in_progress", conclusion: null }]; + f.current[completed.id] = completed; + await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred); + delete f.current[completed.id]; + assert.equal(await readSourceVerification(sha, f.api), undefined); + }); + + test("a competing consumer's accepted retry does not fail the losing consumer", async () => { + const f = fixture(); + f.rerun = async () => { + f.current[1123] = run(f.file, { run_attempt: 2 }); + throw new Error("duplicate restart rejected"); + }; + assert.equal(await requestDeferredProducer(f), "pending"); + assert.equal(f.requested.size, 1); + const rejected = fixture(); + rejected.rerun = async () => { throw new Error("token denied"); }; + await assert.rejects(requestDeferredProducer(rejected), /token denied/); + assert.equal(rejected.requested.size, 0); + }); + + test("a queued migrator is pending until its build actually starts", async () => { + const f = fixture("cloud-migrator-artifacts.yml", { status: "in_progress", conclusion: null }); + const producer = f.runs[f.file][0]; + const build = { ...job(producer, "Build migrator", null), status: "queued", labels: ["cloud-release-32"] }; + f.jobs[producer.id] = [job(producer, migratorAdmissionJob, "success"), build]; + assert.equal(await requestDeferredProducer(f), "pending"); + build.status = "in_progress"; + assert.equal(await requestDeferredProducer(f), "admitted"); + build.run_attempt = 2; + assert.equal(await requestDeferredProducer(f), "pending", "a stale build cannot admit work"); + assert.deepEqual(f.writes, []); + }); + + test("a rejected duplicate with stale read visibility remains pending without claiming admission", async () => { + for (const status of [409, 422, 403]) { + const f = fixture(); + f.rerun = async () => { throw Object.assign(new Error("retry rejected"), { status }); }; + if (status === 403) await assert.rejects(requestDeferredProducer(f), /retry rejected/); + else assert.equal(await requestDeferredProducer(f), "pending"); + assert.equal(f.requested.size, 0); + } + }); + + test("source proof waits for admission, restarts it, and accepts only the new attempt", async () => { + const f = fixture(); + await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred); + let time = 0; + const proof = await waitForSourceVerification(sha, { ...f, now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, + sleep: async ms => { + time += ms; + const selected = run(f.file, { run_attempt: 2 }); + f.runs[f.file] = [selected]; + f.jobs[selected.id] = [job(selected, sourceAdmissionJob, "success"), job(selected, sourceVerificationJob, "success")]; + } }); + assert.equal(proof.attempt, 2); + assert.equal(time, 10); + assert.equal(f.writes.length, 1); + }); + + test("the image monitor admits a late migrator even when source proof is ready", async () => { + const f = fixture("cloud-migrator-artifacts.yml"); + const source = run("cloud-readiness.yml"); + f.runs["cloud-readiness.yml"] = [source]; + f.jobs[source.id] = [job(source, sourceVerificationJob, "success")]; + const image = run("docker.yml"); + f.runs["docker.yml"] = [image]; + f.jobs[image.id] = []; + let time = 0; + const proof = await watchReleaseSource({ ...f, runId: image.id, attempt: 1, cancel: () => assert.fail("must not cancel"), + now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, summary: async () => {}, sleep: async ms => { + time += ms; + const selected = run(f.file, { run_attempt: 2 }); + f.runs[f.file] = [selected]; + f.jobs[selected.id] = [{ ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"] }]; + } }); + assert.equal(proof.sha, sha); + assert.equal(time, 10); + assert.equal(f.writes.length, 1); + }); + + test("the retry transport is a bounded POST to a canonical run", async () => { + const calls = []; + const rerun = createReleaseRerunner("fixture", async (url, options) => { calls.push({ url, options }); return { ok: true }; }); + await assert.rejects(rerun("/repos/fork/paperclip/actions/runs/1/rerun"), /Invalid release retry/); + await rerun("/repos/paperclipai/paperclip/actions/runs/1/rerun"); + assert.equal(calls.length, 1); + assert.equal(calls[0].options.method, "POST"); + assert.equal(calls[0].options.redirect, "error"); + }); +}); diff --git a/scripts/watch-release-source.mjs b/scripts/watch-release-source.mjs new file mode 100644 index 00000000000..bf4fe032774 --- /dev/null +++ b/scripts/watch-release-source.mjs @@ -0,0 +1,108 @@ +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; +import { requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; + +const repository = "paperclipai/paperclip"; + +export function runnerWaits(jobs, now = Date.now()) { + return jobs.filter(job => ["queued", "in_progress", "completed"].includes(job.status) && + job.conclusion !== "skipped" && job.labels?.length > 0).map(job => ({ + job: job.name, id: job.id, + seconds: Math.max(0, Math.floor(((job.runner_id > 0 && job.started_at ? Date.parse(job.started_at) : now) - Date.parse(job.created_at)) / 1000)), + })).filter(job => Number.isFinite(job.seconds)); +} + +// The write is limited to this canonical master Docker run. Never cancel +// another workflow, a tag release, or a run selected from untrusted input. +export async function cancelImageRun({ api, cancel, runId, attempt, sha }) { + const run = await api(`/repos/${repository}/actions/runs/${runId}`); + if (run.id !== runId || run.run_attempt !== attempt || run.head_sha !== sha || run.head_branch !== "master" || + run.repository?.id !== 1170821064 || run.head_repository?.id !== 1170821064 || + run.path !== ".github/workflows/docker.yml" || !["push", "workflow_dispatch"].includes(run.event)) { + throw new Error("Refusing to cancel a different image workflow identity."); + } + if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${runId}/cancel`); +} + +async function cancelFailedVerification({ api, cancel, sha }, failure) { + const run = await api(`/repos/${repository}/actions/runs/${failure.runId}`); + // A concurrent retry owns the new attempt; a late observer leaves it alone. + if (run.run_attempt !== failure.attempt) return; + if (run.id !== failure.runId || run.head_sha !== sha || run.head_branch !== "master" || run.event !== "push" || + run.repository?.full_name !== repository || run.head_repository?.full_name !== repository || + run.path !== ".github/workflows/cloud-readiness.yml") throw new Error("Failed verification identity changed."); + if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${failure.runId}/cancel`); +} + +export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, now = Date.now, + sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), log = console.log, + timeoutMs = 40 * 60_000, intervalMs = 30_000, requireProof = false, summary = async () => {} }) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !Number.isSafeInteger(runId) || runId < 1 || + !Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release identity."); + const deadline = now() + timeoutMs; + const warned = new Set(); + const requested = new Set(); + let migratorAdmitted = !rerun; + const reportWaits = async jobs => { + for (const wait of runnerWaits(jobs, now())) { + if (wait.seconds > 60 && !warned.has(wait.id)) { + warned.add(wait.id); + log(`::warning title=Release runner waited over 60 seconds::Job ${wait.id} waited ${wait.seconds}s for a runner.`); + await summary(`Release runner wait: ${wait.job} — ${wait.seconds}s (job ${wait.id}).\n`); + } + } + }; + while (now() < deadline) { + if (!migratorAdmitted) { + migratorAdmitted = await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }) !== "pending"; + } + const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`); + if (!Array.isArray(batch.jobs) || batch.total_count > 100) throw new Error("Incomplete image job listing."); + await reportWaits(batch.jobs); + try { + const proof = await readSourceVerification(sha, api, { failFast: true, onJobs: reportWaits }); + if (proof && migratorAdmitted) { + await summary(`Exact source verified: ${sha}; readiness run ${proof.runId}, attempt ${proof.attempt}.\n`); + return proof; + } + } catch (error) { + if (error instanceof SourceVerificationDeferred && rerun) { + await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } else { + if (error instanceof SourceVerificationFailed) { + await cancelFailedVerification({ api, cancel, sha }, error); + await cancelImageRun({ api, cancel, runId, attempt, sha }); + } + throw error; + } + } + await sleep(Math.min(intervalMs, Math.max(0, deadline - now()))); + } + // This early failure observer is not the publication deadline. Healthy + // checks can outlive a Spot worker; a separate exact-source gate still + // waits for proof before publishing any image. + if (requireProof) throw new Error("Exact-source verification did not become ready before the publication deadline."); + await summary("Source monitoring window ended; final publication still requires exact-source verification.\n"); + return undefined; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const token = process.env.GITHUB_TOKEN; + if (!token || process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Canonical master token required."); + const requireProof = process.argv.includes("--require-proof"); + const checkProof = process.argv.includes("--check-proof"); + const api = createActionsReader({ token }); + const proof = checkProof ? await readSourceVerification(process.env.GITHUB_SHA, api, { failFast: true }) : await watchReleaseSource({ requireProof, sha: process.env.GITHUB_SHA, runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT), + api, + rerun: createReleaseRerunner(token), + cancel: async path => { + const response = await fetch(`https://api.github.com${path}`, { method: "POST", redirect: "error", + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw new Error(`Could not cancel failed-source image work: HTTP ${response.status}.`); + }, + summary: text => process.env.GITHUB_STEP_SUMMARY ? appendFile(process.env.GITHUB_STEP_SUMMARY, text) : Promise.resolve(), + }); + if (checkProof && !proof) throw new Error("Exact-source proof changed before publication."); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `verified=${Boolean(proof)}\n`); +} From 523775f78f999f4c30c352018b5edc1bc8b7189a Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Tue, 3 Nov 2026 16:06:53 -0800 Subject: [PATCH 04/28] fix(ui): skip keyless document rows instead of crashing the task panel (#18870) Keep issue-document rows without a string key out of the task panel list and the document caches, so one malformed row cannot crash the page. Co-Authored-By: Claude Opus 5.5 --- .../IssueDocumentsSection.keyless.test.tsx | 161 ++++++++++++++++++ ui/src/components/IssueDocumentsSection.tsx | 6 +- ui/src/lib/issue-artifacts.test.ts | 3 + ui/src/lib/issue-artifacts.ts | 8 +- 4 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 ui/src/components/IssueDocumentsSection.keyless.test.tsx diff --git a/ui/src/components/IssueDocumentsSection.keyless.test.tsx b/ui/src/components/IssueDocumentsSection.keyless.test.tsx new file mode 100644 index 00000000000..d0d2aa6a88d --- /dev/null +++ b/ui/src/components/IssueDocumentsSection.keyless.test.tsx @@ -0,0 +1,161 @@ +// @vitest-environment jsdom + +import { act } from "react"; +import type { ComponentProps, ReactNode } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { Issue, IssueDocument } from "@paperclipai/shared"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { IssueDocumentsSection } from "./IssueDocumentsSection"; +import { queryKeys } from "../lib/queryKeys"; + +const mockIssuesApi = vi.hoisted(() => ({ + listDocuments: vi.fn(), + listDocumentRevisions: vi.fn(), + restoreDocumentRevision: vi.fn(), + upsertDocument: vi.fn(), + lockDocument: vi.fn(), + unlockDocument: vi.fn(), + deleteDocument: vi.fn(), + getDocument: vi.fn(), +})); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true; + +vi.mock("../api/issues", () => ({ issuesApi: mockIssuesApi })); +vi.mock("../api/auth", () => ({ authApi: { getSession: async () => ({ user: { id: "user-1" } }) } })); +vi.mock("@/lib/router", () => ({ useLocation: () => ({ hash: "" }) })); +vi.mock("./IssueDocumentAnnotations", () => ({ + DocumentAnnotationsCountChip: () => null, + IssueDocumentAnnotations: ({ children }: { children: ReactNode }) => <>{children}, +})); +vi.mock("./DocumentMarkdownBody", () => ({ + DocumentMarkdownBody: ({ children }: { children: string }) =>
{children}
, +})); +vi.mock("./MarkdownBody", () => ({ + MarkdownBody: ({ children }: { children: string }) =>
{children}
, +})); +vi.mock("./MarkdownEditor", () => ({ + MarkdownEditor: ({ value }: { value: string }) =>
{value}
, +})); +vi.mock("@/components/ui/button", () => ({ + Button: ({ children, type = "button", ...props }: ComponentProps<"button">) => ( + + ), +})); +vi.mock("@/components/ui/dropdown-menu", () => ({ + DropdownMenu: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuTrigger: ({ children }: { children: ReactNode }) => <>{children}, + DropdownMenuContent: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuItem: ({ children, onClick }: { children: ReactNode; onClick?: () => void }) => ( + + ), + DropdownMenuLabel: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuRadioGroup: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuRadioItem: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuSeparator: () =>
, +})); + +function createIssueDocument(overrides: Partial = {}): IssueDocument { + return { + id: "document-1", + companyId: "company-1", + issueId: "issue-1", + key: "plan", + title: "Plan", + format: "markdown", + body: "", + latestRevisionId: "revision-1", + latestRevisionNumber: 1, + createdByAgentId: null, + createdByUserId: "user-1", + updatedByAgentId: null, + updatedByUserId: "user-1", + lockedAt: null, + lockedByAgentId: null, + lockedByUserId: null, + createdAt: new Date("2026-03-31T12:00:00.000Z"), + updatedAt: new Date("2026-03-31T12:05:00.000Z"), + ...overrides, + }; +} + +function createIssue(): Issue { + return { + id: "issue-1", + companyId: "company-1", + identifier: "DOC-1", + title: "Keyless documents", + documentSummaries: [], + legacyPlanDocument: null, + } as unknown as Issue; +} + +/** A cached row that lost its key, as seen by the task panel before this fix. */ +function keylessDocument(overrides: Partial = {}): IssueDocument { + return { ...createIssueDocument(overrides), key: undefined } as unknown as IssueDocument; +} + +async function flush() { + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); +} + +describe("IssueDocumentsSection keyless rows", () => { + let container: HTMLDivElement; + let root: Root; + let client: QueryClient; + + beforeEach(() => { + vi.clearAllMocks(); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + client.clear(); + container.remove(); + }); + + it.each([false, true])("skips a cached row without a key (general only: %s)", async generalDocumentsOnly => { + mockIssuesApi.listDocuments.mockResolvedValue([ + keylessDocument({ id: "keyless", body: "Keyless row content" }), + createIssueDocument({ id: "note", key: "notes", title: "Task note", body: "General document content" }), + ]); + await act(async () => root.render( + + + , + )); + await flush(); + expect(container.textContent).toContain("General document content"); + expect(container.textContent).not.toContain("Keyless row content"); + }); + + it("keeps a keyless mutation response out of the cached document list", async () => { + const document = createIssueDocument({ body: "Lockable plan body" }); + // Hold the post-lock refetch so the test observes the synced cache, not fresh server rows. + mockIssuesApi.listDocuments + .mockResolvedValueOnce([document]) + .mockReturnValue(new Promise(() => {})); + mockIssuesApi.lockDocument.mockResolvedValue(keylessDocument({ lockedAt: new Date("2026-03-31T12:06:00.000Z") })); + await act(async () => root.render( + + + , + )); + await flush(); + await act(async () => container.querySelector('button[title="Lock document"]')?.click()); + await flush(); + expect(mockIssuesApi.lockDocument).toHaveBeenCalledWith("issue-1", "plan"); + const cached = client.getQueryData(queryKeys.issues.documents("issue-1")) ?? []; + expect(cached.map(entry => entry.key)).toEqual(["plan"]); + expect(container.textContent).toContain("Lockable plan body"); + }); +}); diff --git a/ui/src/components/IssueDocumentsSection.tsx b/ui/src/components/IssueDocumentsSection.tsx index 22e80663b05..1c2f53a2f50 100644 --- a/ui/src/components/IssueDocumentsSection.tsx +++ b/ui/src/components/IssueDocumentsSection.tsx @@ -452,6 +452,8 @@ function AccountIssueDocumentsSection({ }, [documentSubject, queryClient]); const syncDocumentCaches = useCallback((document: IssueDocument) => { + // Only a complete document can replace a list row; the refetch fills the gap. + if (typeof document?.key !== "string") return; if (documentSubject.hideSystemDocuments && isSystemIssueDocumentKey(document.key)) return; queryClient.setQueryData( documentSubject.documentsQueryKey, @@ -527,8 +529,10 @@ function AccountIssueDocumentsSection({ }); const sortedDocuments = useMemo(() => { + // A keyless row cannot be rendered or edited; skip it instead of failing the whole page. return (documents ?? []).filter((doc) => - (!generalDocumentsOnly || isGeneralTaskDocumentKey(doc.key)) + typeof doc?.key === "string" + && (!generalDocumentsOnly || isGeneralTaskDocumentKey(doc.key)) && (!documentSubject.hideSystemDocuments || !isSystemIssueDocumentKey(doc.key)), ).sort((a, b) => { if (a.key === "plan" && b.key !== "plan") return -1; diff --git a/ui/src/lib/issue-artifacts.test.ts b/ui/src/lib/issue-artifacts.test.ts index b10c0c7acfd..ff87c35dca7 100644 --- a/ui/src/lib/issue-artifacts.test.ts +++ b/ui/src/lib/issue-artifacts.test.ts @@ -16,6 +16,9 @@ it.each(["plan", " Plan ", "artifact-review-output", "pipeline-case-body", "cont it("allows ordinary general document keys", () => { expect(isGeneralTaskDocumentKey("notes")).toBe(true); }); +it.each([undefined, null])("treats a missing document key (%s) as not general", key => { + expect(isGeneralTaskDocumentKey(key)).toBe(false); +}); function makeAttachment(overrides: Partial & { id: string }): IssueAttachment { return { diff --git a/ui/src/lib/issue-artifacts.ts b/ui/src/lib/issue-artifacts.ts index c4be8194406..624befe1248 100644 --- a/ui/src/lib/issue-artifacts.ts +++ b/ui/src/lib/issue-artifacts.ts @@ -1,8 +1,12 @@ import type { IssueAttachment, IssueDocumentSummary, IssueWorkProduct } from "@paperclipai/shared"; import { getAttachmentArtifactWorkProductMetadata, isArtifactReviewDocumentKey, isSystemIssueDocumentKey } from "@paperclipai/shared"; -/** Documents authored here; Plans and file reviews retain their own workflows. */ -export function isGeneralTaskDocumentKey(key: string): boolean { +/** + * Documents authored here; Plans and file reviews retain their own workflows. + * A cached row without a string key cannot be authored, so it is never general. + */ +export function isGeneralTaskDocumentKey(key: string | null | undefined): boolean { + if (typeof key !== "string") return false; return key.trim().toLowerCase() !== "plan" && !isArtifactReviewDocumentKey(key) && !isSystemIssueDocumentKey(key); } From 44a4ca1c6148a2bed33601415bd7cb0ad4e339b4 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:21:55 -0600 Subject: [PATCH 05/28] fix(ci): remove the admission CLI import cycle (#18889) Move the unchanged Actions read transport into an independent module so the admission CLI can evaluate consumers without awaiting a circular import of source verification. Preserve the existing source verification export and all verification and placement contracts. Five child-process cases reproduce the original exit-13 failure before the repair and pass afterward. All 59 source/admission/image checks and 665 workflow contracts pass. Both CLI modes also pass read-only smoke checks against real GitHub metadata. Complete latest-head CI passed on its first attempt, with Greptile 5/5 and no unresolved comments. Co-Authored-By: Paperclip --- scripts/cloud-release-admission.mjs | 2 +- scripts/cloud-source-verification.mjs | 75 +--------------------- scripts/cloud-source-verification.test.mjs | 56 ++++++++++++++++ scripts/github-actions-reader.mjs | 73 +++++++++++++++++++++ 4 files changed, 132 insertions(+), 74 deletions(-) create mode 100644 scripts/github-actions-reader.mjs diff --git a/scripts/cloud-release-admission.mjs b/scripts/cloud-release-admission.mjs index c7ed134ba4b..31ca67fa9db 100644 --- a/scripts/cloud-release-admission.mjs +++ b/scripts/cloud-release-admission.mjs @@ -1,5 +1,6 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; +import { createActionsReader } from "./github-actions-reader.mjs"; const repository = "paperclipai/paperclip"; const repositoryId = 1170821064; @@ -115,7 +116,6 @@ export function createReleaseRerunner(token, fetchImpl = fetch) { if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { if (process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master" || !process.env.GITHUB_TOKEN) throw new Error("Canonical master admission context required."); - const { createActionsReader } = await import("./cloud-source-verification.mjs"); const admitted = await releaseAdmitted(process.env.GITHUB_SHA, { api: createActionsReader({ token: process.env.GITHUB_TOKEN }), event: process.env.GITHUB_EVENT_NAME, attempt: Number(process.env.GITHUB_RUN_ATTEMPT), imageOnly: process.argv[2] === "image-only" }); console.log(admitted ? "Release candidate admitted." : "Full build deferred until a release consumer admits this source."); diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs index 6a6f316da6f..09c4687fed1 100644 --- a/scripts/cloud-source-verification.mjs +++ b/scripts/cloud-source-verification.mjs @@ -1,5 +1,6 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; +import { createActionsReader } from "./github-actions-reader.mjs"; import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; const repository = "paperclipai/paperclip"; @@ -85,79 +86,7 @@ export async function readSourceVerification(sha, api, { failFast = false, onJob return undefined; } -// Statuses that say "ask again", not "the answer is no". A release must not be -// blocked because GitHub returned a gateway error during a 60-minute poll. -const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); - -// A rate-limited read also says "ask again", and GitHub reports both primary -// and secondary rate limits as 403. Only the headers separate that from a -// token that may not read Actions, which must still fail at once. -function rateLimited(response) { - if (response.status !== 403) return false; - const header = (name) => response.headers?.get?.(name) ?? null; - return header("retry-after") !== null || header("x-ratelimit-remaining") === "0"; -} - -/** Milliseconds from a Retry-After header, when it carries a sane delay. */ -function retryAfterMs(response, capMs) { - const value = Number(response?.headers?.get?.("retry-after")); - if (!Number.isFinite(value) || value <= 0) return null; - return Math.min(value * 1_000, capMs); -} - -/** - * The GitHub Actions read used by the polling below, with transient transport - * failures retried: network errors, the statuses above, rate-limited 403s, and - * a body that fails while it is being read. Any other non-OK status throws on - * the first response, because waiting out a wrong token only delays the news. - * - * `deadlineAt` bounds retries by the caller's own polling deadline, so a read - * cannot extend the wait past the timeout it belongs to. - */ -export function createActionsReader({ - token, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), - attempts = 4, backoffMs = 1_000, log = console.log, - now = Date.now, deadlineAt = () => Infinity, maxRetryAfterMs = 60_000, -} = {}) { - return async (path) => { - for (let attempt = 1; ; attempt += 1) { - // Retry only while both the attempt budget and the caller's deadline - // leave room for the wait this attempt would cost. - const waitMs = backoffMs * attempt; - const retryable = attempt < attempts && now() + waitMs < deadlineAt(); - const pause = (response) => sleep(retryAfterMs(response, maxRetryAfterMs) ?? waitMs); - let response; - try { - response = await fetchImpl(`https://api.github.com${path}`, { - headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, - signal: AbortSignal.timeout(30_000), redirect: "error", - }); - } catch (cause) { - if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); - log(`GitHub Actions read failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); - await pause(); - continue; - } - if (response.ok) { - try { - // A 200 whose body is truncated or undecodable is a transport - // failure like any other, so it belongs inside the retry. - return await response.json(); - } catch (cause) { - if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); - log(`GitHub Actions read body failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); - await pause(); - continue; - } - } - if (!retryable || !(TRANSIENT_READ_STATUSES.has(response.status) || rateLimited(response))) { - throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`); - } - log(`GitHub Actions read failed (HTTP ${response.status}); retrying (${attempt}/${attempts - 1}).`); - await pause(response); - } - }; -} +export { createActionsReader }; export async function waitForSourceVerification(sha, { api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs index 8221bba4bd3..6e055607267 100644 --- a/scripts/cloud-source-verification.test.mjs +++ b/scripts/cloud-source-verification.test.mjs @@ -1,5 +1,10 @@ import assert from "node:assert/strict"; import test, { describe } from "node:test"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; @@ -7,6 +12,57 @@ import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmis import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs"; const sha = "a".repeat(40); + +function runAdmissionCli({ active = false, imageOnly = false, repositoryName = "paperclipai/paperclip" } = {}) { + const root = mkdtempSync(join(tmpdir(), "release-admission-cli-")); + try { + const preload = join(root, "github-fixture.mjs"); + const installFixture = (active) => { + const repository = { id: 1170821064, full_name: "paperclipai/paperclip" }; + const run = { id: 77, workflow_id: 21, path: ".github/workflows/docker.yml", head_sha: "a".repeat(40), + repository, head_repository: repository, head_branch: "master", event: "push", run_attempt: 1, status: "in_progress" }; + globalThis.fetch = async url => { + const parsed = new URL(url); + if (parsed.origin !== "https://api.github.com") throw new Error("Unexpected admission request origin"); + const path = parsed.pathname; + if (path.endsWith("/docker.yml")) return Response.json({ id: 21, path: ".github/workflows/docker.yml" }); + if (path.endsWith("/release.yml")) return Response.json({ id: 22, path: ".github/workflows/release.yml" }); + if (path.endsWith("/workflows/21/runs")) return Response.json({ total_count: active ? 1 : 0, workflow_runs: active ? [run] : [] }); + if (path.endsWith("/workflows/22/runs")) return Response.json({ total_count: 0, workflow_runs: [] }); + if (path.endsWith("/runs/77")) return Response.json(run); + throw new Error("Unexpected admission request path"); + }; + }; + writeFileSync(preload, "(" + installFixture.toString() + ")(" + JSON.stringify(active) + ");\n"); + const output = join(root, "output"); + const result = spawnSync(process.execPath, ["--import", preload, fileURLToPath(new URL("./cloud-release-admission.mjs", import.meta.url)), ...(imageOnly ? ["image-only"] : [])], { + encoding: "utf8", timeout: 5_000, + env: { ...process.env, GITHUB_REPOSITORY: repositoryName, GITHUB_REF: "refs/heads/master", GITHUB_SHA: "a".repeat(40), + GITHUB_EVENT_NAME: "push", GITHUB_RUN_ATTEMPT: "1", GITHUB_TOKEN: "fixture", GITHUB_OUTPUT: output }, + }); + return { ...result, output: result.status === 0 ? readFileSync(output, "utf8") : "" }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +describe("admission CLI startup", () => { + for (const imageOnly of [false, true]) { + for (const active of [false, true]) { + test("executes " + (imageOnly ? "image-only" : "source") + " admission with active consumer=" + active, () => { + const result = runAdmissionCli({ active, imageOnly }); + assert.equal(result.status, 0, result.stderr || result.error?.message); + assert.equal(result.output, "admitted=" + active + "\n"); + }); + } + } + test("rejects a noncanonical CLI context before discovering consumers", () => { + const result = runAdmissionCli({ repositoryName: "fork/paperclip" }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Canonical master admission context required/); + assert.equal(result.output, ""); + }); +}); const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; const baseRun = { id: 456, workflow_id: workflow.id, path: workflow.path, run_attempt: 2, diff --git a/scripts/github-actions-reader.mjs b/scripts/github-actions-reader.mjs new file mode 100644 index 00000000000..1898164792a --- /dev/null +++ b/scripts/github-actions-reader.mjs @@ -0,0 +1,73 @@ +// Statuses that say "ask again", not "the answer is no". A release must not be +// blocked because GitHub returned a gateway error during a 60-minute poll. +const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); + +// A rate-limited read also says "ask again", and GitHub reports both primary +// and secondary rate limits as 403. Only the headers separate that from a +// token that may not read Actions, which must still fail at once. +function rateLimited(response) { + if (response.status !== 403) return false; + const header = (name) => response.headers?.get?.(name) ?? null; + return header("retry-after") !== null || header("x-ratelimit-remaining") === "0"; +} + +/** Milliseconds from a Retry-After header, when it carries a sane delay. */ +function retryAfterMs(response, capMs) { + const value = Number(response?.headers?.get?.("retry-after")); + if (!Number.isFinite(value) || value <= 0) return null; + return Math.min(value * 1_000, capMs); +} + +/** + * The GitHub Actions read used by the polling below, with transient transport + * failures retried: network errors, the statuses above, rate-limited 403s, and + * a body that fails while it is being read. Any other non-OK status throws on + * the first response, because waiting out a wrong token only delays the news. + * + * `deadlineAt` bounds retries by the caller's own polling deadline, so a read + * cannot extend the wait past the timeout it belongs to. + */ +export function createActionsReader({ + token, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + attempts = 4, backoffMs = 1_000, log = console.log, + now = Date.now, deadlineAt = () => Infinity, maxRetryAfterMs = 60_000, +} = {}) { + return async (path) => { + for (let attempt = 1; ; attempt += 1) { + // Retry only while both the attempt budget and the caller's deadline + // leave room for the wait this attempt would cost. + const waitMs = backoffMs * attempt; + const retryable = attempt < attempts && now() + waitMs < deadlineAt(); + const pause = (response) => sleep(retryAfterMs(response, maxRetryAfterMs) ?? waitMs); + let response; + try { + response = await fetchImpl(`https://api.github.com${path}`, { + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + signal: AbortSignal.timeout(30_000), redirect: "error", + }); + } catch (cause) { + if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); + log(`GitHub Actions read failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); + await pause(); + continue; + } + if (response.ok) { + try { + // A 200 whose body is truncated or undecodable is a transport + // failure like any other, so it belongs inside the retry. + return await response.json(); + } catch (cause) { + if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); + log(`GitHub Actions read body failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); + await pause(); + continue; + } + } + if (!retryable || !(TRANSIENT_READ_STATUSES.has(response.status) || rateLimited(response))) { + throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`); + } + log(`GitHub Actions read failed (HTTP ${response.status}); retrying (${attempt}/${attempts - 1}).`); + await pause(response); + } + }; +} From 4e29f001396c9fc76075850165ccfa3a6743e3ef Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Tue, 3 Nov 2026 16:30:58 -0800 Subject: [PATCH 06/28] fix(native): keep settled ACPX sessions across Runner upgrades and run scheduled resumes (#18879) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Native Runner keeps one ACPX provider conversation across the per-turn runs of a task. Each run cold-attaches the session that the previous run settled. > - The saved ACPX state binds the digest of the launch profile that wrote it. Every deploy ships a new provider pack, so that digest changes. > - After a deploy, the next run of such a task fails `run.attach` with `ACPX durable launch profile digest does not match runner startup`. Every same-run retry fails the same way. > - On one instance this happened seven times. Six tasks used their retry budget and ended blocked, or in progress with no runner. In the seventh, the scheduled same-run resume never ran at all: the task stayed `in_progress` with nothing running, and its monitor did not fire. > - This pull request re-binds a settled session to the upgraded launch profile, types the remaining mismatches so the server stops retrying them, and makes a scheduled remote resume survive the stale-lock sweep. > - The benefit is that a task continues its conversation after a deploy. When it cannot, it stops with a clear next action instead of stalling. ## Linked Issues or Issue Description Refs #18846, #18847, #18854 and #18860 (nearby native recovery changes). Refs #15795, an open change to the same stale-lock sweep function. It changes run teardown, not lock cleanup, so a second merge needs at most a small rebase. **What happened?** A per-turn native task used provider `acpx` with Claude, driver `acpx_runtime`, on a remote sandbox. 1. Run A succeeded and settled its provider session. The next run with the same build cold-attached it. 2. The instance deployed a new build. 3. A board approval woke run B in a fresh sandbox. Run B failed after about five minutes: `native_session_recovery_failed: Error: PRP command run.attach failed: {... "result":{"code":"command_execution_failed","message":"ACPX durable launch profile digest does not match runner startup"}}`. 4. The coordinator recorded `retryable_failure`, attempt 1, `native_session_interrupted`, recovery mode `exact_checkpoint_resume`, recovery state `resuming_session`, and a retry 30 seconds later. 5. Two seconds after the failure, the stale-lock sweep cleared the task's execution and checkout locks, because run B was `failed`. 6. Forty minutes later the attempt was still 1. The scheduled monitor had not fired. The task stayed `in_progress` with nothing running. **Expected behavior** A session that the previous run settled continues after a deploy. If it cannot continue, the run stops at once with a clear next action. A scheduled resume either runs or fails visibly. **Steps to reproduce** 1. Save a suspended ACPX session under one launch profile. 2. Change the sidecar bytes and the provider-pack authority at the same paths. 3. Cold-attach the next run in a new runner. Restore fails with the digest mismatch. 4. For the stall: leave a remote native run `failed` with a due `retryable_failure` coordinator, run the stale-lock sweep, then run restart recovery for that run. It returns `blocked: issue_execution_lock_changed` and writes nothing. The new Rust and PostgreSQL tests run these steps. **Paperclip version or commit** `master` at `b0c4e345267`. **Deployment mode** Native Runner, per-turn lifecycle, remote sandbox. The re-bind applies to every ACPX agent with an ordinary startup. ## What Changed ### Root causes - **Upgrade failure.** `AcpxLaunchProfile::canonical_digest` hashes the provider-pack authority, the sidecar command and arguments, each artifact path and SHA-256, and any candidate admission. For a remote sandbox the authority is the provider-pack manifest digest, which covers the Runner source revision and dist digest. So every deploy changes it. `AcpxDurableState` stores only the digest, and `validate()` rejects any difference. Only a Kimi candidate run binding had a carve-out. - **Silent stall.** The failure write keeps the task locks for `retryable_failure` and clears them only for `terminal_failure`. The stale-lock sweep treats any `failed` run as cleanable, so it cleared them. A remote retry goes through `claimNativeRestartRecoveries`. That claim requires `issues.execution_run_id` to name the run, so it returned `blocked: issue_execution_lock_changed` with no coordinator write and no deadline. The live recovery queue then dropped the entry. The local claim (`claimNativeSessionResumptions`) already reclaims a free lock, so only remote runs stalled. - **Monitor.** `noActiveNativeMonitorRun` holds a task's monitor while any native run for it has a coordinator that is not `committed` or `terminal_failure`. Run B's `retryable_failure` held it. That is by design; it ends when the resume runs. ### Runner (Rust) - `rebinds_settled_session` in `acpx_provider_backend.rs`. Restore re-binds a saved session to the current launch profile only when all of these are true: - The lifecycle is `suspended` and an identity exists. There is no active turn, pending runtime request, undelivered provider event, or unconfirmed provider exit. - The startup has no candidate admission. - The saved descriptor's sidecar command and arguments equal this startup's. - Any other digest mismatch keeps failing with the same message, plus `classification: "provider_state_incompatible"` in the durable command result. `DurableRunnerError` carries an optional closed classification for this. - A saved descriptor that names an agent release this build does not qualify (ACPX version, agent server package or version, or command digest) also gets the class. `validate_session()` and its message are unchanged; restore only adds the class. ### Runner (TypeScript) - `runnerRecoveryFailureClass` reads the classification from the authenticated `runnerCommandResult`. The Codex app-server driver (which also drives ACPX through runnerd) and `HarnessDriverBackend` pass it on as `failureClass`. - If replacement after a failed resume is not allowed, `executeNativeSession` throws `NativeProviderStateIncompatibleError` (message unchanged: `native_session_recovery_failed: ...`). If replacement is allowed, the existing path opens a fresh session in the same run, as before. ### Server - `nativeSessionFailureSourceCode` maps that error class to `native_provider_state_incompatible` with `instanceof`, not text. The disposition is permanent: `terminal_failure` at once, the task blocks, and the board owns the recovery action. Its next action says that a Retry continues in a fresh provider session with the same workspace. - `findNativeSessionResumeRun` also returns the barrier's `errorCode`. `rebindNativeSessionCheckpoint` does not resume a source with `native_provider_state_incompatible`. The existing fallback in `buildNativeExecutionWithCheckpoint` then starts a fresh normalized session with the full task bootstrap. - `sweepStaleIssueLocks` keeps both locks while a referenced run's coordinator owns a result-less `retryable_failure`, unless the task is done or cancelled. Its update also requires that no referenced run has a live status at the write, because a resume claim can return the run to running after the status read. - `claimNativeRestartRecoveries` reclaims a free execution lock for a `retryable_failure` resume, like the local claim does, when the checkout lock is free or held by this run and no other run for the task is queued, running or scheduled. It records `native.recovery.execution_reclaimed`. A lock held by another run still blocks. - Docs: `packages/paperclip-runner/docs/durable-recovery.md` and `doc/execution-semantics.md`. ### Safety argument for the re-bind What the binding protects, and what this change keeps: - **Candidate admission** carries company, model, platform and expiry authority. A candidate startup never re-binds. The Kimi run-binding carve-out and all candidate rejections are unchanged. - **The agent that loads the conversation** stays exact. `validate()` runs on the saved descriptor before the digest check and requires this build's qualified agent profile: ACPX version, agent server package and version, and `commandDigest`. The saved identity must still match that profile and the provider mode. - **In-flight provider work** is never re-bound. A turn, runtime request, undelivered event or unconfirmed exit keeps the strict binding, so no old provider process or request continues under a new build. The next launch only loads a settled conversation. - **Launch shape.** A changed sidecar command path or arguments still rejects. Only the content at the same paths may change. - **Artifacts** are not trusted from the saved state. The current profile comes from the trusted controller at startup, and session start verifies every artifact's bytes and the restored provider identity, as before. What changes: a new provider-pack authority, or new sidecar or Node bytes, at the same paths no longer invalidates a settled conversation. The saved digest cannot show which part changed, so the rule does not depend on it. I did not add state fields to tell authority changes from artifact changes, because the rule would treat both the same. One existing assertion changes. `released_kimi_cold_restore_rejects_candidate_injection_and_changed_full_profile` expected a settled ordinary Kimi session to reject a changed authority, artifact digest, or saved digest. Those three cases moved to a new positive test. Command, arguments, every candidate grant field, a missing profile and run binding still reject. ### How a task in the reported state recovers after this deploys The task has old-digest saved state, a `retryable_failure` coordinator at attempt 1, and cleared locks. 1. Startup recovery, and the live recovery queue, run restart recovery for the due run. It reclaims the free execution lock and claims the run. The resume runs now. 2. Run B's failed `run.attach` is a durable outcome of B's exact run authority. If the resume replays that outcome instead of executing it again, it fails with the old, untyped text. The remaining attempts then fail the same way, and the run ends `terminal_failure` with `native_session_retry_exhausted`. The task blocks with a board recovery action. This takes about a minute. 3. If the task blocks, press **Retry** once. Retry selects run B as the source (it has process identity and controller evidence, so the #18860 skip does not apply) and rebinds B's checkpoint, which holds A's provider session. The new run cold-attaches under the new Runner, and restore re-binds the old digest because the session is suspended and quiescent. The conversation continues. Run B's failed attach never wrote the state file, so the state is still A's settled state. 4. Tasks that already exhausted their retries (blocked, or in progress after `native_session_retry_exhausted`) need the same single Retry. Their earlier attempts ran under the old Runner. Wait for the task to show blocked before pressing Retry. A Retry pressed while the resume is running joins that run. ## Verification - Rust, failing first: `settled_session_rebinds_to_the_upgraded_runner_launch_profile` failed on `master` with the production message. `unsettled_or_relocated_session_keeps_the_strict_launch_binding` failed on the missing classification. Both pass now. - `cargo test -p paperclip-runner-core --no-fail-fast`: all 41 test binaries pass. The library has 471 passed and 1 ignored. - `cargo fmt --check`, and `pnpm --filter @paperclipai/paperclip-runner typecheck` (TypeScript, generated contracts, `cargo check`): pass. - Server, failing first: `native-scheduled-resume-lock.test.ts` (PostgreSQL). On `master` the sweep cleared the locks and restart recovery returned `issue_execution_lock_changed`. Both negative controls (another run's lock, a queued run) already passed. All four tests pass now. - `pnpm --filter @paperclipai/server typecheck`: pass. - Runner vitest (`src/native-session-runtime.test.ts`, `src/backends`, `src/drivers/codex`, `src/drivers/provider-failure.test.ts`, `src/live/runnerd-codex-transport*.test.ts`, `src/control-plane/acpx-controller-recovery.test.ts`): 31 files, 1,011 tests pass. New: the typed error when same-session recovery is required, and the recovery class reader and pass-through. - Server vitest: the native executor, status corpus and provider-pack admission suites (1,268 tests) pass. New: the classifier makes the typed error permanent, and the same text stays retryable. Checkpoint selection returns `errorCode`, and an incompatible source starts a fresh session with the same workspace (unit and PostgreSQL). - Server vitest over the recovery area: stale-lock sweep, native resumption, process recovery, failed-workspace recovery, restart recovery (integration, diskless, preparation), controller recovery, startup retry, startup cleanup, recovered-owner follow-up, session resume, runtime selection and recovery deadlines. 926 passed. Four `heartbeat-process-recovery` tests failed in that loaded run. Three pass when I run them alone. `redacts opaque environment-bound credentials from Sentry diagnostics: result` fails the same way on unmodified `master` on this host (a local environment value is redacted). - Review round 1: a new sweep test returns the run to running before the clear. It cleared the locks without the write recheck and keeps them with it. The strict Rust restore test covers an older agent server release and an older ACPX release. The ACPX backend and durable runner unit tests, the lock, sweep, resume projection and controller recovery suites, `cargo fmt --check` and the server typecheck pass. In the second full cargo run, the two `stdio_mcp` relay tests timed out waiting 5 s for a child exit on this loaded host. They do not use the changed code and passed in the first full run. - After the rebase onto the current `master`: the new lock test, failed-workspace recovery, stale-lock sweep and session resume suites pass (189 tests). Server typecheck, `pnpm check:module-boundaries` and `git diff --check` pass. - CI on `929ef4508d2`: in the first run, `isolated-repository-rehome` hit its 15 s test timeout, `idle-completed-history` could not reach its fixture runtime service, and e2e shard 3 was cancelled by a runner shutdown. A rerun of the failed jobs passed. All 55 checks pass. - Not verified: a live remote sandbox deploy, and whether run B's same-run resume replays its recorded `run.attach` failure or executes it again. The Retry path is tested at source selection and checkpoint rebind (server) and at cold attach (Runner). It is not tested end to end through a sandbox. ## Risks - A settled session now continues under new sidecar bytes. If a sidecar change made an old ACPX record unreadable, session start fails as any failed resume does. It does not corrupt state, and the run follows its normal recovery policy. - The sweep now keeps locks for a failed native run with a scheduled resume. If that resume can never be claimed (for example the agent stays paused), the task keeps the lock until the coordinator settles or the task closes. Before, the sweep cleared it, and the resume still never ran. - If another run holds the task's lock, a scheduled remote resume still stays blocked and the coordinator stays `retryable_failure`. This change does not settle that case. It needs a separate decision about superseding the resume. - `native_provider_state_incompatible` is a new failure code. No UI or migration change is needed. Older servers never see it. - No schema migration. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`) through Claude Code, with tool use (shell, file edits, test runs). The runtime does not expose the context window or reasoning configuration. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/execution-semantics.md | 7 + .../paperclip-runner/docs/durable-recovery.md | 29 ++ .../runner-core/src/acpx_provider_backend.rs | 351 ++++++++++++++++-- .../crates/runner-core/src/durable/mod.rs | 20 +- .../crates/runner-core/src/durable/runner.rs | 54 +++ .../backends/harness-driver-backend.test.ts | 31 ++ .../src/backends/harness-driver-backend.ts | 6 +- .../src/contracts/harness-driver.ts | 19 + .../src/contracts/native-session-backend.ts | 13 + .../codex/codex-app-server-driver-impl.ts | 9 +- .../src/native-session-runtime.test.ts | 36 ++ .../src/native-session-runtime.ts | 6 +- .../native-runtime/native-restart-recovery.ts | 34 +- .../native-scheduled-resume-lock.test.ts | 137 +++++++ .../native-session-executor.test.ts | 16 + .../native-runtime/native-session-executor.ts | 13 +- .../native-session-resume-projection.test.ts | 1 + .../native-session-resume.test.ts | 58 +++ .../native-runtime/native-session-resume.ts | 9 + server/src/services/recovery/service.ts | 31 ++ server/src/vendor/paperclip-runner/index.ts | 1 + 21 files changed, 850 insertions(+), 31 deletions(-) create mode 100644 server/src/services/native-runtime/native-scheduled-resume-lock.test.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index bf481923610..f18aa7128a7 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1258,6 +1258,13 @@ checks coordinator ownership in its terminal update, so a result recorded after the process check is also protected. Terminal task status remains authoritative; exhausted finalization retries do not keep an otherwise orphaned run alive. +A native failure that schedules a same-run resume marks the run `failed` but +keeps the task's execution and checkout locks for that resume. The sweep leaves +those locks in place and clears them only when the task is done or cancelled. +If a scheduled resume finds its execution lock free, restart recovery reclaims +it, as the local resume claim does, unless another run for the task is queued, +running, or scheduled. A lock held by another run still blocks the resume. + Automatic productivity reviews are retired. Run counts, missing comments, and elapsed task time do not create review tasks or impose continuation holds. Bounded continuation, provider recovery, budget limits, explicit blockers, and normal review/approval stages remain in force. Existing productivity-review tasks, comments, assignments, and dependencies remain unchanged and readable; their historical origins still identify them as recovery work for recursion suppression. ### Issue-thread interaction resolution diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 3205943c454..d1e0be1a1af 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -70,6 +70,35 @@ a scripted harness, removes the controller's launch files, then recovers the same process and provider session and completes another turn. This exercises the real transport; it does not qualify a hosted OpenCode or ACPX deployment. +## Settled ACPX sessions across a Runner upgrade + +Saved ACPX provider state records the digest of the launch profile that wrote +it: the provider-pack authority, the sidecar command and arguments, each launch +artifact's SHA-256, and any candidate admission. A deploy that ships a new +provider pack or sidecar build changes the digest. The state stores only the +digest, so restore cannot tell which part changed. + +Restore re-binds a saved session to the current launch profile, and keeps the +conversation, only when all of these are true: + +- The session is suspended. It has no active turn, pending runtime request, + undelivered provider event, or unconfirmed provider exit. +- The startup has no candidate admission. +- The saved descriptor names this startup's sidecar command and arguments. + +Every other binding stays exact. The saved descriptor must match this build's +qualified agent profile (ACPX version, agent server package and version, and +command digest). Session start verifies each artifact and the restored provider +identity. Candidate grants and run bindings keep their own checks. + +Any other launch-profile mismatch, and a saved descriptor that names a release +this build does not qualify, fails restore with the closed command-result +classification `provider_state_incompatible`. The controller reads the classification from the +authenticated result, not from the message. If the run allows a replacement +after a failed resume, it opens a fresh provider session as before. Otherwise the +run fails with `native_provider_state_incompatible` and is not retried. A board +Retry then continues the task in a fresh provider session in the same workspace. + ## Profile identity Every ACPX provider's `commandDigest` attests only its own upstream release. diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 1e445b261d1..bcc1625dc88 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -26,7 +26,7 @@ use crate::durable::QualifiedLaunchArtifact; use crate::durable::{ create_private_temporary_file, open_private_regular_file, verify_private_directory, AcpxLaunchProfile, Command, CommandExecution, CommandExecutor, DurableRunnerConfig, - DurableRunnerError, EventPriority, PolledEvent, + DurableRunnerError, EventPriority, PolledEvent, PROVIDER_STATE_INCOMPATIBLE, }; use crate::generated_acpx_sidecar_contract::GeneratedAcpxSidecarCommand; use crate::process_supervisor::{VerifiedProcessArgument, VerifiedProcessLaunch}; @@ -279,6 +279,19 @@ fn only_registered_asset_text_changed( } impl AcpxProviderDescriptor { + // Whether this build qualifies the release that the descriptor names. + fn names_qualified_release(&self) -> bool { + acpx_release_profile(&self.agent).is_some_and(|expected| { + self.provider_version == QUALIFIED_ACPX_VERSION + && self.acpx_version == QUALIFIED_ACPX_VERSION + && self.agent_server_package == expected.agent_server_package + && self.agent_server_version == expected.agent_server_version + && self.agent_runtime_package.as_deref() == expected.agent_runtime_package + && self.agent_runtime_version.as_deref() == expected.agent_runtime_version + && self.command_digest == expected.command_digest + }) + } + fn validate_session( &self, context: &AcpxEventProjectionContext, @@ -622,11 +635,21 @@ impl AcpxDurableState { // so its persisted descriptor legitimately carries the prior run ID // until run.attach rotates authority. Fresh command descriptors still // use validate(), which binds them to the current run. - self.descriptor.validate_session(context)?; + // A deploy that changes the agent's qualified release also leaves + // saved state that this build cannot restore. Keep the same check and + // only name its class. + self.descriptor.validate_session(context).map_err(|error| { + if self.descriptor.names_qualified_release() { + error + } else { + error.with_classification(PROVIDER_STATE_INCOMPATIBLE) + } + })?; if self.launch_profile_digest != expected_launch_profile_digest { return Err(DurableRunnerError::invalid( "ACPX durable launch profile digest does not match runner startup", - )); + ) + .with_classification(PROVIDER_STATE_INCOMPATIBLE)); } validate_pending_runtime_requests(&self.pending_runtime_requests)?; let mut ids = HashSet::new(); @@ -960,6 +983,34 @@ impl AcpxCommandExecutor { prior.canonical_digest() } + // A deploy changes the provider-pack authority and the sidecar and Node + // artifact digests, so a saved digest from the previous build never + // matches. The digest is a hash, so restore cannot tell which part + // changed. Re-bind only when no provider work is in flight: + // - The session is suspended. No turn, request, undelivered event, or + // unconfirmed provider exit remains. Nothing from the old provider + // process continues; the next launch only loads the conversation. + // - The startup is ordinary. Candidate admission carries company, model + // and expiry authority in the digest, so candidate sessions stay + // exactly bound. + // - The saved descriptor names this startup's sidecar command and + // arguments. Only the qualified content at those paths changed. + // validate() still binds the descriptor to this build's qualified agent + // profile (ACPX, agent server and command digest), and session start + // still verifies every artifact and the restored provider identity. + fn rebinds_settled_session(&self, state: &AcpxDurableState) -> bool { + self.launch_profile.as_ref().is_some_and(|profile| { + profile.candidate_admission.is_none() + && state.descriptor.sidecar_command == profile.command + && state.descriptor.sidecar_args == profile.args + }) && state.lifecycle == "suspended" + && state.identity.is_some() + && state.active_turn_id.is_none() + && !state.provider_exit_unconfirmed + && state.pending_events.is_empty() + && state.pending_runtime_requests.is_empty() + } + fn restore(&mut self) -> Result<(), DurableRunnerError> { self.ensure_persistence_healthy()?; if self.restore_checked { @@ -1016,10 +1067,15 @@ impl AcpxCommandExecutor { file.read_to_end(&mut bytes).map_err(|error| { DurableRunnerError::invalid(format!("failed to read ACPX provider state: {error}")) })?; - let state: AcpxDurableState = serde_json::from_slice(&bytes).map_err(|error| { + let mut state: AcpxDurableState = serde_json::from_slice(&bytes).map_err(|error| { DurableRunnerError::invalid(format!("ACPX provider state is malformed: {error}")) })?; let launch_profile_digest = self.restore_launch_profile_digest(&state)?; + if state.launch_profile_digest != launch_profile_digest + && self.rebinds_settled_session(&state) + { + state.launch_profile_digest = launch_profile_digest.clone(); + } state.validate(&self.context, &launch_profile_digest)?; self.state = Some(state); self.restore_session_if_needed() @@ -5298,14 +5354,11 @@ mod tests { "grant_run", "injected_candidate", "context_run", - "authority", "command", "args", - "artifact", "missing_profile", - "saved_digest", ] { - let (directory, mut config, payload, before, sentinel, candidate) = + let (directory, mut config, payload, _, sentinel, candidate) = kimi_cold_restore_fixture("kimi-released-cold-reject"); assert!(config .acpx_launch_profile @@ -5316,26 +5369,11 @@ mod tests { match field { "context_run" => config.run_id = "foreign-run".into(), "missing_profile" => config.acpx_launch_profile = None, - "saved_digest" => { - let mut state: AcpxDurableState = serde_json::from_slice(&before).unwrap(); - state.launch_profile_digest = format!("sha256:{}", "0".repeat(64)); - fs::write( - directory.join("acpx-provider-state.json"), - serde_json::to_vec(&state).unwrap(), - ) - .unwrap(); - } _ => { let profile = config.acpx_launch_profile.as_mut().unwrap(); match field { - "authority" => { - profile.authority_digest = format!("sha256:{}", "0".repeat(64)) - } "command" => profile.command = directory.join("foreign-command"), "args" => profile.args.push("foreign-argument".into()), - "artifact" => { - profile.artifacts[0].sha256 = format!("sha256:{}", "0".repeat(64)) - } _ => { let mut grant = candidate; match field { @@ -5382,6 +5420,60 @@ mod tests { } } + #[test] + #[cfg(any( + all(target_os = "macos", target_arch = "aarch64"), + all(target_os = "linux", target_arch = "x86_64") + ))] + fn released_kimi_cold_restore_rebinds_a_settled_session_to_new_launch_content() { + // An ordinary startup re-binds a settled session after a provider-pack + // or artifact change at the same launch paths. The checks above for + // candidate grants, command, arguments and run binding still apply. + for field in ["authority", "artifact", "saved_digest"] { + let (directory, mut config, payload, before, sentinel, _) = + kimi_cold_restore_fixture("kimi-released-cold-rebind"); + let profile = config.acpx_launch_profile.as_mut().unwrap(); + match field { + "authority" => profile.authority_digest = format!("sha256:{}", "0".repeat(64)), + "artifact" => { + let command = profile.command.clone(); + write_artifact(&command, b"#!/bin/sh\nexit 2\n", true); + profile.artifacts[0] = artifact(&command); + } + "saved_digest" => { + let mut state: AcpxDurableState = serde_json::from_slice(&before).unwrap(); + state.launch_profile_digest = format!("sha256:{}", "0".repeat(64)); + fs::write( + directory.join(ACPX_PROVIDER_STATE_FILE), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + } + _ => unreachable!(), + } + config.validate().unwrap(); + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + recovered.restore().expect(field); + recovered.attach_run(&payload).expect(field); + let saved: AcpxDurableState = + serde_json::from_slice(&fs::read(recovered.state_path()).unwrap()).unwrap(); + assert_eq!(saved.descriptor.run_id, "run-2", "{field}"); + assert_eq!( + saved.launch_profile_digest, + config + .acpx_launch_profile + .as_ref() + .unwrap() + .canonical_digest() + .unwrap(), + "{field}" + ); + assert!(recovered.session.is_none(), "{field}"); + assert!(!sentinel.exists(), "{field}"); + fs::remove_dir_all(directory).unwrap(); + } + } + #[test] #[cfg(any( all(target_os = "macos", target_arch = "aarch64"), @@ -5498,4 +5590,217 @@ mod tests { assert!(!sentinel.exists()); fs::remove_dir_all(directory).unwrap(); } + + // A settled Claude session saved by one runner build. The next deploy + // ships a new sidecar build and provider-pack authority at the same + // qualified launch paths, and the next run cold-attaches in a new runner. + #[cfg(unix)] + fn settled_session_after_runner_upgrade( + label: &str, + ) -> (PathBuf, DurableRunnerConfig, Value, Vec) { + let directory = temporary_directory(label); + let command = directory.join("node"); + let sidecar = directory.join("acpx-runtime-sidecar.cjs"); + write_artifact(&command, b"#!/bin/sh\nexit 1\n", true); + write_artifact(&sidecar, b"// sidecar build 1\n", false); + let profile = AcpxLaunchProfile { + candidate_admission: None, + authority_digest: format!("sha256:{}", "d".repeat(64)), + command: command.clone(), + args: vec![sidecar.to_string_lossy().into_owned()], + artifacts: vec![artifact(&command), artifact(&sidecar)], + }; + let runtime = directory.join("runtime"); + let workspace = directory.join("workspace"); + fs::create_dir_all(&runtime).unwrap(); + fs::create_dir_all(&workspace).unwrap(); + fs::set_permissions(&runtime, fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(&workspace, fs::Permissions::from_mode(0o700)).unwrap(); + let mut value = descriptor("claude"); + value["runtimeDirectory"] = json!(runtime); + value["cwd"] = json!(workspace); + value["sidecarCommand"] = json!(command); + value["sidecarArgs"] = json!([sidecar]); + let desc: AcpxProviderDescriptor = serde_json::from_value(value.clone()).unwrap(); + let identity = AcpxProviderSessionIdentity { + kind: "acpx".to_owned(), + normalized_session_id: "session-1".to_owned(), + acpx_record_id: "record-1".to_owned(), + backend_session_id: "backend-1".to_owned(), + agent_session_id: "agent-1".to_owned(), + profile_digest: desc.command_digest.clone(), + workspace_digest: format!("sha256:{}", "a".repeat(64)), + requested_model: desc.model.clone(), + effective_model: desc.model.clone(), + permission_mode: Some(desc.permission_mode), + mode: None, + pi_thinking_level: None, + claude_effort: None, + grok_effort: None, + grok_disable_web_search: None, + provider_lifetime_fence_candidates: [60_001, 60_002, 60_003], + }; + let operations = Vec::new(); + let tools = AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }; + let mut state = AcpxDurableState::new(desc, tools, profile.canonical_digest().unwrap()); + state.lifecycle = "suspended".to_owned(); + state.identity = Some(identity); + let config = test_config(&directory, Some(profile)); + let mut executor = AcpxCommandExecutor::with_runner_config(&directory, &config); + executor.state = Some(state); + executor.save_state().unwrap(); + let saved = fs::read(executor.state_path()).unwrap(); + + write_artifact(&sidecar, b"// sidecar build 2\n", false); + let mut next_config = config; + next_config.run_id = "run-2".to_owned(); + next_config.turn_id = "turn-2".to_owned(); + next_config.item_id = "item-2".to_owned(); + let next_profile = next_config.acpx_launch_profile.as_mut().unwrap(); + next_profile.authority_digest = format!("sha256:{}", "e".repeat(64)); + next_profile.artifacts = vec![artifact(&command), artifact(&sidecar)]; + next_config.validate().unwrap(); + let mut provider = value; + provider["runId"] = json!("run-2"); + ( + directory, + next_config, + json!({ "provider": provider }), + saved, + ) + } + + #[cfg(unix)] + #[test] + fn settled_session_rebinds_to_the_upgraded_runner_launch_profile() { + let (directory, config, payload, saved) = + settled_session_after_runner_upgrade("upgrade-rebind"); + let prior: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); + let current_digest = config + .acpx_launch_profile + .as_ref() + .unwrap() + .canonical_digest() + .unwrap(); + assert_ne!(prior.launch_profile_digest, current_digest); + + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + recovered + .restore() + .expect("a settled session survives a new qualified sidecar build"); + assert!(recovered.session.is_none()); + assert!(recovered.poll_events().unwrap().is_empty()); + assert_eq!(fs::read(recovered.state_path()).unwrap(), saved); + recovered + .attach_run(&payload) + .expect("the next run cold-attaches the same conversation"); + assert!(recovered.session.is_none()); + + let attached: AcpxDurableState = + serde_json::from_slice(&fs::read(recovered.state_path()).unwrap()).unwrap(); + assert_eq!(attached.launch_profile_digest, current_digest); + assert_eq!(attached.descriptor.run_id, "run-2"); + assert_eq!(attached.lifecycle, "suspended"); + assert_eq!(attached.identity, prior.identity); + assert_eq!( + attached.descriptor.command_digest, + prior.descriptor.command_digest + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[cfg(unix)] + #[test] + fn unrestorable_saved_sessions_keep_strict_bindings_and_name_their_class() { + for field in [ + "active_turn", + "pending_request", + "pending_event", + "session_open", + "exit_unconfirmed", + "no_identity", + "command_moved", + "args_changed", + "agent_server_release", + "acpx_release", + ] { + let (directory, mut config, _, saved) = + settled_session_after_runner_upgrade("upgrade-strict"); + let state_path = directory.join(ACPX_PROVIDER_STATE_FILE); + let mut state: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); + let profile = config.acpx_launch_profile.as_mut().unwrap(); + match field { + "active_turn" => { + state.lifecycle = "turn_active".into(); + state.active_turn_id = Some("turn-1".into()); + } + "pending_request" => { + state.lifecycle = "turn_active".into(); + state.active_turn_id = Some("turn-1".into()); + state + .push(NormalizedProviderEvent { + event_type: "runtime_request.created".into(), + priority: EventPriority::P0, + payload: json!({"request":pending_input_request("input-1")}), + }) + .unwrap(); + state.pending_events.clear(); + } + "pending_event" => state + .push(NormalizedProviderEvent { + event_type: "session.updated".into(), + priority: EventPriority::P1, + payload: json!({}), + }) + .unwrap(), + "session_open" => state.lifecycle = "session_open".into(), + "exit_unconfirmed" => { + state.lifecycle = "closed".into(); + state.provider_exit_unconfirmed = true; + } + "no_identity" => { + state.lifecycle = "prepared".into(); + state.identity = None; + } + "command_moved" => { + let moved = directory.join("relocated-node"); + write_artifact(&moved, b"#!/bin/sh\nexit 1\n", true); + profile.command = moved.clone(); + profile.artifacts[0] = artifact(&moved); + } + "args_changed" => profile.args.push("--changed".into()), + "agent_server_release" => state.descriptor.agent_server_version = "0.0.1".into(), + "acpx_release" => { + state.descriptor.acpx_version = "0.0.1".into(); + state.descriptor.provider_version = "0.0.1".into(); + } + _ => unreachable!(), + } + config.validate().unwrap(); + fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); + let retained = fs::read(&state_path).unwrap(); + + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + let error = recovered.restore().unwrap_err(); + let expected = if field.ends_with("_release") { + "does not match a qualified immutable profile" + } else { + "launch profile digest does not match runner startup" + }; + assert!(error.to_string().contains(expected), "{field}: {error}"); + assert_eq!( + error.classification(), + Some(crate::durable::PROVIDER_STATE_INCOMPATIBLE), + "{field}" + ); + assert!(recovered.session.is_none(), "{field}"); + assert_eq!(fs::read(&state_path).unwrap(), retained, "{field}"); + fs::remove_dir_all(directory).unwrap(); + } + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs index fc33db5cc3c..4b026797f66 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs @@ -37,12 +37,17 @@ pub fn redact_diagnostic_text(input: &str) -> String { state::redact_text(input) } +/// Saved provider state was written under a launch or provider profile that +/// this runner cannot adopt. The outcome is deterministic for this runner +/// build: retrying the same restore cannot succeed. +pub const PROVIDER_STATE_INCOMPATIBLE: &str = "provider_state_incompatible"; + #[derive(Clone, Debug, PartialEq, Eq)] -pub struct DurableRunnerError(String, Option); +pub struct DurableRunnerError(String, Option, Option<&'static str>); impl DurableRunnerError { pub fn invalid(message: impl Into) -> Self { - Self(message.into(), None) + Self(message.into(), None, None) } pub fn with_provider_failure(mut self, value: Option) -> Self { @@ -53,6 +58,17 @@ impl DurableRunnerError { pub fn provider_failure(&self) -> Option { self.1.clone() } + + /// Adds a closed failure class that the controller reads from the command + /// result instead of parsing the message. + pub fn with_classification(mut self, classification: &'static str) -> Self { + self.2 = Some(classification); + self + } + + pub fn classification(&self) -> Option<&'static str> { + self.2 + } } impl Display for DurableRunnerError { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index ddfd4034c94..97bf9644745 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -1930,6 +1930,9 @@ fn process_command( "code": "command_execution_failed", "message": message, }); + if let Some(classification) = error.classification() { + failure["classification"] = json!(classification); + } if let Some(provider_failure) = error.provider_failure() { failure["providerFailure"] = provider_failure; } @@ -2489,6 +2492,17 @@ mod tests { } } + struct IncompatibleStateExecutor; + + impl CommandExecutor for IncompatibleStateExecutor { + fn execute(&mut self, _command: &Command) -> Result { + Err( + DurableRunnerError::invalid("saved provider state cannot be adopted") + .with_classification(crate::durable::PROVIDER_STATE_INCOMPATIBLE), + ) + } + } + impl CommandExecutor for ShutdownFailingExecutor { fn execute(&mut self, _command: &Command) -> Result { Ok(CommandExecution::result(json!({"status": "completed"}))) @@ -4865,6 +4879,46 @@ mod tests { .diagnostics .iter() .all(|diagnostic| !diagnostic.contains("test-secret"))); + assert!(failed.result.get("classification").is_none()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn classified_executor_failure_keeps_its_closed_class_in_the_durable_result() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-command-classified-failure-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let command = command("run.attach"); + let (failed, _) = process_command( + &mut state, + &store, + &config, + &mut IncompatibleStateExecutor, + &command, + ) + .unwrap(); + let (mut recovered, _) = store.load_or_create(&config).unwrap(); + let replay = process_command( + &mut recovered, + &store, + &config, + &mut IncompatibleStateExecutor, + &command, + ) + .unwrap() + .0; + assert_eq!(failed.status, "failed"); + assert_eq!(failed.result["code"], "command_execution_failed"); + assert_eq!( + failed.result["classification"], + "provider_state_incompatible" + ); + assert_eq!(failed, replay); fs::remove_dir_all(directory).unwrap(); } diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 094362e0fa6..6791372425a 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; +import { PROVIDER_STATE_INCOMPATIBLE, runnerRecoveryFailureClass } from "../contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; @@ -323,6 +324,36 @@ describe("HarnessDriverBackend", () => { expect(recoveredDispositionAllowance).toBe(true); }); + it("keeps the runner's closed class of a failed recovery", async () => { + const commandFailure = Object.defineProperty(new Error("PRP command run.attach failed"), "runnerCommandResult", { + value: { status: "failed", commandType: "run.attach", + result: { code: "command_execution_failed", message: "saved launch profile changed", + classification: PROVIDER_STATE_INCOMPATIBLE } }, + enumerable: false, + }); + expect(runnerRecoveryFailureClass(commandFailure)).toBe(PROVIDER_STATE_INCOMPATIBLE); + expect(runnerRecoveryFailureClass(new Error("wrapped", { cause: commandFailure }))).toBe(PROVIDER_STATE_INCOMPATIBLE); + expect(runnerRecoveryFailureClass(new Error(`text only: ${PROVIDER_STATE_INCOMPATIBLE}`))).toBeUndefined(); + const unclassified = Object.defineProperty(new Error("PRP command run.attach failed"), "runnerCommandResult", { + value: { status: "failed", result: { code: "command_execution_failed", classification: "other" } }, + enumerable: false, + }); + expect(runnerRecoveryFailureClass(unclassified)).toBeUndefined(); + + const backend = new HarnessDriverBackend({ + ...driver, + recoverSession: async () => ({ recovered: false, reason: String(commandFailure), + failureClass: runnerRecoveryFailureClass(commandFailure) }), + }); + await expect(backend.recoverSession({ + backendKind: "runner", driverKind: "fake", sessionId: "driver-1", providerSessionId: "provider-1", + providerIdentity, + identity: { runId: "run-2", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }, { signal: new AbortController().signal })).resolves.toEqual({ + recovered: false, reason: "Error: PRP command run.attach failed", failureClass: PROVIDER_STATE_INCOMPATIBLE, + }); + }); + it("preserves a consumed disposition allowance in native snapshots", async () => { class ConsumedRecoverySession extends FakeHarnessSession { override async snapshot(): Promise { diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 1339830d047..56caad7ecab 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -162,7 +162,11 @@ export class HarnessDriverBackend implements NativeSessionBackend { recoveryOptions, ); if (!recovered.recovered || recovered.session === undefined) { - return { recovered: false, reason: recovered.reason }; + return { + recovered: false, + reason: recovered.reason, + ...(recovered.failureClass ? { failureClass: recovered.failureClass } : {}), + }; } try { assertProviderSessionIdentity( diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index ede129c07b7..adccad8d49b 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -496,10 +496,29 @@ export interface PersistedHarnessSession { | "allow_replacement_after_resume_failure"; } +/** Runner proof that saved provider state cannot be restored by this runner + * build. Restoring the same state again fails the same way. */ +export const PROVIDER_STATE_INCOMPATIBLE = "provider_state_incompatible"; + export interface HarnessSessionRecoveryResult { recovered: boolean; session?: HarnessSession; reason?: string; + /** Closed runner class of a deterministic recovery failure. */ + failureClass?: typeof PROVIDER_STATE_INCOMPATIBLE; +} + +/** Reads the closed class of an authenticated runner command failure. */ +export function runnerRecoveryFailureClass( + error: unknown, + depth = 0, +): typeof PROVIDER_STATE_INCOMPATIBLE | undefined { + if (!error || typeof error !== "object" || depth > 5) return undefined; + const value = error as Record; + const result = value.runnerCommandResult as Record | undefined; + const detail = result?.result as Record | undefined; + if (detail?.classification === PROVIDER_STATE_INCOMPATIBLE) return PROVIDER_STATE_INCOMPATIBLE; + return runnerRecoveryFailureClass(value.cause, depth + 1); } export interface HarnessSession { diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 9761858e790..07b829c0fd5 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -15,6 +15,7 @@ import type { HarnessRuntimeRequestHandoff, HarnessRuntimeRequestResolution, HarnessGoalOperation, + HarnessSessionRecoveryResult, HarnessThreadGoal, HarnessThreadLineageEntry, NativeRuntimeContextCapabilities, @@ -82,6 +83,7 @@ export interface NativeSessionRecoveryResult { recovered: boolean; session?: NativeSession; reason?: string; + failureClass?: HarnessSessionRecoveryResult["failureClass"]; } /** Runner-owned cause: the process was restored, but its active turn was lost. */ @@ -278,6 +280,17 @@ export interface NativeSessionBackend { ): Promise; } +/** Same-session recovery is forbidden here, and the runner proved that the + * saved provider session cannot be restored by its build. Retrying the same + * run cannot succeed. */ +export class NativeProviderStateIncompatibleError extends Error { + readonly code = "native_provider_state_incompatible"; + constructor(reason: string) { + super(`native_session_recovery_failed: ${reason}`); + this.name = "NativeProviderStateIncompatibleError"; + } +} + /** A provider failed terminal is not a missing completion proposal. */ export class NativeProviderTerminalFailure extends Error { readonly code = "native_provider_terminal_failed"; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 995424acc63..c3ea63d6b03 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -20,7 +20,7 @@ import type { PersistedHarnessTurnTerminal, } from "../../contracts/harness-driver.js"; import { NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; -import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; +import { HarnessReconciliationError, runnerRecoveryFailureClass } from "../../contracts/harness-driver.js"; import { CODEX_CODEX_PROTOCOL_VERSION, CODEX_SKILLLESS_BASE_INSTRUCTIONS, @@ -677,7 +677,12 @@ export class CodexAppServerDriver implements HarnessDriver { await cancellation.close().catch(() => {}); if (error instanceof NativeSessionProtocolIntegrityError) throw error; if (options.signal.aborted) options.signal.throwIfAborted(); - return { recovered: false, reason: redactCodexDiagnostic(String(error)) }; + const failureClass = runnerRecoveryFailureClass(error); + return { + recovered: false, + reason: redactCodexDiagnostic(String(error)), + ...(failureClass ? { failureClass } : {}), + }; } finally { cancellation.detach(); } diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 11f665119dd..784f95ed6f4 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -15,10 +15,12 @@ import type { PersistedNativeSession, } from "./contracts/native-session-backend.js"; import { + NativeProviderStateIncompatibleError, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, NativeSessionProtocolIntegrityError, } from "./contracts/native-session-backend.js"; +import { PROVIDER_STATE_INCOMPATIBLE } from "./contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, @@ -5641,6 +5643,40 @@ describe("executeNativeSession recovery", () => { expect(openSession).not.toHaveBeenCalled(); }); + it("types a runner-proven incompatible session when same-session recovery is required", async () => { + for (const providerRecoveryPolicy of ["same_session_only", "allow_replacement_after_governed_wait"] as const) { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", sessionId: "driver-incompatible", identity, + providerSessionId: "provider-incompatible", providerRecoveryPolicy, + cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], + }; + const openRun = vi.fn(async () => undefined); + const openSession = vi.fn(); + const openReplacementSession = vi.fn(); + const backend: NativeSessionBackend = { + descriptor: async () => ({ kind: "mock", name: "incompatible", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true } }), + openSession, openReplacementSession, + recoverSession: async () => ({ recovered: false, reason: "saved launch profile changed", + failureClass: PROVIDER_STATE_INCOMPATIBLE }), + }; + const port: ControlPlanePort = { + openRun, appendEvent: vi.fn(), completeRun: vi.fn(), + loadSessionCheckpoint: async () => structuredClone(checkpoint), + replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), + }; + const failure = await executeNativeSession({ + input, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + }).catch((error: unknown) => error); + expect(failure, providerRecoveryPolicy).toBeInstanceOf(NativeProviderStateIncompatibleError); + expect((failure as Error).message).toBe("native_session_recovery_failed: saved launch profile changed"); + expect(openRun).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + expect(openReplacementSession).not.toHaveBeenCalled(); + } + }); + it("bounds paginated recovery replay with one signal and observes a late rejection", async () => { vi.useFakeTimers(); let rejectStalledReplay = (_error: Error) => {}; diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index fbbb6a98133..a5f55fdf47a 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -19,8 +19,9 @@ import type { NativeSessionBackend, } from "./contracts/native-session-backend.js"; import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; -import type { HarnessThreadGoal } from "./contracts/harness-driver.js"; +import { PROVIDER_STATE_INCOMPATIBLE, type HarnessThreadGoal } from "./contracts/harness-driver.js"; import { + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, @@ -2164,6 +2165,9 @@ export async function executeNativeSession( }; if (!recovery.recovered || !recovery.session) { if (!replacementAllowed) { + if (recovery.failureClass === PROVIDER_STATE_INCOMPATIBLE) { + throw new NativeProviderStateIncompatibleError(recovery.reason ?? "unknown"); + } throw new Error( `native_session_recovery_failed: ${recovery.reason ?? "unknown"}`, ); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 6f0cdaf91d8..24afea8b80f 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -40,6 +40,7 @@ import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; import { nativeRestartInterruptedTurnId, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { retireSupersededNativeRetry } from "./native-superseded-retry.js"; import { legacyWorkspaceRecoveryNotMigrated } from "../legacy-workspace-recovery-migration.js"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { requestNativeStartupCleanup, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-startup-cleanup.js"; export type NativeControllerIdentity = { @@ -583,6 +584,7 @@ export async function claimNativeRestartRecoveries(input: { run: heartbeatRuns, coordinator: nativeRunFinalizations, issueExecutionRunId: issues.executionRunId, + issueCheckoutRunId: issues.checkoutRunId, issueAssigneeAgentId: issues.assigneeAgentId, issueStatus: issues.status, }) @@ -630,7 +632,20 @@ export async function claimNativeRestartRecoveries(input: { if (!input.liveRunExecutions?.has(row.run.id) && await retireSupersededNativeRetry(tx as unknown as Db, { run: row.run, coordinator: row.coordinator, issueExecutionRunId: row.issueExecutionRunId, now, })) return { kind: "blocked", runId: row.run.id, reason: "native_retry_superseded" } as const; - if (row.issueExecutionRunId !== row.run.id && !(stopCleanup && row.issueExecutionRunId === null)) { + // A scheduled same-run resume can find its task lock released while no + // other run holds the task. Like the local resume claim, reclaim the + // free lock unless another run is admitted for this task. + const reclaimsExecution = row.issueExecutionRunId === null + && row.coordinator.phase === "retryable_failure" + && (row.issueCheckoutRunId === null || row.issueCheckoutRunId === row.run.id) + && !(await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, row.run.companyId), ne(heartbeatRuns.id, row.run.id), + inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + or(eq(heartbeatRuns.issueId, row.coordinator.issueId), eq(heartbeatRuns.nativeIssueId, row.coordinator.issueId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${row.coordinator.issueId}`), + )).limit(1)).length; + if (row.issueExecutionRunId !== row.run.id && !(stopCleanup && row.issueExecutionRunId === null) + && !reclaimsExecution) { return { kind: "blocked", runId: row.run.id, @@ -1165,6 +1180,23 @@ export async function claimNativeRestartRecoveries(input: { updatedAt: now, }) .where(eq(heartbeatRuns.id, row.run.id)); + if (reclaimsExecution) { + const [agent] = await tx.select({ name: agents.name }).from(agents) + .where(eq(agents.id, row.run.agentId)).limit(1); + await tx.update(issues).set({ + executionRunId: row.run.id, + executionAgentNameKey: agent?.name.trim().toLowerCase() || null, + executionLockedAt: now, + updatedAt: now, + }).where(and(eq(issues.id, row.coordinator.issueId), eq(issues.companyId, row.run.companyId), + isNull(issues.executionRunId))); + await appendHeartbeatRunEvent(tx as unknown as Db, { + companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId, + eventType: "native.recovery.execution_reclaimed", stream: "system", level: "info", + message: "Reclaimed the original task execution for native session recovery.", + payload: { issueId: row.coordinator.issueId }, + }); + } claimScope = { companyId: row.run.companyId, issueId: row.coordinator.issueId, controller, remote: Boolean(durableAuthority) }; const common = { diff --git a/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts b/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts new file mode 100644 index 00000000000..6ddf261b653 --- /dev/null +++ b/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts @@ -0,0 +1,137 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, applyPendingMigrations, companies, createDb, environmentLeases, environments, heartbeatRunEvents, + heartbeatRuns, issues, nativeRunFinalizations, +} from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { recoveryService } from "../recovery/service.js"; +import { claimNativeRestartRecoveries } from "./native-restart-recovery.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +// A remote per-turn run fails while restoring its provider session. The +// coordinator schedules a same-run resume and keeps the task locks. The +// stale-lock sweeper then sees a failed run, and the remote resume must still +// run when its retry delay ends. +(support.supported ? describe : describe.skip)("scheduled remote native resume and the task execution lock", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + beforeAll(async () => { + if (externalUrl) { + await applyPendingMigrations(externalUrl); + db = createDb(externalUrl); + } else { + temporary = await startEmbeddedPostgresTestDatabase("native-scheduled-resume-lock-"); + db = createDb(temporary.connectionString); + } + }, 30_000); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function seed(options: { lockedBy?: "self" | "none" } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const sessionId = randomUUID(); + const failedAt = new Date(Date.now() - 5 * 60_000); + await db.insert(companies).values({ id: companyId, name: "Scheduled resume", issuePrefix: companyId.slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Engineer", adapterType: "paperclip_runner" }); + const lock = options.lockedBy === "none" ? null : runId; + await db.insert(issues).values({ id: issueId, companyId, title: "Continue after approval", status: "in_progress", + assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, nativeIssueId: issueId, + runtimeMode: "native", status: "failed", nativePhase: "retryable_failure", nativeSessionId: sessionId, + processPid: 4_242, processStartedAt: failedAt, startedAt: failedAt, finishedAt: new Date(), + errorCode: "native_session_interrupted", error: "native_session_recovery_failed: provider session was not restored", + contextSnapshot: { issueId }, + }); + const [environment] = await db.insert(environments).values({ name: `Scheduled resume ${runId}`, driver: "sandbox" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId, + environmentId: environment.id, provider: "daytona", providerLeaseId: randomUUID(), status: "active", + leasePolicy: "ephemeral", metadata: { driver: "sandbox", remoteCwd: "/work" }, + }).returning(); + await db.update(heartbeatRuns).set({ runnerProfileJson: { + nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: runId, + leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work", resourceDisposition: "destroy", + }, + sessionCheckpoint: { + sessionId: "driver-session", providerSessionId: "provider-session", + identity: { runId, companyId, issueId, agentId, sessionId }, + }, + } }).where(eq(heartbeatRuns.id, runId)); + await db.update(issues).set({ executionRunId: lock, checkoutRunId: lock, executionLockedAt: lock ? failedAt : null }) + .where(eq(issues.id, issueId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "retryable_failure", attempt: 1, + nextAttemptAt: new Date(Date.now() - 60_000), recoveryState: "resuming_session", + failureCode: "native_session_interrupted", + failureDetail: { recoveryMode: "exact_checkpoint_resume", checkpointExists: true, + providerSessionEstablished: true, providerEventsExist: false }, + }); + return { companyId, agentId, issueId, runId }; + } + + const readIssue = async (id: string) => (await db.select().from(issues).where(eq(issues.id, id)))[0]!; + const readCoordinator = async (runId: string) => + (await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]!; + const sweep = () => recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks(); + + it("keeps the task locks while the failed run's coordinator owns a scheduled resume", async () => { + const f = await seed(); + const result = await sweep(); + expect(result.issueIds).not.toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId }); + }); + + it("keeps the task locks when a resume claim returns the run to running before the clear", async () => { + const f = await seed(); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", nextAttemptAt: null }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const beforeStaleLockClear = vi.fn(async (issueId: string) => { + if (issueId !== f.issueId) return; + await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(heartbeatRuns).set({ status: "running", nativePhase: "observed" }).where(eq(heartbeatRuns.id, f.runId)); + }); + const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeStaleLockClear }).sweepStaleIssueLocks(); + expect(beforeStaleLockClear).toHaveBeenCalledWith(f.issueId); + expect(result.issueIds).not.toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId }); + }); + + it("still clears a closed task's locks for a failed run with a scheduled resume", async () => { + const f = await seed(); + await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, f.issueId)); + expect((await sweep()).issueIds).toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: null, checkoutRunId: null }); + }); + + it("claims a due remote resume whose cleared task lock is free and reclaims it", async () => { + const f = await seed({ lockedBy: "none" }); + const dispositions = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.runId] }); + expect(dispositions).toEqual([expect.objectContaining({ kind: "reattach_remote_runner", runId: f.runId })]); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId }); + expect(await readCoordinator(f.runId)).toMatchObject({ phase: "observed", attempt: 1, nextAttemptAt: null }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]) + .toMatchObject({ status: "running", nativePhase: "observed" }); + const events = await db.select({ eventType: heartbeatRunEvents.eventType }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "native.recovery.execution_reclaimed"))); + expect(events).toHaveLength(1); + }); + + it("does not reclaim a task that another run owns or has admitted", async () => { + for (const competitor of ["lock", "queued"] as const) { + const f = await seed({ lockedBy: "none" }); + const otherRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: otherRunId, companyId: f.companyId, agentId: f.agentId, + issueId: f.issueId, status: competitor === "lock" ? "running" : "queued", contextSnapshot: { issueId: f.issueId } }); + if (competitor === "lock") await db.update(issues).set({ executionRunId: otherRunId }).where(eq(issues.id, f.issueId)); + expect(await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.runId] })).toEqual([ + { kind: "blocked", runId: f.runId, reason: "issue_execution_lock_changed" }, + ]); + expect((await readIssue(f.issueId)).executionRunId).toBe(competitor === "lock" ? otherRunId : null); + expect(await readCoordinator(f.runId)).toMatchObject({ phase: "retryable_failure", attempt: 1 }); + } + }); +}); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 86df326e85f..4e76dd2b69c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -84,6 +84,7 @@ import * as noLaunchProofModule from "./native-maintenance-no-launch.js"; import { NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionProtocolIntegrityError, } from "../../vendor/paperclip-runner/index.js"; @@ -11752,6 +11753,21 @@ describe("native session bounded recovery", () => { }, ); + it("stops same-run retries when the runner proves the saved provider session incompatible", () => { + const failure = new NativeProviderStateIncompatibleError( + "Error: PRP command run.attach failed: ACPX durable launch profile digest does not match runner startup", + ); + const code = nativeSessionFailureSourceCode(failure); + expect(code).toBe("native_provider_state_incompatible"); + const disposition = nativeSessionFailureDisposition(1, new Date(), code); + expect(disposition).toEqual({ phase: "terminal_failure", failureCode: code, nextAttemptAt: null }); + expect(nativeSessionRecoveryProjection({ ...disposition, agentId: "agent" })).toMatchObject({ + issueStatus: "blocked", recoveryOwner: { kind: "board" }, + }); + // Only the typed runner class is deterministic; the same text stays retryable. + expect(nativeSessionFailureSourceCode(new Error(failure.message))).toBe("native_session_interrupted"); + }); + it("makes only typed operator-required cleanup quarantine terminal on the first attempt", () => { const code = nativeSessionFailureSourceCode( new NativeSessionCleanupQuarantinedError(), diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 06a4f78b4a2..c152cb32e15 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -140,6 +140,7 @@ import type { PrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; import { + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, @@ -218,7 +219,7 @@ import { renderNativeRunnerStagedAttachmentPrompt, stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; -import { nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { NATIVE_PROVIDER_STATE_INCOMPATIBLE, nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; @@ -6989,6 +6990,7 @@ export function nativeSessionFailureDisposition( sourceFailureCode === "runner_remote_provider_artifact_incompatible" || sourceFailureCode === "runner_remote_provider_verification_unavailable" || sourceFailureCode === "native_provider_terminal_failed" || + sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE || sourceFailureCode === "native_current_wake_comments_unread" || sourceFailureCode === "native_current_wake_comments_changed_after_read" || sourceFailureCode === "native_session_cleanup_quarantined" || @@ -7039,6 +7041,7 @@ export function nativeSessionFailureSourceCode( ): | "native_permission_declined" | "native_provider_terminal_failed" + | typeof NATIVE_PROVIDER_STATE_INCOMPATIBLE | "native_provider_approval_required" | "native_provider_usage_limit" | "native_session_cleanup_quarantined" @@ -7065,6 +7068,7 @@ export function nativeSessionFailureSourceCode( | "provider_session_open_rejected" | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; + if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; if (error instanceof NativeProviderTerminalFailure) { if (error.providerCode === "approval_required") return "native_provider_approval_required"; // Failed terminals retain their security meaning across the provider facade. @@ -7155,6 +7159,9 @@ export function nativeSessionFailureSourceCode( return "native_session_interrupted"; } +const NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION = + "The current runner cannot restore this task's saved provider session, so retrying this run cannot succeed. Retry the task to continue in a fresh provider session with the same workspace. The earlier provider conversation is not carried over."; + const NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE = "Send a new message to continue after Paperclip verifies that the previous provider and its tools have stopped. If cleanup cannot be verified, inspect the run and its environment. Clearing a task session does not resolve this quarantine. Automatic retries are stopped."; @@ -10491,6 +10498,8 @@ async function executePaperclipNativeSessionWithinScope( ? "Provider files could not be verified after two read attempts. Inspect the retained workspace and command transport before retrying the original run." : sourceFailureCode === "native_provider_terminal_failed" ? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation." + : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE + ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and durable events; state is ambiguous and a replacement provider session is forbidden." : integrityFailure @@ -10683,6 +10692,8 @@ async function executePaperclipNativeSessionWithinScope( ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE : sourceFailureCode === "native_provider_terminal_failed" ? "Verify that the failed provider stopped and reconcile its action outcomes. A linked continuation can proceed only after these checks succeed." + : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE + ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and explicitly resolve the ambiguous session state; do not open a replacement provider session." : integrityFailure diff --git a/server/src/services/native-runtime/native-session-resume-projection.test.ts b/server/src/services/native-runtime/native-session-resume-projection.test.ts index 502bcaf68f3..2838347e5bd 100644 --- a/server/src/services/native-runtime/native-session-resume-projection.test.ts +++ b/server/src/services/native-runtime/native-session-resume-projection.test.ts @@ -28,6 +28,7 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport( }); const select = () => findNativeSessionResumeRun(db, input); const source = () => run(0, { runnerInstanceId: randomUUID(), processPid: 17, processGroupId: 19, processStartedAt: at(0), + errorCode: "native_session_interrupted", runnerProfileJson: profile, contextSnapshot: { paperclipWake, paperclipIssue: { ...paperclipIssue, unrelated: "Omit" }, prompt: "Omit" } }); const writeProfile = (id: string, value: unknown) => db.execute(sql`update ${heartbeatRuns} set runner_profile_json = ${JSON.stringify(value)}::jsonb where ${heartbeatRuns.id} = ${id}`); diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 932260c5640..9765563ee98 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -62,6 +62,7 @@ import { buildNativeExecutionInput } from "./native-execution-input.js"; import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, + NATIVE_PROVIDER_STATE_INCOMPATIBLE, NATIVE_TOOL_CONTRACT_FINGERPRINT, isUnusedLegacyNativeRetryReplacement, nativeToolContractFingerprintForTarget, @@ -278,6 +279,28 @@ describe("capability-gated connection tool refresh", () => { }); }); +describe("a source whose saved provider session the runner proved incompatible", () => { + it("continues the task in a fresh provider session with the same workspace", () => { + const current = execution(currentRunId); + expect(rebindNativeSessionCheckpoint({ previousRun: { ...previousRun(), errorCode: "native_session_interrupted" }, + currentExecution: current })).not.toBeNull(); + const source = { ...previousRun(), errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE }; + expect(rebindNativeSessionCheckpoint({ previousRun: source, currentExecution: current })).toBeNull(); + const buildExecution = vi.fn(({ normalizedSessionId: sessionId, resumedSession }) => { + const built = execution(currentRunId); + built.session.normalizedSessionId = sessionId; + built.task.prompt = resumedSession ? "wake delta" : "original goal and complete handoff"; + return built; + }); + const result = buildNativeExecutionWithCheckpoint({ previousRun: source, normalizedSessionId, buildExecution }); + expect(result.checkpoint).toBeNull(); + expect(result.normalizedSessionId).not.toBe(normalizedSessionId); + expect(result.execution.task.prompt).toContain("complete handoff"); + expect(result.execution.binding.executionWorkspaceId).toBe(current.binding.executionWorkspaceId); + expect(result.execution.workspace).toEqual(current.workspace); + }); +}); + it("wires exact-session recovery and guarded selected identity into heartbeat persistence", () => { const source = readFileSync( new URL("../heartbeat/runtime-selection.ts", import.meta.url), @@ -1219,6 +1242,41 @@ const recoveryFakeCodex = resolve( currentExecution: execution(currentRunId), }), ).toBeNull(); + // A run whose saved provider session the runner proved unrestorable + // stays the barrier, and its checkpoint is not resumed. + const unrestorableId = randomUUID(); + const sourceCheckpoint = source.runnerProfileJson.sessionCheckpoint; + await db.insert(heartbeatRuns).values({ + ...unused, + id: unrestorableId, + createdAt: at(8), + errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + runnerProfileJson: { + ...source.runnerProfileJson, + nativeExecutionInput: execution(unrestorableId), + sessionCheckpoint: { + ...sourceCheckpoint, + identity: { ...sourceCheckpoint.identity, runId: unrestorableId }, + }, + }, + }); + const unrestorable = await findNativeSessionResumeRun(db, selection); + expect(unrestorable).toMatchObject({ + id: unrestorableId, + errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + }); + expect( + rebindNativeSessionCheckpoint({ + previousRun: { ...unrestorable!, errorCode: null }, + currentExecution: execution(currentRunId), + }), + ).not.toBeNull(); + expect( + rebindNativeSessionCheckpoint({ + previousRun: unrestorable!, + currentExecution: execution(currentRunId), + }), + ).toBeNull(); const fresh = buildNativeExecutionWithCheckpoint({ previousRun: null, diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 135eb9c27b2..9e4fe58b486 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -94,6 +94,10 @@ export function nativeToolContractFingerprintForTarget( export const NATIVE_TOOL_CONTRACT_FINGERPRINT = nativeToolContractFingerprintForTarget("local"); +/** The runner proved that this run's saved provider session cannot be + * restored by the current runner build. Its checkpoint is not resumable. */ +export const NATIVE_PROVIDER_STATE_INCOMPATIBLE = "native_provider_state_incompatible"; + function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -341,6 +345,7 @@ async function selectNativeSessionResumeBarrier(db: Db, input: NativeSessionResu nativeIssueId: heartbeatRuns.nativeIssueId, runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, + errorCode: heartbeatRuns.errorCode, createdAt: heartbeatRuns.createdAt, processPid: heartbeatRuns.processPid, processGroupId: heartbeatRuns.processGroupId, @@ -596,6 +601,7 @@ export function rebindNativeSessionCheckpoint(input: { companyId: string; agentId: string; nativeSessionId: string | null; + errorCode?: string | null; runnerProfileJson: unknown; }; currentExecution: NativeExecutionInput; @@ -604,6 +610,9 @@ export function rebindNativeSessionCheckpoint(input: { refreshTools?: boolean; }): PersistedNativeSession | null { if (input.refreshTools && input.toolRefreshOnResume !== true) return null; + // The source is still the progress barrier. Its provider session cannot be + // restored, so the next run starts a fresh session in the same workspace. + if (input.previousRun.errorCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE) return null; const previousProfile = record(input.previousRun.runnerProfileJson); if ( previousProfile.nativeToolContractFingerprint !== diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index f935b8a2df9..b380f431ed5 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -975,6 +975,7 @@ export function recoveryService( ) => boolean; liveRunExecutions?: Readonly<{ has(id: string): boolean }>; beforeOrphanedRunTerminalWrite?: (runId: string) => Promise; + beforeStaleLockClear?: (issueId: string) => Promise; }, ) { const issuesSvc = issueService(db); @@ -6477,6 +6478,26 @@ export function recoveryService( if (!status) return true; // missing run row → no real claim return TERMINAL_HEARTBEAT_RUN_STATUSES.has(status); }; + // A native failure that schedules a same-run resume marks its run failed + // but keeps both task locks for that resume. Clearing them lets another + // run take the task first, and then the resume can never run. Only a + // closed task releases them here. + const pendingResumeLock = sql`exists (select 1 from ${nativeRunFinalizations} pending_resume + where pending_resume.company_id = ${issues.companyId} + and pending_resume.run_id in (${issues.checkoutRunId}, ${issues.executionRunId}) + and pending_resume.phase = 'retryable_failure' + and pending_resume.result_id is null)`; + const lockRunLive = sql`exists (select 1 from ${heartbeatRuns} lock_run + where lock_run.id in (${issues.checkoutRunId}, ${issues.executionRunId}) + and lock_run.status not in (${sql.join([...TERMINAL_HEARTBEAT_RUN_STATUSES].map(status => sql`${status}`), sql`, `)}))`; + const pendingResumeRunIds = new Set(referencedRunIds.length === 0 ? [] : (await db + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where(and( + inArray(nativeRunFinalizations.runId, referencedRunIds), + eq(nativeRunFinalizations.phase, "retryable_failure"), + isNull(nativeRunFinalizations.resultId), + ))).map(row => row.runId)); for (const issue of candidates) { const originalOwner = issue.executionRunId ? runById.get(issue.executionRunId) : undefined; @@ -6497,7 +6518,13 @@ export function recoveryService( ) { continue; } + const taskClosed = issue.status === "done" || issue.status === "cancelled"; + if (!taskClosed && [issue.checkoutRunId, issue.executionRunId] + .some(runId => runId !== null && pendingResumeRunIds.has(runId))) { + continue; + } + await deps.beforeStaleLockClear?.(issue.id); const updated = await db.transaction(async tx => { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); @@ -6519,6 +6546,10 @@ export function recoveryService( issue.executionRunId ? eq(issues.executionRunId, issue.executionRunId) : isNull(issues.executionRunId), + // The reads above can be stale here. A resume claim can return a + // referenced run to running, and a failure can schedule a resume. + not(lockRunLive), + taskClosed ? undefined : not(pendingResumeLock), ), ) .returning({ id: issues.id }) diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 80b70e5b9f2..51ddeff3ea2 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -182,6 +182,7 @@ export const validatePrpStructuredRunResult = runner.validatePrpStructuredRunResult; export const NativeProviderTerminalFailure = runner.NativeProviderTerminalFailure; +export const NativeProviderStateIncompatibleError = runner.NativeProviderStateIncompatibleError; export const nativeRestartInterruptedTurnId = runner.nativeRestartInterruptedTurnId; export const completeTerminatedRemoteNativeSessionCleanup = runner.completeTerminatedRemoteNativeSessionCleanup; From 82dd3bb7025846b4798132174167654343d83e00 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:33:41 -0600 Subject: [PATCH 07/28] Preserve unstarted queued turns during offline enrollment (#18887) ## Thinking Path > - Paperclip manages AI agents and their work. > - Existing-volume enrollment moves server files to durable object storage. > - Enrollment stops application deployments before it changes file authority. > - A task drain keeps unstarted turns queued in the database. > - The migration rejected every queued row, so a busy instance could never finish enrollment without discarding requests. > - This change preserves unstarted queue intent and continues to reject running or previously started executions. > - Users keep their pending turns and saved files through the conversion. ## Linked Issues or Issue Description **What happened?** A preserving enrollment found queued turns after task admission stopped. The workspace, instruction-file, and historical-log guards treated these rows as live writers. The drain also prevented the rows from starting, so waiting could not satisfy the migration guard. **Expected behavior** After every old deployment has stopped, maintenance can move files while unstarted queued turns remain unchanged. The replacement claims those turns through normal admission. Actual executions and retained execution ownership continue to block migration. **Steps to reproduce** 1. Queue a turn behind another task. 2. Start a task drain and let the current task finish. 3. Stop the application deployments and run offline enrollment. 4. Observe `active_execution_requires_drain` for the unstarted queued turn. **Paperclip version or commit** Reproduced on the migration contract through #18864. This branch starts at `ab2d2098a2764c511978d1c3e7a3fc5dfcef07f4`. Related: #18826, #18835, #18841, #18864. Searches for queued enrollment and queued migration found no duplicate fix. ## What Changed - Share one offline execution predicate across workspace, agent-file, and historical-log migration. - Reuse the existing pre-dispatch evidence predicate; queued turns with execution markers, saved checkpoints, source events, coordinators, or environment leases remain blockers. - Keep task bindings and normal live-migration restrictions unchanged. - Exercise normal replacement admission after deleting the old filesystem: the same queued run reads restored workspace bytes and instructions, completes its original wake, and cannot execute twice. - Document the offline boundary and queued-turn behavior. ## Verification - **111 tests passed** across the final 110-case migration suite and assembled queued-turn heartbeat proof, with real PostgreSQL and local S3. This includes workspace/log migration, queued ownership blockers, ordinary agent-file controls, and the actual bundled maintenance process. - The maintenance process preserves the queued row while archiving historical output. The new assembled heartbeat test creates a normal wake under drain, migrates without changing either row, deletes the original filesystem, and resumes through normal admission. It reads saved workspace bytes and the restored instruction bundle, persists one successful native result, completes the same wake, and verifies a second sweep does not duplicate execution. Only sandbox transport/provider execution are scripted; no live model or Daytona calls. - An earlier optional run-evidence worker integration invocation was skipped without its dedicated environment; it is outside the changed offline path. - Server typecheck, server build, and runtime bundle passed. - The first maintenance test exposed the separate agent-file guard; that guard now uses the same offline predicate and the test passes. - Initial local typecheck attempts lacked CLI dependency links in this isolated checkout. Restoring dependencies fixed setup without source changes. - Full repository checks run in CI. The first head lost spot runners in Build and serialized tests; one unchanged idle-deadline fixture crossed UTC midnight and failed its timestamp assertion. Final-head CI passed on run 41744678994 after retrying a spot-runner shutdown. Greptile reviewed the exact final head at 5/5; all inline findings are resolved. The separate Paperclip Review app remains queued. Hosted enrollment remains pending release qualification and compatibility verification for the existing provider configuration. ## Risks The offline predicate relies on Cloud stopping every old application deployment. It does not make online migration safe. Running executions always block. Requeued executions with retained ownership also block. No queued request, task binding, or recovery state is removed or rewritten. No database schema changes. ## Model Used OpenAI Codex, GPT-6 based, with reasoning and tool-assisted code execution. The exact deployed model identifier and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- ...-existing-workspace-ephemeral-migration.md | 23 ++ .../src/__tests__/ephemeral-migration.test.ts | 19 +- ...artbeat-ephemeral-queued-migration.test.ts | 214 ++++++++++++++++++ .../maintenance/ephemeral-migration-data.ts | 4 +- server/src/services/agent-file-maintenance.ts | 4 +- .../services/offline-migration-execution.ts | 18 ++ .../remote-workspace-migration-tree.ts | 3 +- .../remote-workspace-migration.test.ts | 33 ++- .../services/remote-workspace-migration.ts | 3 +- ...run-evidence-migration.integration.test.ts | 10 +- server/src/services/run-evidence-migration.ts | 3 +- 11 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 server/src/__tests__/heartbeat-ephemeral-queued-migration.test.ts create mode 100644 server/src/services/offline-migration-execution.ts diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 0f028d6a037..f4150f62fc9 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -102,3 +102,26 @@ Controller leases, in-flight recovery, active writers, and uncertain workspace operations remain blockers. Tests must prove migration leaves failure state intact, rejects stale copyback even if the old folder exists, and preserves recovery source validation and company boundaries. + +## Queued turns during offline enrollment + +The task drain stops new executions. It does not consume queued turns. After +Cloud has stopped every application deployment, offline maintenance preserves +never-started queued run and wake rows unchanged. Workspace, instruction-file, +and historical-log migration use the same execution predicate at this boundary. +The replacement uses normal queue admission against the migrated authority. + +Running executions still block migration. A queued row also blocks if it has +started, retains process/session/controller evidence, or has a coordinator or +environment lease. Task checkout and execution bindings remain separate blockers. +Live migration retains its original queued-run exclusion. The offline flag alone +is not evidence that old deployments have stopped; Cloud owns that provider fence. + +PostgreSQL/S3 tests cover queued turns and wake payloads surviving migration and +loss of the predecessor filesystem, plus queued execution evidence that must +remain blocked. The packaged maintenance entry must also retain an unstarted +queued turn while it archives historical run output. The assembled heartbeat +test resumes that same queued run after deleting the old disk, verifies saved +workspace bytes and instructions at provider dispatch, and checks one result and +one wake consumption even after a second queue sweep. Its sandbox transport and +provider session are scripted; database, S3, admission and restoration are real. diff --git a/server/src/__tests__/ephemeral-migration.test.ts b/server/src/__tests__/ephemeral-migration.test.ts index 16ac2b79372..a2687c0e4ec 100644 --- a/server/src/__tests__/ephemeral-migration.test.ts +++ b/server/src/__tests__/ephemeral-migration.test.ts @@ -10,7 +10,7 @@ import { randomBytes, randomUUID } from "node:crypto"; import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; +import { activityLog, agentWakeupRequests, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js"; @@ -53,6 +53,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll await db.delete(issues).where(eq(issues.companyId, companyId)); await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId)); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); await db.delete(agents).where(eq(agents.companyId, companyId)); await db.delete(companies).where(eq(companies.id, companyId)); let token: string | undefined; @@ -104,7 +105,8 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task"); await fs.mkdir(child, { recursive: true }); const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }]; - const ids: string[] = []; + const ids: string[] = [], queued = []; + const queuedAgent = await agent(); for (const fixture of fixtures) { await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes); const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning(); @@ -112,10 +114,20 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning(); ids.push(workspace!.id); await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id)); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: queuedAgent.id, + source: "on_demand", payload: { issueId: issue!.id, text: "Continue my saved work" } }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: queuedAgent.id, + issueId: issue!.id, wakeupRequestId: wake!.id, status: "queued", contextSnapshot: { issueId: issue!.id, + executionWorkspaceId: workspace!.id } }).returning(); + queued.push({ run: run!, wake: wake! }); } expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } }); await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-")); vi.stubEnv("PAPERCLIP_HOME", home); + for (const { run, wake } of queued) { + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]).toEqual(wake); + } for (const [index, workspaceId] of ids.entries()) { const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!; @@ -137,6 +149,8 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: row.id, status: "interrupted", runtimeMode: "native", logStore: "local_file", logRef }); vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", ""); + const [queued] = await db.insert(heartbeatRuns).values({ companyId, agentId: row.id, status: "queued", + contextSnapshot: { input: "Resume this turn after maintenance" } }).returning(); const original = resolveDecisionSigningSecret(); const listener = createServer(); await new Promise(resolve => listener.listen(0, "127.0.0.1", resolve)); @@ -167,6 +181,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll } expect(status).toMatchObject({ id, commit: stamp.commit, state: "ready", eligibility: { eligible: true } }); expect(JSON.stringify(status)).not.toContain(original); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued!.id)))[0]).toEqual(queued); const exported = await fetch(`${origin}/export`, { method: "POST", headers }).then(r => r.json()); expect(exported).toMatchObject({ id, commit: stamp.commit, decisionSigningSecret: original }); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); diff --git a/server/src/__tests__/heartbeat-ephemeral-queued-migration.test.ts b/server/src/__tests__/heartbeat-ephemeral-queued-migration.test.ts new file mode 100644 index 00000000000..a1876242ca7 --- /dev/null +++ b/server/src/__tests__/heartbeat-ephemeral-queued-migration.test.ts @@ -0,0 +1,214 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command } from "@aws-sdk/client-s3"; +import { agents, authUsers, companies, companyMemberships, createDb, closeRegisteredClients, environmentLeases, + environments, executionWorkspaces, heartbeatRuns, agentWakeupRequests, issues, issueWorkProducts, nativeRunResults } from "@paperclipai/db"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeExecutionInput, NativeSession, NativeSessionBackend, PrpEvent } from "@paperclipai/paperclip-runner"; +import type { EnvironmentLease } from "@paperclipai/shared"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createIsolatedTestHome } from "./helpers/isolated-test-home.js"; +import { heartbeatService, startTaskDrain, stopTaskDrain } from "../services/heartbeat.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import * as environmentTarget from "../services/environment-execution-target.js"; +import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js"; +import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; +import { materializeInstructionBytes } from "../services/agent-instruction-files.js"; +import { flushInFlightRunLogMirrors } from "../services/run-log-store.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js"; + +vi.mock("@paperclipai/adapter-utils/execution-target", async importOriginal => ({ + ...await importOriginal(), + // The scripted session makes no sandbox-to-host API calls. + startAdapterExecutionTargetPaperclipBridge: async () => null, +})); + +// Real database, migration, object storage, queue claim, workspace restoration, +// instruction materialization and result settlement. Only the sandbox transport +// and provider session are scripted; neither can read the deleted predecessor. +describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("queued turn after preserving offline enrollment", () => { + let database: Awaited>; + let db: ReturnType, root: string, bucket: string, s3: S3Client; + let home: ReturnType; + let replacement: ReturnType | undefined; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("queued-enrollment-"); + db = createDb(database.connectionString); + home = createIsolatedTestHome(); root = await fs.realpath(home.home); + bucket = `queued-enrollment-${randomUUID()}`; + s3 = new S3Client({ endpoint: process.env.PAPERCLIP_TEST_S3_ENDPOINT, region: "us-east-1", forcePathStyle: true }); + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + vi.stubEnv("DATABASE_URL", database.connectionString); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "predecessor")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "74".repeat(32)); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!); + vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true"); + }, 90_000); + afterAll(async () => { + stopTaskDrain(); + await replacement?.drainActiveRunExecutions(); + await flushInFlightRunLogMirrors(); + vi.restoreAllMocks(); + if (database) await closeRegisteredClients(database.connectionString, { scope: "database" }); + await database?.cleanup(); + if (s3) { + for (;;) { + const page = await s3.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!page.Contents?.length) break; + await s3.send(new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: page.Contents.map(row => ({ Key: row.Key! })) } })); + } + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); s3.destroy(); + } + home?.cleanup(); + vi.unstubAllEnvs(); + }); + + it("admits the original queued run once and reads its saved workspace and instructions without the old disk", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), userId = randomUUID(); + const workProductId = randomUUID(), evidenceRef = `work_product:${workProductId}`; + await instanceSettingsService(db).updateExperimental({ enableNativeRunner: true, enableIsolatedWorkspaces: true }); + await db.insert(authUsers).values({ id: userId, name: "Owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Queued enrollment", issuePrefix: "QEM", defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); + const [environment] = await db.insert(environments).values({ name: `Scripted sandbox ${companyId}`, driver: "sandbox", + status: "active", config: { provider: "daytona" } }).returning(); + const instructions = "Preserved instructions: read saved.txt before completing this turn."; + const instructionsRoot = resolveManagedInstructionsRoot({ id: agentId, companyId, name: "Queued worker", adapterConfig: {} }); + await materializeInstructionBytes(instructionsRoot, "AGENTS.md", Buffer.from(instructions)); + await db.insert(agents).values({ id: agentId, companyId, name: "Queued worker", adapterType: "paperclip_runner", status: "idle", + defaultEnvironmentId: environment.id, adapterConfig: { provider: "codex", instructionsBundleMode: "managed", instructionsRootPath: instructionsRoot }, + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue saved work", issueNumber: 1, identifier: "QEM-1", + status: "todo", assigneeAgentId: agentId, responsibleUserId: userId, executionWorkspacePreference: "reuse_existing", + executionWorkspaceSettings: { mode: "isolated_workspace" } }); + const oldHome = process.env.PAPERCLIP_HOME!, cwd = path.join(oldHome, "task"); + await fs.mkdir(cwd, { recursive: true }); await fs.writeFile(path.join(cwd, "saved.txt"), "unfinished work before migration"); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issueId, name: "Saved task", + mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issueId)); + await db.insert(issueWorkProducts).values({ id: workProductId, companyId, issueId, type: "artifact", provider: "paperclip", + title: "Migration fixture evidence", status: "ready_for_review", reviewState: "approved" }); + const predecessor = heartbeatService(db); + startTaskDrain({}); + await predecessor.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", reason: "manual", + requestedByActorType: "user", requestedByActorId: userId, payload: { issueId }, contextSnapshot: { issueId, skipIssueComment: true } }); + await predecessor.drainActiveRunExecutions(); + const [queued] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + expect(queued).toMatchObject({ status: "queued", startedAt: null, wakeupRequestId: wake.id }); + expect(wake.status).toBe("queued"); + expect(await migrateEphemeralServerData(db, { signal: AbortSignal.timeout(60_000), phase: () => {} })) + .toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 1 } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)))[0]).toEqual(queued); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]).toEqual(wake); + await fs.rm(oldHome, { recursive: true }); + await expect(fs.stat(oldHome)).rejects.toMatchObject({ code: "ENOENT" }); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "replacement")); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "false"); + + const remoteCwd = path.join(root, "independent-sandbox"); await fs.mkdir(remoteCwd); + const exec = promisify(execFile); + const transfer: NonNullable = async operations => { + const results = []; + for (const operation of operations) { + for (const file of operation.files) { + await fs.mkdir(path.dirname(file.targetPath), { recursive: true }); + await fs.cp(file.sourcePath, file.targetPath, { recursive: file.kind === "directory" }); + } + results.push({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred: 0 }); + } + return { operations: results }; + }; + const runner: CommandManagedRuntimeRunner = { syncIn: transfer, syncOut: transfer, execute: async input => { + const command = exec(input.command, input.args ?? [], { cwd: input.cwd ?? remoteCwd, env: { ...process.env, ...input.env }, + timeout: input.timeoutMs, maxBuffer: 4 * 1024 * 1024 }); + command.child.stdin?.end(input.stdin ?? ""); + const result = await command; + return { exitCode: 0, stdout: result.stdout, stderr: result.stderr, signal: null, timedOut: false, pid: process.pid, startedAt: new Date().toISOString() }; + } }; + vi.spyOn(environmentTarget, "resolveEnvironmentExecutionTarget").mockImplementation(async input => ({ + kind: "remote", transport: "sandbox", remoteCwd: typeof input.leaseMetadata?.remoteCwd === "string" ? input.leaseMetadata.remoteCwd : remoteCwd, + runner, provider: "daytona", leaseId: input.leaseId, sandboxId: `fixture-${queued.id}`, timeoutMs: 60_000, + }) as AdapterExecutionTarget); + const runtime = environmentRuntimeService(db); + vi.spyOn(runtime, "acquireRunLease").mockImplementation(async input => { + const [lease] = await db.insert(environmentLeases).values({ companyId, environmentId: environment.id, heartbeatRunId: input.heartbeatRunId, + issueId, executionWorkspaceId: workspace.id, provider: "daytona", providerLeaseId: `fixture-${queued.id}`, status: "active", metadata: { remoteCwd } }).returning(); + return { environment: input.environment, lease: lease as EnvironmentLease, + leaseContext: { executionWorkspaceId: workspace.id, executionWorkspaceMode: "isolated_workspace" } }; + }); + // The scripted provider owns no external allocation. Retain its directory + // until after the normal native workspace finalizer has completed. + vi.spyOn(runtime, "releaseRunLeases").mockResolvedValue([]); + const startTurn = vi.fn(); + const backendFactory = (execution: NativeExecutionInput): NativeSessionBackend => { + const turnId = `turn-${queued.id}`; + const identity = { companyId, agentId, issueId, runId: queued.id, sessionId: execution.session.normalizedSessionId! }; + const result = structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT); + result.completionClaim.contractRevision = execution.completionContract.contract.revision; + result.completionClaim.criteria = execution.completionContract.contract.criteria.map(criterion => ({ + ...result.completionClaim.criteria[0]!, criterionId: criterion.id, evidenceRefs: [evidenceRef], + })); + result.evidence = [{ kind: "work_product", ref: evidenceRef }]; result.verification[0]!.artifactRef = evidenceRef; + let started!: () => void; + const turnStarted = new Promise(resolve => { started = resolve; }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + async *events() { + await turnStarted; + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)); + yield { schema: "paperclip.prp.event.v1", sourceEventId: `${queued.id}:terminal`, sourceSeq: 1, sourceInstanceId: run.runnerInstanceId!, + sourceKind: "runner", runId: queued.id, normalizedSessionId: identity.sessionId, turnId, eventType: "turn.completed", + schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), payload: {} } satisfies PrpEvent; + }, + async startTurn(input) { + expect(execution.binding.runId).toBe(queued.id); + expect(execution.workspace.cwd).toBeTruthy(); + expect(await fs.readFile(path.join(execution.workspace.cwd!, "saved.txt"), "utf8")).toBe("unfinished work before migration"); + if (!("runtimeContext" in execution)) throw new Error("Expected native runtime context"); + const bundle = execution.runtimeContext.instructions; + const instructionPath = path.join(bundle.bundle.rootPath, bundle.entryPath); + expect(instructionPath.startsWith(`${oldHome}/`)).toBe(false); + expect(await fs.readFile(instructionPath, "utf8")).toBe(instructions); + await expect(fs.stat(path.join(cwd, "saved.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + startTurn(input); started(); return { turnId }; + }, + async result() { await turnStarted; return { result, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId }; }, + async snapshot() { return { backendKind: "mock", sessionId: "queued-migration", identity, providerSessionId: "queued-migration", + activeTurnId: null, pendingRuntimeRequests: [], lineage: [], cursor: "1" }; }, + async close() {}, + }; + return { async descriptor() { return { kind: "mock", name: "queued-migration", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; }, + async openSession() { return session; } }; + }; + replacement = heartbeatService(db, { environmentRuntime: runtime, nativeSessionBackendFactory: backendFactory }); + stopTaskDrain(); + await replacement.resumeQueuedRunsAfterDrain(); await replacement.drainActiveRunExecutions(); + const finished = await replacement.getRun(queued.id); + expect(finished?.status, JSON.stringify({ errorCode: finished?.errorCode, error: finished?.error, result: finished?.resultJson })).toBe("succeeded"); + expect(finished?.wakeupRequestId).toBe(wake.id); + await replacement.resumeQueuedRunsAfterDrain(); await replacement.drainActiveRunExecutions(); + expect(startTurn).toHaveBeenCalledOnce(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(1); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, queued.id))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))) + .toEqual([expect.objectContaining({ id: wake.id, status: "completed", runId: queued.id, + claimedAt: expect.any(Date), finishedAt: expect.any(Date), payload: wake.payload })]); + }, 120_000); +}); diff --git a/server/src/maintenance/ephemeral-migration-data.ts b/server/src/maintenance/ephemeral-migration-data.ts index 81b54d68f72..a815a51db65 100644 --- a/server/src/maintenance/ephemeral-migration-data.ts +++ b/server/src/maintenance/ephemeral-migration-data.ts @@ -1,5 +1,5 @@ -import { inArray } from "drizzle-orm"; import { heartbeatRuns, type Db } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "../services/offline-migration-execution.js"; import { getStorageService } from "../storage/index.js"; import { migrateAgentFiles, verifyAgentFileObjects } from "../services/agent-file-maintenance.js"; import { readEphemeralServerDataEligibility } from "../services/ephemeral-server-contract.js"; @@ -19,7 +19,7 @@ export async function migrateEphemeralServerData(db: Db, input: { const { signal, phase } = input; const counts: Record = {}; const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(inArray(heartbeatRuns.status, ["running", "queued"])).limit(1); + .where(offlineMigrationExecutionWhere()).limit(1); if (active) throw new Error("active_execution_requires_drain"); if (await hasUnsettledLocalInstructionCopies(db)) throw new Error("local_instruction_copies_require_settlement"); const before = await readEphemeralServerDataEligibility(db); diff --git a/server/src/services/agent-file-maintenance.ts b/server/src/services/agent-file-maintenance.ts index 1888e742008..eec32d2fbed 100644 --- a/server/src/services/agent-file-maintenance.ts +++ b/server/src/services/agent-file-maintenance.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { and, asc, eq, gt, inArray } from "drizzle-orm"; import { agentFileRoots, agents, fileStorageDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import type { StorageProvider, StorageService } from "../storage/types.js"; import { adoptAgentFiles, inspectAgentDirectory } from "./agent-file-store.js"; import { agentInstructionsBundleMode, deriveBundleState, resolveManagedInstructionsRoot } from "./agent-instructions.js"; @@ -33,7 +34,8 @@ export async function migrateAgentFiles(db: Db, input: { if (agentInstructionsBundleMode(agent) === "external") return "external_instructions"; const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), - inArray(heartbeatRuns.status, ["running", "queued"]), + input.writersStopped && process.env.PAPERCLIP_EPHEMERAL_MIGRATION === "true" + ? offlineMigrationExecutionWhere() : inArray(heartbeatRuns.status, ["running", "queued"]), )).limit(1); if (run) return "active_execution"; const [head] = await tx.select({ generation: agentFileRoots.generation }).from(agentFileRoots) diff --git a/server/src/services/offline-migration-execution.ts b/server/src/services/offline-migration-execution.ts new file mode 100644 index 00000000000..2c0b0ed9e8d --- /dev/null +++ b/server/src/services/offline-migration-execution.ts @@ -0,0 +1,18 @@ +import { and, eq, isNotNull, not, or } from "drizzle-orm"; +import { heartbeatRuns } from "@paperclipai/db"; +import { noRetainedExecutionPredicate } from "./pre-dispatch-execution-evidence.js"; + +/** Only for offline maintenance after every application deployment has stopped. + * Never-started queued turns are durable admission intent, not filesystem + * writers. Keep their rows untouched so the replacement can claim them normally. + * A requeued execution or one with retained execution ownership still blocks. */ +export function offlineMigrationExecutionWhere() { + return or( + eq(heartbeatRuns.status, "running"), + and(eq(heartbeatRuns.status, "queued"), or( + not(noRetainedExecutionPredicate()!), + isNotNull(heartbeatRuns.runnerInstanceId), + isNotNull(heartbeatRuns.nativePhase), + )), + ); +} diff --git a/server/src/services/remote-workspace-migration-tree.ts b/server/src/services/remote-workspace-migration-tree.ts index 3c7deed9113..4685a231f62 100644 --- a/server/src/services/remote-workspace-migration-tree.ts +++ b/server/src/services/remote-workspace-migration-tree.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { isDeepStrictEqual } from "node:util"; import { and, eq, inArray, ne, sql } from "drizzle-orm"; import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, workspaceOperations, type Db } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import type { StorageService } from "../storage/types.js"; import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; @@ -41,7 +42,7 @@ export function validateImportedWorkspaceChildren(raw: unknown): ImportedWorkspa export async function offlineWorkspaceMigrationBlockers(db: ReadDb) { if (process.env.PAPERCLIP_EPHEMERAL_MIGRATION !== "true") return ["offline_maintenance_required"]; const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(inArray(heartbeatRuns.status, ["running", "queued"])).limit(1); + .where(offlineMigrationExecutionWhere()).limit(1); return active ? ["active_execution_requires_drain"] : []; } diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index 8b75b7be4e1..498ac0797d7 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -112,14 +112,43 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("offline_maintenance_required"); await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).rejects.toThrow("offline_maintenance_required"); }); - it.each(["running", "queued"])("refuses offline archived imports while any execution is %s", async status => { + it.each(["running", "queued"])("refuses offline archived imports while any execution is running or requeued (%s)", async status => { const f = await seed(); await retainDirectory(f); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Unrelated writer" }).returning(); - await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, status }); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, status, startedAt: status === "queued" ? new Date() : null }); await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) .rejects.toThrow("active_execution_requires_drain"); expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); }); + it.each(["startedAt", "runtimeModeResolvedAt", "processPid", "processStartedAt", "runnerInstanceId", "nativeSessionId", "nativePhase", "controllerLeaseExpiresAt", "executionStage", "checkpoint", "finalizer", "lease"])("keeps queued execution with %s as an offline blocker", async evidence => { + const f = await seed(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Retained execution" }).returning(); + const patch = evidence.endsWith("At") ? { [evidence]: new Date() } + : evidence === "processPid" ? { processPid: 123 } + : evidence.endsWith("Id") ? { [evidence]: randomUUID() } + : evidence === "nativePhase" ? { nativePhase: "observed" } + : evidence === "executionStage" ? { executionStage: "dispatch" } + : evidence === "checkpoint" ? { runnerProfileJson: { sessionCheckpoint: {} } } : {}; + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, + nativeIssueId: evidence === "finalizer" ? f.issueId : null, status: "queued", ...patch }).returning(); + if (evidence === "finalizer") await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, + issueId: f.issueId, runId: run!.id, phase: "observed" }); + if (evidence === "lease") await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: run!.id }); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) + .rejects.toThrow("active_execution_requires_drain"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + }); + it("preserves an unstarted queued turn during offline import while live migration still refuses it", async () => { + const f = await seed(); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Queued turn" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, + issueId: f.issueId, status: "queued", contextSnapshot: { issueId: f.issueId, executionWorkspaceId: f.workspaceId } }).returning(); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("active_execution"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id)))[0]).toEqual(run); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).not.toBeNull(); + }); it("detects overlapping folders reached through symlinked parent directories", async () => { const f = await seed(), alias = path.join(root, "alias"); await fs.symlink(root, alias); await db.insert(executionWorkspaces).values({ companyId: f.companyId, name: "Physical alias", mode: "isolated_workspace", diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index 7ba8b29a79c..f95ed723b3f 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -8,6 +8,7 @@ import { heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db, } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import type { StorageService } from "../storage/types.js"; import { persistActivity, publishActivity } from "./activity-log.js"; import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; @@ -93,7 +94,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri if (workspace.mode === "shared_workspace" && (issuePolicy.sharedWorkspaceConcurrency ?? (projectPolicy.enabled === true ? projectPolicy.sharedWorkspaceConcurrency : undefined)) === "allow") reasons.push("parallel_shared_writers_unsupported"); if (issue?.executionRunId || issue?.checkoutRunId) reasons.push("task_writer_bound"); const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, workspace.companyId), - inArray(heartbeatRuns.status, ["queued", "running"]), sql`(${heartbeatRuns.nativeIssueId} = ${workspace.sourceIssueId}::uuid + input.writersStopped ? offlineMigrationExecutionWhere() : inArray(heartbeatRuns.status, ["queued", "running"]), sql`(${heartbeatRuns.nativeIssueId} = ${workspace.sourceIssueId}::uuid or ${heartbeatRuns.issueId} = ${workspace.sourceIssueId}::uuid or ${heartbeatRuns.contextSnapshot}->>'executionWorkspaceId' = ${workspace.id} or ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'workspaceId' = ${workspace.id})`, diff --git a/server/src/services/run-evidence-migration.integration.test.ts b/server/src/services/run-evidence-migration.integration.test.ts index 7a57c72d91b..1d9cfd463ba 100644 --- a/server/src/services/run-evidence-migration.integration.test.ts +++ b/server/src/services/run-evidence-migration.integration.test.ts @@ -35,7 +35,11 @@ describe.skipIf(!endpoint || !databaseUrl)("run evidence migration (Postgres and await db.execute(sql.raw(`create schema ${schema}`)); await db.execute(sql.raw(`set search_path=${schema}`)); await db.execute(sql`create table heartbeat_runs (id uuid primary key, company_id uuid not null, agent_id uuid not null, - status text not null, runtime_mode text default 'legacy', log_store text, log_ref text, log_bytes bigint, log_sha256 text, last_output_seq int not null default 0, controller_lease_expires_at timestamptz)`); + status text not null, runtime_mode text default 'legacy', log_store text, log_ref text, log_bytes bigint, log_sha256 text, last_output_seq int not null default 0, controller_lease_expires_at timestamptz, started_at timestamptz, process_pid int, process_started_at timestamptz, + runner_instance_id uuid, native_session_id uuid, native_phase text, runtime_mode_resolved_at timestamptz, + process_group_id int, native_issue_id uuid, session_id_after text, controller_boot_id uuid, + execution_stage text, result_json jsonb, runner_profile_json jsonb)`); + await db.execute(sql`create table environment_leases(company_id uuid, heartbeat_run_id uuid)`); await db.execute( sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`, ); @@ -123,7 +127,11 @@ describe.skipIf(!endpoint || !databaseUrl)("run evidence migration (Postgres and await db.execute(sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${active},${randomUUID()},${randomUUID()},'running')`); await expect(migrateRunEvidence(db, options)).rejects.toThrow("active_execution_requires_drain"); await db.execute(sql`delete from heartbeat_runs where id=${active}`); + await db.execute(sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${active},${companyId},${agentId},'queued')`); const result = await migrateRunEvidence(db, options); + expect((await db.$client.unsafe("select status,started_at from heartbeat_runs where id=$1", [active]))[0]) + .toMatchObject({ status: "queued", started_at: null }); + await db.execute(sql`delete from heartbeat_runs where id=${active}`); const store = getRunLogStore(); for (const name of cases.slice(0, 3)) { const run = runs.get(name)!; diff --git a/server/src/services/run-evidence-migration.ts b/server/src/services/run-evidence-migration.ts index 66d0f786323..41d636a2f55 100644 --- a/server/src/services/run-evidence-migration.ts +++ b/server/src/services/run-evidence-migration.ts @@ -9,6 +9,7 @@ import { providerTraceRecords, type Db, } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { createObjectByteJournal } from "./object-byte-journal.js"; import { @@ -63,7 +64,7 @@ export async function migrateRunEvidence( async function assertNoActiveRuns() { options.signal?.throwIfAborted(); const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(inArray(heartbeatRuns.status, ["queued", "running"])).limit(1); + .where(offlineMigrationExecutionWhere()).limit(1); if (active) throw new Error("active_execution_requires_drain"); } if (preserveHistoricalFiles) { From 8fd917aedfe53c5a7e062e7ec440323aa5870102 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Tue, 3 Nov 2026 16:37:39 -0800 Subject: [PATCH 08/28] fix(native): accept late provider notices in recovery (#18890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Native Runner sessions keep a provider conversation across task turns. > - When a later run gets a new task lease, recovery must prove that the prior run settled. > - Recovery required `run.terminal` to be the last committed event. > - A provider can send a late notification for a settled turn. The Runner then records a `provider.notice.recorded` warning after `run.terminal`. > - This false denial stranded a clean session. The notice stays in the journal, so each later run on that session meets the same denial. > - This pull request lets recovery accept a recoverable notice after the terminal event. All other new events still block recovery. > - The benefit is that a late warning no longer ends session continuity. A denial now also names the check that failed. ## Linked Issues or Issue Description No issue exists. I searched open and closed pull requests and issues for the error code, the event name and the function name. I found no duplicate. **What happened?** A run failed in about nine seconds with `runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven`. No provider work started. The prior run had finished and was committed as succeeded. Its journal had one `provider.notice.recorded` warning, "ignored a notification for a settled Codex turn", 1 ms after `run.terminal`. A second session had the same warning about 61 seconds after a normally completed turn. **Expected behavior** Recovery accepts the session and continues the provider conversation. A recoverable notice does not start work, so it must not block recovery. **Steps to reproduce** 1. Start a warm native session in a plugin sandbox environment. 2. Finish a run so that a provider notice follows `run.terminal` in the journal. 3. Wait until the idle timeout stops the Runner task and releases the lease. 4. Start the next run on that session. It gets a new task lease. 5. The run fails with the error above. The notice stays in the journal, so a retry on the same session meets the same denial. The `local-receipt-late-notice` test mode in `native-session-executor.test.ts` reproduces this failure on the unfixed code. **Paperclip version or commit** Observed on `d8a9b3eb0c`. The same check is on `master` before this change. **Deployment mode** Cloud deployment, with a native Runner in a plugin sandbox environment. ## What Changed - `isSettledRemoteWarmControlState` accepts a `provider.notice.recorded` event after the terminal event. The notice must be recoverable and have `info` or `warning` severity. It must come from the same runner, run, session and lease. - Error notices, unrecoverable notices, notices from another source and new work still block recovery. - Checkpoint recovery and local quiescent recovery now use this function. They no longer have separate checks that require `run.terminal` to be the last event. - Each checkpoint recovery denial adds a specific reason after the existing code, for example `plugin_checkpoint_recovery_unproven: journal_not_settled`. The `runner_state_identity_mismatch: ` prefix does not change. - I reworded the existing comment on `isSettledRemoteWarmControlState` so that it stays correct. I added no new code comments. ## Verification - **Failure before the fix:** I added the tests first and ran them on the unfixed code. 38 tests failed. In the 4 `local-receipt-late-notice` cases, checkpoint recovery rejected with `runner_state_identity_mismatch`. In the `late provider notice` case, quiescent recovery rejected with `quarantine_owner_unverified`. The new unit test for notices also failed. The other 32 failures were assertions for the new denial reasons. - **Targeted tests after the fix:** 134 tests passed, with 987 skipped by the filter. - `cd server && pnpm exec vitest run src/services/native-runtime/remote-warm-session-recovery.test.ts src/services/native-runtime/native-session-executor.test.ts -t "settled remote warm control evidence|recovers a blocked|automatically recovers only a proven settled local session"` - **Type error found in review:** Vitest does not check types, and my first version passed these tests. Review of the diff showed an incompatible `identity` parameter type. I changed that type to `Record`. - **After the type change:** `remote-warm-session-recovery.test.ts` passed (23 tests) and `pnpm run typecheck` in `server` passed. I did not run the 134 tests again, because this change affected only types. - **Real data, read-only:** I ran the new function on 285 session journals from a test deployment. The result changed for 2 journals. These are the 2 stranded sessions, and each has only provider notices after `run.terminal`. - **Not run locally:** I did not run the full test suite. I stopped a run of the wider native-runtime test files before it reported a result. CI provides the full regression coverage. - **Not verified:** no live run has reached `read_state` on a stranded session after this fix. The tests use a fixture for that call. ## Risks - **Stricter identity check:** checkpoint and quiescent recovery now also require the journal's saved identity to match. Recovery reads this identity from the same file, so only damaged state should fail this check. - **Wider goal snapshot rule:** these two paths now accept idle goal snapshots after the terminal event. Warm handoff already used this rule. - **Wider notice rule:** warm handoff and the settled projection check also accept recoverable notices now. A notice does not start work, and the checks for pending commands and unanswered tool input stay in place. - **Longer error text:** code that compares the full `plugin_checkpoint_recovery_unproven` message for equality would stop matching. I found no other reference to this code in the repository. - **Stranded sessions:** these sessions need no data repair. They can still fail with `checkpoint_not_settled` if the sandbox checkpoint is not clean. - **Migrations:** this change has no migration. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), through Claude Code, with tool use and code execution. A human directed the diagnosis and approved the change. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude --- .../native-session-executor.test.ts | 56 +++++++++++++++++-- .../native-runtime/native-session-executor.ts | 37 +++++------- .../remote-warm-session-recovery.test.ts | 26 +++++++++ .../remote-warm-session-recovery.ts | 19 ++++--- 4 files changed, 102 insertions(+), 36 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4e76dd2b69c..ef43bf02e7f 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -14022,7 +14022,7 @@ describe("runnerd provider runtime wiring", () => { || request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false); }); - it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))( + it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-late-notice", "local-receipt-late-work", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))( "recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => { const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude; const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`), @@ -14038,7 +14038,12 @@ describe("runnerd provider runtime wiring", () => { const scoped = await initializeScopedRunnerRoot(first, identity.runnerInstanceId); await mkdir(join(scoped, "control-plane"), { recursive: true }); const journal = JSON.stringify({ ...durableControlPlaneState(identity), commands: [{ status: mode.endsWith("pending-command") ? "pending" : "completed" }], - committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }] }); + committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }, + ...(mode === "local-receipt-late-notice" ? [{ eventType: "provider.notice.recorded", envelope: { ...identity, payload: { schema: "paperclip.prp.event.v1", eventType: "provider.notice.recorded", + runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: identity.runnerInstanceId, sourceKind: "runner", + payload: { schema: "paperclip.provider.notice.v1", noticeId: "codex-warning", severity: "warning", category: "warning", scope: "turn", recoverable: true, userActionable: true, + summary: "ignored a notification for a settled Codex turn" } } } }] : []), + ...(mode === "local-receipt-late-work" ? [{ eventType: "turn.started", envelope: { ...identity } }] : [])] }); await writeFile(join(scoped, "control-plane/control-plane-state.json"), journal); if (mode.startsWith("empty-runner") || mode === "nonempty-runner") { await mkdir(join(scoped, "runner")); @@ -14111,7 +14116,7 @@ describe("runnerd provider runtime wiring", () => { expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); } const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch); - if (["canonical", "resumed", "resumed-timeout", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) { + if (["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-late-notice", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) { await expect(backend).resolves.toBeDefined(); expect(state.createBackend).toHaveBeenCalledOnce(); expect(owner).toHaveBeenCalledWith(expect.objectContaining({ priorIdentity: identity, @@ -14121,9 +14126,18 @@ describe("runnerd provider runtime wiring", () => { expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" }); expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); } else { - await expect(backend).rejects.toThrow(); + await expect(backend).rejects.toThrow({ + "local-receipt-late-work": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "local-receipt-missing-terminal": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "local-receipt-pending-command": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "pending-command": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "missing-terminal": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "ambiguous": "plugin_checkpoint_recovery_unproven: quarantine_ambiguous", + "not-suspended": "plugin_checkpoint_recovery_unproven: checkpoint_not_settled", + "pending-outbox": "plugin_checkpoint_recovery_unproven: checkpoint_not_settled", + }[mode as string] ?? ""); expect(state.createBackend).not.toHaveBeenCalled(); - if (["local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) { + if (["local-receipt-late-work", "local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) { expect(task).not.toHaveBeenCalled(); } if (mode === "ambiguous") expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); @@ -15241,6 +15255,8 @@ describe("runnerd provider runtime wiring", () => { it.each([ "quarantined", "large control plane", + "late provider notice", + "post-terminal work", "empty retry shell", "unsuspended current", "live runner", @@ -15457,6 +15473,35 @@ describe("runnerd provider runtime wiring", () => { })) : []), { eventType: "run.terminal", envelope: identity }, + ...(scenario === "late provider notice" + ? [{ + eventType: "provider.notice.recorded", + envelope: { + ...identity, + payload: { + schema: "paperclip.prp.event.v1", + eventType: "provider.notice.recorded", + runId: identity.runId, + normalizedSessionId: identity.normalizedSessionId, + sourceInstanceId: identity.runnerInstanceId, + sourceKind: "runner", + payload: { + schema: "paperclip.provider.notice.v1", + noticeId: "codex-warning", + severity: "warning", + category: "warning", + scope: "turn", + recoverable: true, + userActionable: true, + summary: "ignored a notification for a settled Codex turn", + }, + }, + }, + }] + : []), + ...(scenario === "post-terminal work" + ? [{ eventType: "turn.started", envelope: identity }] + : []), ], }), ); @@ -15497,6 +15542,7 @@ describe("runnerd provider runtime wiring", () => { "quarantined", "released process identity", "large control plane", + "late provider notice", "empty retry shell", "unsuspended current", "active goal", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c152cb32e15..6bbff7593c7 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5049,25 +5049,25 @@ async function recoverPluginRunnerCheckpoint(input: { target.sandboxLeaseAcquisition?.outcome === "resumed") return; let root = scoped; const quarantined = !existsSync(scoped); - const deny = (): never => { throw new Error("runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven"); }; + const deny = (reason: string): never => { throw new Error(`runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven: ${reason}`); }; if (quarantined) { const quarantine = resolve(runnerdStateBase(), "quarantine"); if (!existsSync(quarantine)) return; - if (!isSafeNativeStateDirectory(quarantine)) return deny(); + if (!isSafeNativeStateDirectory(quarantine)) return deny("quarantine_unsafe"); const candidates = readdirSync(quarantine).filter(name => name.startsWith(`${basename(scoped)}.identity_indeterminate.`)); if (candidates.length === 0) return; - if (candidates.length !== 1) return deny(); + if (candidates.length !== 1) return deny("quarantine_ambiguous"); root = resolve(quarantine, candidates[0]!); } if (!existsSync(root)) return; - if (!isSafeNativeStateDirectory(root)) return deny(); + if (!isSafeNativeStateDirectory(root)) return deny("root_unsafe"); if (hasRetainedWarmTransitionEvidence(root)) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_warm_transition"); return; } const identity = readRunnerdDurableIdentity(root); if (!durableIdentityMatchesSession(identity, input.execution) || identity.runId === input.execution.binding.runId) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_identity_mismatch"); return; } const direct = runnerdAuthorityLifecycle(root, identity); @@ -5075,32 +5075,29 @@ async function recoverPluginRunnerCheckpoint(input: { !existsSync(resolve(root, "runner", "runner-state.json")) && pluginRunnerdAuthorityLifecycle(root, identity) === "suspended"; // Never hide contradictory direct authority behind a remote checkpoint. if (direct !== "absent" && !(quarantined && direct === "suspended") && !verifiedPluginReceipt) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_authority_conflict"); return; } await assertCleanupActivationCommitted(input.db, root, input.execution); const prior = await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }); if (prior !== "terminal_state_indeterminate" && !((quarantined || verifiedPluginReceipt) && prior === "verified")) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_prior_owner_unverified"); return; } const control = record(JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"), NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8"))); - const terminal = record(Array.isArray(control.committedEvents) ? control.committedEvents.at(-1) : null); - if (!Array.isArray(control.commands) || control.commands.some(command => record(command).status !== "completed") || - terminal.eventType !== "run.terminal" || record(terminal.envelope).runId !== identity.runId || - record(terminal.envelope).normalizedSessionId !== identity.normalizedSessionId) return deny(); + if (!isSettledRemoteWarmControlState(control, identity)) return deny("journal_not_settled"); const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId, leaseId: target.leaseId, priorIdentity: identity, inspectCheckpoint: true }); const checkpoint = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "read_state", identity, resumeFrom })); if (checkpoint.kind !== "state" || checkpoint.state.lifecycle !== "suspended" || !Array.isArray(checkpoint.state.outbox) || checkpoint.state.outbox.length !== 0 || checkpoint.state.pendingTerminalDelivery != null || checkpoint.state.pendingProviderCleanup != null || - checkpoint.state.warmTransition != null) return deny(); + checkpoint.state.warmTransition != null) return deny("checkpoint_not_settled"); savePluginRunnerCheckpoint(root, checkpoint.state, identity); - if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny(); + if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny("checkpoint_unverified"); if (quarantined) { - if (existsSync(scoped)) return deny(); + if (existsSync(scoped)) return deny("quarantine_destination_exists"); renameSync(root, scoped); } } @@ -5517,11 +5514,7 @@ async function recoverQuiescentRunnerdState(input: { const runner = record(JSON.parse(runnerBytes)); const control = record(JSON.parse(controlBytes)); const provider = record(JSON.parse(providerBytes)); - const commands = control.commands; const events = control.committedEvents; - const terminalEnvelope = record( - record(Array.isArray(events) ? events.at(-1) : null).envelope, - ); if ( runner.schema !== RUNNERD_STATE_SCHEMA || !["ready", "suspended"].includes(String(runner.lifecycle)) || @@ -5537,12 +5530,8 @@ async function recoverQuiescentRunnerdState(input: { !Array.isArray(runner.outbox) || runner.outbox.length !== 0 || runner.pendingTerminalDelivery !== null || - !Array.isArray(commands) || - commands.some((command) => record(command).status !== "completed") || !Array.isArray(events) || - record(events.at(-1)).eventType !== "run.terminal" || - terminalEnvelope.runId !== identity.runId || - terminalEnvelope.normalizedSessionId !== identity.normalizedSessionId || + !isSettledRemoteWarmControlState(control, identity) || provider.completedTurnAuthoritative !== true ) continue; diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.test.ts b/server/src/services/native-runtime/remote-warm-session-recovery.test.ts index 91317d2af4b..91231e07f14 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.test.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.test.ts @@ -7,6 +7,10 @@ const goal = { schema: "paperclip.session_goal.snapshot.v1", goal: null, working const notification = { schema: "paperclip.prp.event.v1", eventType: "session.goal.snapshot", runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: "runner", sourceKind: "runner", payload: goal }; const snapshot = { eventType: "session.goal.snapshot", envelope: { ...identity, payload: notification } }; +const notice = { schema: "paperclip.provider.notice.v1", noticeId: "codex-warning", severity: "warning", category: "warning", scope: "turn", + recoverable: true, userActionable: true, summary: "ignored a notification for a settled Codex turn" }; +const noticeNotification = { ...notification, eventType: "provider.notice.recorded", payload: notice }; +const lateNotice = { eventType: "provider.notice.recorded", envelope: { ...identity, payload: noticeNotification } }; const state = (events: unknown[]) => ({ identity, commands: [{ status: "completed" }], committedEvents: events }); describe("settled remote warm control evidence", () => { @@ -34,6 +38,28 @@ describe("settled remote warm control evidence", () => { expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false); }); + it("accepts recoverable provider notices recorded after the terminal", () => { + const info = { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, severity: "info" } } } }; + expect(isSettledRemoteWarmControlState(state([terminal, lateNotice]), identity)).toBe(true); + expect(isSettledRemoteWarmControlState(state([terminal, lateNotice, snapshot, info]), identity)).toBe(true); + }); + + it.each([ + ["error notice", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, severity: "error" } } } }], + ["unrecoverable notice", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, recoverable: false } } } }], + ["unknown notice schema", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, schema: "paperclip.provider.notice.v2" } } } }], + ["notification type mismatch", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, eventType: "session.goal.snapshot" } } }], + ["foreign notice source", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, sourceInstanceId: "other" } } }], + ["foreign notice lease", { ...lateNotice, envelope: { ...lateNotice.envelope, environmentLeaseId: "other" } }], + ["control plane notice source", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, sourceKind: "control_plane" } } }], + ])("holds a %s suffix", (_name, suffix) => { + expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false); + }); + + it("holds new work recorded after a late provider notice", () => { + expect(isSettledRemoteWarmControlState(state([terminal, lateNotice, { ...snapshot, eventType: "turn.started" }]), identity)).toBe(false); + }); + it("holds missing terminals, pending commands and mixed core identities", () => { expect(isSettledRemoteWarmControlState(state([snapshot]), identity)).toBe(false); expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), commands: [{ status: "dispatched" }] }, identity)).toBe(false); diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index ec63a72325e..09222910c96 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -14,9 +14,10 @@ export type RemoteWarmSessionRecovery = { revalidate: (leaseOwner: string) => Promise; }; -/** A goal read can publish an idle snapshot after the terminal. It carries no - * new work; every other suffix still requires reconciliation before handoff. */ -export function isSettledRemoteWarmControlState(control: Record, identity: Record): boolean { +/** A goal read can publish an idle snapshot and a provider can publish a + * recoverable notice after the terminal. Neither carries new work; every other + * suffix still requires reconciliation before handoff. */ +export function isSettledRemoteWarmControlState(control: Record, identity: Record): boolean { const record = (value: unknown): Record => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; const commands = control.commands; @@ -34,13 +35,17 @@ export function isSettledRemoteWarmControlState(control: Record const envelope = record(record(event).envelope); const notification = record(envelope.payload); const payload = record(notification.payload); - return record(event).eventType === "session.goal.snapshot" && matchesSession(event) && + const eventType = record(event).eventType; + const inert = eventType === "session.goal.snapshot" + ? payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false + : eventType === "provider.notice.recorded" && payload.schema === "paperclip.provider.notice.v1" && + payload.recoverable === true && (payload.severity === "info" || payload.severity === "warning"); + return inert && matchesSession(event) && Object.entries(identity).every(([key, value]) => envelope[key] === value) && (notification.schema === "paperclip.prp.event.v1" || notification.schema === "paperclip.prp.event.v2") && - notification.eventType === "session.goal.snapshot" && + notification.eventType === eventType && notification.sourceInstanceId === identity.runnerInstanceId && notification.sourceKind === "runner" && - notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId && - payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false; + notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId; }); } From db9be4c5a6074f0755344923422f6b135fcc398a Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:44:14 -0600 Subject: [PATCH 09/28] fix(workspaces): preserve safe one-shot failure messages (#18888) ## Thinking Path > - Paperclip manages agent work and keeps saved work safe. > - Remote workspaces use provider commands to build and copy a verified archive. > - Some providers return a one-shot command's combined output in stdout. > - The workspace transport checked only stderr for its two safe failure messages. > - This hid those messages behind a generic error on that provider path. > - This pull request checks both streams for the same exact messages. > - The benefit is a useful existing error message without exposing other output. ## Linked Issues or Issue Description **What happened?** Daytona one-shot execution returns combined output in stdout and leaves stderr empty. The remote workspace transport checked only stderr after a failed command. It therefore lost the existing full-disk and unfinished-Git-operation messages. **Expected behavior** Preserve those same two exact safe messages from either returned stream. Keep all other output private. Keep the existing stderr preference when both streams contain different safe messages. **Steps to reproduce** Run the real workspace capture helper with a conflicted Git merge or an injected ENOSPC write. Apply the provider's one-shot output contract to the real child result. Before this fix, all three new cases return only remote_workspace_operation_failed. Related provider contract: #10941. The duplicate search found no PR for this exact transport classification gap. ## What Changed - Check stderr and stdout for only the two existing exact safe failure messages. - Add real helper regressions for one-shot Git and full-disk failures. Keep the conflicted files and original saved result intact. - Add privacy, unknown-output, split-message, stderr-preference and healthy verified capture/restore controls. ## Verification - RED: three real helper cases fail on the unchanged transport. They cover a conflicted Git merge and ENOSPC during helper installation and archive creation. - GREEN: 73 tests pass in remote-workspace-transport.test.ts and remote-workspace-snapshot-archive.test.ts. - Server layered typecheck passes: `cd server && node scripts/typecheck-layers.mjs` with Node24.22.2. All seven source/test layers pass. - Full CI is requested with the full-ci label. Current-head CI and review are pending. - The local install used the frozen lock with scripts disabled. The maintained root SDK-link helper restored its skipped local postinstall links. The original prerequisite failure is retained. Generated package outputs were copied only after complete input, installed-toolchain and output-hash equality checks; server declarations were checked in this worktree. - The full workspace typecheck, test suite and build were not run locally. This two-file change uses focused real-helper tests and server typechecking; full CI supplies the broader checks. ## Risks - A failed command can now show one of the existing safe messages when the provider puts it in stdout. Unknown, prefixed, suffixed and private output still produces the generic error. - This change does not repair an archive failure or prove that a retained workspace was saved. It changes no capture, ownership, archive validation, stop, save or retry guard. No live provider command or recovery action was run for this PR. - No API, permission, schema, configuration, map or documentation contract changes are required. ## Model Used - OpenAI GPT-6 (Codex), with reasoning, tool use and code execution. The host does not expose the exact deployment identifier or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .../remote-workspace-transport.test.ts | 74 ++++++++++++++++--- .../services/remote-workspace-transport.ts | 7 +- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 88b13a85991..523e8c40848 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -50,7 +50,11 @@ describe("remote workspace transport safety with real sandbox commands", () => { return root; } - function target(ambientEnv: Record = {}) { + function target(ambientEnv: Record = {}, outputMode: "split" | "one-shot" = "split") { + // Daytona one-shot execution returns the child streams together in stdout. + // Run the real helper first, then apply that provider response contract. + const streams = (stdout: string, stderr: string) => outputMode === "one-shot" + ? { stdout: stdout + stderr, stderr: "" } : { stdout, stderr }; const execute = vi.fn(async input => { try { const result = await exec(input.command, input.args ?? [], { @@ -58,11 +62,12 @@ describe("remote workspace transport safety with real sandbox commands", () => { // Carry only generated fixture values across this execution boundary. env: { PATH: process.env.PATH, ...ambientEnv, ...input.env }, }); - return { ...result, exitCode: 0, timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }; + return { ...streams(result.stdout, result.stderr), exitCode: 0, timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }; } catch (error) { const failure = error as { code?: number; stdout?: string; stderr?: string; signal?: string }; - return { exitCode: typeof failure.code === "number" ? failure.code : 1, stdout: failure.stdout ?? "", - stderr: failure.stderr ?? "", timedOut: false, signal: failure.signal ?? null, + return { exitCode: typeof failure.code === "number" ? failure.code : 1, + ...streams(failure.stdout ?? "", failure.stderr ?? ""), timedOut: false, signal: failure.signal ?? null, pid: process.pid, startedAt: new Date().toISOString() }; } }); @@ -287,7 +292,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { expect(await fs.readdir(outside)).toEqual(["workspace.pcws"]); }); - it("preserves only the fixed unfinished Git operation diagnostic across capture commands", async () => { + it.each(["split", "one-shot"] as const)("preserves only the fixed unfinished Git operation diagnostic with %s output", async outputMode => { await git(workspace, "init", "--initial-branch=main"); await fs.writeFile(path.join(workspace, "work.txt"), "base\n"); await git(workspace, "add", "."); await git(workspace, "commit", "-m", "base"); await git(workspace, "checkout", "-b", "feature"); @@ -295,10 +300,14 @@ describe("remote workspace transport safety with real sandbox commands", () => { await git(workspace, "checkout", "main"); await fs.writeFile(path.join(workspace, "work.txt"), "main\n"); await git(workspace, "commit", "-am", "main"); await expect(git(workspace, "merge", "--no-edit", "feature")).rejects.toThrow(); - const { remote, execute } = target(); + const mergeHead = await fs.readFile(path.join(workspace, ".git", "MERGE_HEAD"), "utf8"); + const conflictedWork = await fs.readFile(path.join(workspace, "work.txt"), "utf8"); + const { remote, execute } = target({}, outputMode); const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; const operationId = randomUUID(); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })).rejects.toThrow(gitOperation); + expect(await fs.readFile(path.join(workspace, ".git", "MERGE_HEAD"), "utf8")).toBe(mergeHead); + expect(await fs.readFile(path.join(workspace, "work.txt"), "utf8")).toBe(conflictedWork); const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId); for (const name of ["workspace.pcws", "receipt.json"]) await expect(fs.access(path.join(directory, name))).rejects.toMatchObject({ code: "ENOENT" }); const failure = { exitCode: 1, stdout: "", stderr: `${gitOperation}\nSYNTHETIC_PRIVATE_PATH_OR_CREDENTIAL`, @@ -312,9 +321,11 @@ describe("remote workspace transport safety with real sandbox commands", () => { }); it.each([ - { step: "helper install", failing: String.raw`/archive\.cjs\.[^/]+\.tmp$/` }, - { step: "archive write", failing: String.raw`/workspace\.pcws$/` }, - ])("names a full sandbox disk when the $step runs out of space", async ({ failing }) => { + { step: "helper install", failing: String.raw`/archive\.cjs\.[^/]+\.tmp$/`, outputMode: "split" as const }, + { step: "archive write", failing: String.raw`/workspace\.pcws$/`, outputMode: "split" as const }, + { step: "helper install", failing: String.raw`/archive\.cjs\.[^/]+\.tmp$/`, outputMode: "one-shot" as const }, + { step: "archive write", failing: String.raw`/workspace\.pcws$/`, outputMode: "one-shot" as const }, + ])("names a full sandbox disk when the $step runs out of space with $outputMode output", async ({ failing, outputMode }) => { await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n"); // Every sandbox Node process inherits this preload. A matching write keeps // one byte and then fails with ENOSPC, as a real full disk does. @@ -327,7 +338,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { fs.open=async(file,...rest)=>{const handle=await open(file,...rest);if(failing.test(String(file))){const write=handle.writeFile.bind(handle); handle.writeFile=async data=>{await write(Buffer.from(data).subarray(0,1));throw enospc(file);};}return handle;}; require('node:module').syncBuiltinESMExports();`); - const { remote, execute } = target(); + const { remote, execute } = target({}, outputMode); const run = execute.getMockImplementation()!; execute.mockImplementation(input => run({ ...input, env: { ...input.env, NODE_OPTIONS: `--require ${preload}` } })); const operationId = randomUUID(); @@ -336,6 +347,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId); const leftovers = await fs.readdir(directory).catch(() => []); expect(leftovers.filter(name => name !== "archive.cjs")).toEqual([]); + expect(await fs.readFile(path.join(workspace, "result.txt"), "utf8")).toBe("synthetic saved result\n"); }); it("keeps the full-disk diagnostic exact", async () => { @@ -347,6 +359,48 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed$/); }); + it.each([ + { label: "private stdout", stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "" }, + { label: "private stderr", stdout: "", stderr: "SYNTHETIC_PRIVATE_STDERR" }, + { label: "suffixed stdout diagnostic", stdout: "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace\nSYNTHETIC_PRIVATE_PATH", stderr: "" }, + { label: "prefixed stdout diagnostic", stdout: "SYNTHETIC_PRIVATE_PATH\nremote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace", stderr: "" }, + { label: "unrelated helper diagnostic", stdout: "remote_workspace_capture_failed", stderr: "" }, + { label: "diagnostic split between streams", stdout: "remote_workspace_sandbox_", stderr: "disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace" }, + ])("keeps $label private", async ({ stdout, stderr }) => { + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: 1, stdout, stderr, timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_operation_failed$/); + expect(execute).toHaveBeenCalledTimes(1); + }); + + it("keeps the existing safe stderr diagnostic when stdout reports another safe failure", async () => { + const { remote, execute } = target(); + const diskFull = "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace"; + execute.mockResolvedValueOnce({ exitCode: 1, stdout: diskFull, + stderr: "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace", + timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace$/); + }); + + it("captures and restores verified bytes with one-shot combined output", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic one-shot bytes\n"); + const { remote, execute } = target({}, "one-shot"); + const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + expect(createHash("sha256").update(await fs.readFile(captured.archivePath)).digest("hex")).toBe(captured.sha256); + const restoredRoot = path.join(scratch, "restored-one-shot"); + const restored = await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: restoredRoot, + archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, restorationIdentity: "c".repeat(64) }); + expect(await fs.readFile(path.join(restoredRoot, "saved.txt"), "utf8")).toBe("saved synthetic one-shot bytes\n"); + expect(await fs.readFile(path.join(workspace, "saved.txt"), "utf8")).toBe("saved synthetic one-shot bytes\n"); + await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot, operationId: restored.operationId }); + expect(execute.mock.calls.every(([input]) => input.bypassSession === true)).toBe(true); + } finally { await captured.cleanup(); } + }); + it("identifies a timed-out preparation without exposing transport output or retrying it", async () => { const { remote, execute } = target(); const resolveGitAuth = vi.fn(async () => null); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index aa9df01d06c..fc95be3b5db 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -55,8 +55,11 @@ async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspace if (result.timedOut) throw new RemoteWorkspaceOperationTimeoutError(operation, timeoutMs); if (result.exitCode !== 0) { const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; - const reported = result.stderr.trim(); - throw new Error([gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(reported) ? reported : "remote_workspace_operation_failed"); + // One-shot providers can return combined output in stdout. Keep the same + // exact safe diagnostics; never forward arbitrary output from either stream. + const reported = [result.stderr.trim(), result.stdout.trim()] + .find(value => [gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(value)); + throw new Error(reported ?? "remote_workspace_operation_failed"); } return result.stdout; } From 7b1c53f8f1cd519c2cc4fb09fe9c343acd67263d Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:54:16 -0600 Subject: [PATCH 10/28] fix(recovery): continue verified preparation after lease interruption (#18892) Use recorded system interruption and the existing execution, controller, stopped-lease, and saved-file proof instead of a shutdown-only error label. Preserve bounded retries, user Stops, task authority, and exactly one continuation. Keep the chat recovery fixture faithful to the preparation stage and terminal time written by the real lifecycle. Retain missing-field safety controls. Co-Authored-By: Paperclip --- doc/execution-semantics.md | 10 +- .../chat-completion-delivery.test.ts | 7 +- .../src/services/cancelled-native-startup.ts | 26 ++++-- .../services/execution-recovery-resolution.ts | 6 +- server/src/services/heartbeat.ts | 3 +- server/src/services/heartbeat/retries.ts | 7 +- server/src/services/heartbeat/run-control.ts | 3 +- .../src/services/legacy-execution-recovery.ts | 6 +- .../src/services/workspace-recovery.test.ts | 92 +++++++++++++++++-- server/src/services/workspace-recovery.ts | 2 +- 10 files changed, 128 insertions(+), 34 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index f18aa7128a7..336687a07a6 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1841,9 +1841,10 @@ An interrupted conversation does not permanently block its task. For local conve Shutdown, process loss, and provider failure use the existing durable failure retry counter and delay. Ordinary failure recovery permits at most two automatic retries in a failure chain. Accepted-interaction infrastructure recovery retains its existing bounded policy. Repeated scheduler visits reuse the same successor; restarting the server does not reset the counter. After exhaustion, automatic attempts stop. A new explicit user message can start a fresh run and failure budget. Productive max-turn continuation and confirmed workspace waits keep their separate existing semantics. -If shutdown interrupts Paperclip Runner preparation before native execution was -selected, recovery verifies that no provider, process launch, native result or -coordinator exists. The previous controller must have settled or lost its lease. +If shutdown or a recorded unexpected system interruption stops Paperclip Runner +preparation before native execution was selected, recovery verifies that no +provider, process launch, native result or coordinator exists. The previous +controller must have settled or lost its lease. Required workspace recovery must have an exact saved-file receipt and stopped leases before the task can continue. Once these checks pass, the server retires its own unchanged bootstrap-failure block and schedules one bounded retry. @@ -1852,7 +1853,8 @@ restart. A pending Retry or saved message owns its continuation instead of starting a competing automatic attempt. Retry uses the same preparation proof and preserves the recovered workspace. Current task ownership, human blocks, pauses, approvals, budgets and the existing retry limit remain effective. -Operator Stops retain their explicit continuation requirements. +Operator Stops retain their explicit continuation requirements. Other ambiguous +interruptions without a server system-cancellation record remain held. Graceful shutdown interrupts an owned legacy adapter through its cancellation control, then waits within one shared 30-second window for provider stop and diff --git a/server/src/__tests__/chat-completion-delivery.test.ts b/server/src/__tests__/chat-completion-delivery.test.ts index 2dfd4290972..df769c99372 100644 --- a/server/src/__tests__/chat-completion-delivery.test.ts +++ b/server/src/__tests__/chat-completion-delivery.test.ts @@ -290,6 +290,9 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue. async function interruptedBootstrap() { const f = await seed(); await f.finish(); const run = await f.run(); const interrupted = await terminalizeLegacyExecution({ db, run, status: "interrupted", patch: { + // The real controller claim records preparation before terminal status + // records completion. The delivery fixture otherwise leaves both unset. + executionStage: "preparing", finishedAt: new Date(), errorCode: "server_shutdown_interrupted", runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, } }); expect(await getExecutionBlocker(db, f.companyId, f.sourceId)).toMatchObject({ runId: run.id }); @@ -338,8 +341,10 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue. expect(await settleInterruptedNativeBootstrap(db, { run, providerDispatchStarted: false })).not.toBeNull(); expect(await issueService(db).getById(f.sourceId)).toMatchObject({ status: "blocked" }); }); - it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted"])("retains an interrupted bootstrap hold for %s", async kind => { + it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted", "missing-stage", "unfinished"])("retains an interrupted bootstrap hold for %s", async kind => { const { f, run } = await interruptedBootstrap(); + if (kind === "missing-stage") await db.update(heartbeatRuns).set({ executionStage: null }).where(eq(heartbeatRuns.id, run.id)); + if (kind === "unfinished") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, run.id)); if (kind === "runtime-selected") await db.update(heartbeatRuns).set({ runtimeModeResolvedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); if (kind === "other-adapter") await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "process" } } }).where(eq(heartbeatRuns.id, run.id)); if (kind === "restore-unsafe") await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } }).where(eq(heartbeatRuns.id, run.id)); diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index e742a2133e6..88bb022617a 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -4,7 +4,7 @@ import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinaliza import { claimedAdapterType } from "./conversation-continuation.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; -import { canContinueCancelledRun } from "./run-cancellation.js"; +import { canContinueCancelledRun, readRunCancellation } from "./run-cancellation.js"; import { legacyControllerBootId } from "./legacy-controller-lease.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; @@ -37,14 +37,28 @@ export async function isFailedNativePreparation(db: Db, run: Run, coordinator: C return !lease && !result && !await nativeStartupHasExecutionEvidence(db, run); } +/** Candidate state only. Admission must also verify ownership, execution records, + * stopped leases, and any saved-file obligation below. */ +export function isInterruptedNativePreparationCandidate(run: Run): boolean { + const cancellation = readRunCancellation(run.resultJson); + // Older shutdown records predate structured cancellation. Current records + // identify unexpected system interruption without relying on an error label. + const systemInterruption = cancellation + ? cancellation.initiator.type === "system" && cancellation.source !== "operator" && + (cancellation.source === "shutdown" || cancellation.expected === false) + : run.errorCode === "server_shutdown_interrupted"; + const recovery = run.resultJson?.executionRecovery as Record | undefined; + return Boolean(systemInterruption && run.status === "interrupted" && run.finishedAt + && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && run.executionStage === "preparing" + && claimedAdapterType(run) === "paperclip_runner" && !run.processPid && !run.processGroupId + && !run.processStartedAt && !run.sessionIdAfter && !run.nativeIssueId && !run.nativeSessionId + && run.runnerProfileJson?.nativeExecutionInput == null && recovery?.providerWorkStarted !== true); +} + export async function isInterruptedNativePreparation(db: Db, run: Run): Promise { const recovery = run.resultJson?.executionRecovery as Record | undefined; const settled = recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; - if (run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || !run.finishedAt - || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || run.executionStage !== "preparing" - || claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId - || run.processStartedAt || run.sessionIdAfter || run.nativeIssueId || run.nativeSessionId - || run.runnerProfileJson?.nativeExecutionInput != null || recovery?.providerWorkStarted === true + if (!isInterruptedNativePreparationCandidate(run) || run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date() || !settled && (!run.controllerBootId || run.controllerBootId === legacyControllerBootId)) return false; const [coordinator] = await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and( diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index a6f80f26ba0..5d9d9e3b815 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -7,7 +7,8 @@ import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { randomUUID } from "node:crypto"; import { canonicalExecutionUuid } from "./execution-identity.js"; -import { claimedAdapterType, conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; +import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; +import { isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { persistActivity } from "./activity-log.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { logger } from "../middleware/logger.js"; @@ -612,8 +613,7 @@ export async function settleUnrecoverableExecutions( if (task.status !== "blocked" && run.runtimeMode === "native") { nativeFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion }; } - if (task.status !== "blocked" && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && - run.errorCode === "server_shutdown_interrupted" && claimedAdapterType(run) === "paperclip_runner") { + if (task.status !== "blocked" && isInterruptedNativePreparationCandidate(run)) { nativeBootstrapFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status }; } } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8215d222fc8..8bde92adcda 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3914,7 +3914,8 @@ export function heartbeatService( }); } } - if (latestRun?.status === "interrupted" && latestRun.errorCode === "server_shutdown_interrupted") { + if (latestRun?.status === "interrupted" && latestRun.runtimeMode === "legacy" && + !latestRun.runtimeModeResolvedAt && latestRun.executionStage === "preparing") { latestRun = await settleInterruptedNativeBootstrap(db, { run: latestRun, providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted || nativeOwnershipHeld, }) ?? latestRun; diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 2ea6de7c323..07bf4659789 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -429,10 +429,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } const now = opts?.now ?? new Date(); const preparationRecovery = parseObject(run.resultJson?.executionRecovery); + const interruptedPreparation = run.status === "interrupted" && + preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false; const retryReason = opts?.retryReason ?? - (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted" && - preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false - ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON); + (interruptedPreparation ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON); const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; const maxAttempts = Math.max( @@ -570,6 +570,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen : baseSchedule; const requiresIssueGate = + interruptedPreparation || (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") || isTransientWorkspacePreparationCode(run.errorCode) || hasConversationContinuationPolicy(run.resultJson) || diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index ead808b6623..461fcffb616 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -572,7 +572,8 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont async function resumeRemoteStopComments(run: NonNullable>>, requestId?: string) { if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; let interruptedPreparation = false; - if (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") { + if (run.status === "interrupted" && run.runtimeMode === "legacy" && + !run.runtimeModeResolvedAt && run.executionStage === "preparing") { const current = await getRun(run.id, { includeExecutionEvidence: true }); if (current && await isInterruptedNativePreparation(db, current)) { const settled = await settleInterruptedNativeBootstrap(db, { run: current, providerDispatchStarted: false }); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 68138666771..44bdebd1004 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -20,7 +20,7 @@ import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { logActivity } from "./activity-log.js"; import { isSupersededConversationRun } from "./agent-conversations.js"; import { issueService } from "./issues.js"; -import { isInterruptedNativePreparation } from "./cancelled-native-startup.js"; +import { isInterruptedNativePreparation, isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -318,10 +318,8 @@ export async function settleInterruptedNativeBootstrap( return db.transaction(async tx => { const [task] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, input.run.companyId))).for("update"); const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId))).for("update"); - if (!task || !run || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || + if (!task || !run || !isInterruptedNativePreparationCandidate(run) || run.contextSnapshot?.issueId !== task.id || - run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || - claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId || executionFailureRetryCount(run) >= 2) return null; if (hasWorkspaceRestoreFailure(run.resultJson) && !await hasVerifiedWorkspaceRepair(tx as unknown as Db, run)) return null; const [native] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)).limit(1); diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 7fa4babe6b5..b2d4f1c9155 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -393,16 +393,20 @@ describe("automatic retained workspace recovery", () => { return { ...f, source, workspaceId, resultId, git, commit }; } - async function interruptedRemotePreparation() { + async function interruptedRemotePreparation(interruption: "shutdown" | "lease" = "shutdown") { const f = await acceptedRemoteExport(); await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); await db.delete(nativeRunResults).where(eq(nativeRunResults.runId, f.runId)); const recovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] }; await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, completionContractId: null, - status: "interrupted", executionStage: "preparing", errorCode: "server_shutdown_interrupted", + status: "interrupted", executionStage: "preparing", + errorCode: interruption === "lease" ? "lease_released_before_terminal" : "server_shutdown_interrupted", controllerBootId: randomUUID(), - error: "Interrupted by graceful server shutdown (SIGTERM)", - resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery }, + error: interruption === "lease" ? "Environment lease released before preparation settled" : "Interrupted by graceful server shutdown (SIGTERM)", + resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery, + ...(interruption === "lease" ? { cancellation: { source: "unknown", expected: false, + initiator: { type: "system" }, reason: "Environment lease released before preparation settled", + recordedAt: new Date().toISOString() } } : {}) }, }).where(eq(heartbeatRuns.id, f.runId)); await db.update(issues).set({ statusVersion: 2 }).where(eq(issues.id, f.issueId)); await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation", @@ -416,8 +420,8 @@ describe("automatic retained workspace recovery", () => { return f; } - it("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace", async () => { - const f = await interruptedRemotePreparation(); + it.each(["shutdown", "lease"] as const)("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace (%s)", async interruption => { + const f = await interruptedRemotePreparation(interruption); await recoverRetainedWorkspace(f.input); expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true); const intent = { observedRunId: f.runId, idempotencyKey: randomUUID() }; @@ -441,8 +445,9 @@ describe("automatic retained workspace recovery", () => { finally { await saved.cleanup(); } }, 30_000); - it.each([false, true])("automatically retries interrupted preparation after preservation, including recovered history at startup=%s", async alreadyRecovered => { - const f = await interruptedRemotePreparation(); + it.each([false, true].flatMap(alreadyRecovered => (["shutdown", "lease"] as const).map(interruption => ({ alreadyRecovered, interruption }))))( + "automatically retries interrupted preparation after preservation ($interruption, startup=$alreadyRecovered)", async ({ alreadyRecovered, interruption }) => { + const f = await interruptedRemotePreparation(interruption); if (alreadyRecovered) await recoverRetainedWorkspace(f.input); const restarted = createDb(connectionString); try { @@ -462,6 +467,60 @@ describe("automatic retained workspace recovery", () => { } finally { await restarted.$client.end(); } }, 30_000); + it("retries unpublished preparation from its preserved ancestor without an old failure-block receipt", async () => { + const f = await seed("legacy", false); + await requestWorkspaceRecovery({ ...f, idempotencyKey: randomUUID(), actorId: "board" }); + await recoverRetainedWorkspace(f.input); + const ancestor = await run(f.runId), leaseId = randomUUID(), providerLeaseId = randomUUID(); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: ancestor.agentId, status: "running" }); + const app = boardRecoveryApp(), heartbeat = heartbeatService(db); + const first = (await request(app).post(`/api/issues/${f.issueId}/retry`).send({ + observedRunId: f.runId, idempotencyKey: randomUUID(), + }).expect(202)).body; + await heartbeat.resumeTaskRetry(f.companyId, first.requestId); + const admitted = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.status, "queued"))); + expect(admitted).toHaveLength(1); + const interruptedId = admitted[0].id; + await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "interrupted", executionStage: "preparing", + startedAt: new Date(Date.now() - 60_000), finishedAt: new Date(), controllerBootId: randomUUID(), + errorCode: "lease_released_before_terminal", error: "Lease released before preparation settled", + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: { cancellation: { source: "unknown", expected: false, initiator: { type: "system" }, + reason: "Lease released before preparation settled", recordedAt: new Date().toISOString() } }, + }).where(eq(heartbeatRuns.id, interruptedId)); + await db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId)); + const lease = { id: leaseId, companyId: f.companyId, heartbeatRunId: interruptedId, + provider: "daytona", providerLeaseId }; + await db.insert(environmentLeases).values({ ...lease, issueId: f.issueId, environmentId: f.lease.environmentId, + status: "released", cleanupStatus: "success", releasedAt: new Date(), + metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId, state: "stopped" }) }, + }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", ownerType: "board", returnOwnerAgentId: ancestor.agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${interruptedId}`, + status: "resolved", outcome: "blocked", nextAction: "Recorded actions require reconciliation", + evidence: { runId: interruptedId, originalFailureCode: "lease_released_before_terminal", + automaticRecovery: { policy: "preserve_without_replay_v1", runId: interruptedId, replay: "blocked", actionOutcome: "unknown" } }, + }); + const intent = { observedRunId: interruptedId, idempotencyKey: randomUUID() }; + const receipt = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body; + await Promise.all([heartbeat.resumeTaskRetry(f.companyId, receipt.requestId), + heartbeat.resumeTaskRetry(f.companyId, receipt.requestId)]); + const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.status, "queued"))); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot).toMatchObject({ previousRunId: interruptedId }); + expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(receipt.requestId); + const restored = path.join(root, randomUUID()); await mkdir(restored); + const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId, + sourceRunIds: [interruptedId], cwd: restored }); + expect(copies).toHaveLength(1); + expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + }, 30_000); + it.each(["local", "none"])("continues verified interrupted preparation with %s leases", async provider => { const f = await interruptedRemotePreparation(); await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, @@ -476,9 +535,10 @@ describe("automatic retained workspace recovery", () => { }, 30_000); it.each(["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop", + "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption", "retry_exhausted", "human_reblock", "paused", "reassigned"] as const)( "keeps the interrupted preparation continuation behind %s", async gate => { - const f = await interruptedRemotePreparation(); + const f = await interruptedRemotePreparation("lease"); await recoverRetainedWorkspace(f.input); const source = await run(f.runId); if (gate === "provider_started") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, @@ -492,6 +552,17 @@ describe("automatic retained workspace recovery", () => { remoteWorkspace: { ...(source.runnerProfileJson!.remoteWorkspace as Record), snapshotId: randomUUID() }, } }).where(eq(heartbeatRuns.id, f.runId)); if (gate === "operator_stop") await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "operator_interrupted" }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "operator_stop_metadata") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + cancellation: requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator" }, "Stop requested"), + } }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "unrecorded_interruption") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + cancellation: null, + } }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "missing_stop_receipt") { + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteExecutionTermination: null } }) + .where(eq(environmentLeases.id, f.lease.id)); + } if (gate === "retry_exhausted") await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 2 }).where(eq(heartbeatRuns.id, f.runId)); if (gate === "human_reblock") await db.update(issues).set({ statusVersion: 3 }).where(eq(issues.id, f.issueId)); if (gate === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, source.agentId)); @@ -500,7 +571,8 @@ describe("automatic retained workspace recovery", () => { await heartbeat.resumeRemoteStopComments(await run(f.runId)); expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(0); - if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop"].includes(gate)) { + if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop", + "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption"].includes(gate)) { const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() }); await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index ad1d8ab4f6c..dc8de2e9f1a 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -447,7 +447,7 @@ export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedO and ${agentWakeupRequests.status} = 'deferred_issue_execution' and ${agentWakeupRequests.requestedByActorType} = 'user' and ${agentWakeupRequests.payload}->>'issueId' = ${issueRecoveryActions.sourceIssueId}::text)`; const interruptedPreparation = sql`(${heartbeatRuns.runtimeMode} = 'legacy' - and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.errorCode} = 'server_shutdown_interrupted' + and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.runtimeModeResolvedAt} is null and ${heartbeatRuns.executionStage} = 'preparing' and ${heartbeatRuns.runnerProfileJson}#>>'{adapterDispatch,adapterType}' = 'paperclip_runner' and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked' From 985b175e832260231f857a871eece5dbce146833 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:55:49 -0600 Subject: [PATCH 11/28] test(e2e): verify saved agent instructions on the current QA harness (#18891) ## Thinking Path > - Paperclip lets users configure agents and verify their work. > - Daily QA must exercise saved instructions on the QA staging instance. > - A fresh owned agent can test an instruction change without changing a shared agent. > - The test must prove that saved instructions caused the next run's output. > - It must also prove that the original instructions were restored. > - This pull request brings the qualified settings journey onto the current shared registry layout. > - The benefit is a reusable daily QA candidate with independent evidence and fresh current-source checks. ## Linked Issues or Issue Description Refs #18788. This is the replacement review PR for the same settings journey after integration with the merged lifecycle registry layout in #18782. **What happened?** The settings journey passed its original staging qualification at source898ae8ab4e49d2408688088ce82e3316b91cc09c. The earlier PR's review workspace remains retained for file preservation. That workspace and its recovery evidence remain intact. **Expected behavior** Review the integrated settings source through this PR's own normal CI and fresh reviews. The journey saves instructions through ordinary UI controls, verifies their effect on one current native run, and restores the original bytes. **Steps to reproduce** Use the documented dedicated agent-settings-persistence campaign. The provider must supply a fresh owned agent and the agent-settings-write capability. Save the instruction edit, reload it, complete the ordinary assigned task, verify its independently saved result and current-run attribution, then restore and verify the original instruction entry. ## What Changed - Add the settings browser journey, its independent evidence grader, fixtures, retained evidence contract and focused controls from #18788. - Require one fresh dedicated agent. Select its exact configured instruction entry through the file tree. Confirm both board saves through persisted source and audit evidence. - Require one committed native run on the configured provider, model and environment. Keep the synthetic instruction reference out of the task request. - Restore the original instruction bytes through the same ordinary UI and verify the saved result after reload. - Normally merge actual lifecycle squash8771606fd87f69ed4057779e3dce3a9bca8088e6 once. Place only settings registrations at the documented distinct anchors. Preserve all eight dedicated source/oracle/fixture/docs blobs from qualified898 exactly. ## Verification - Current integrated head: 330 focused checks pass across eight settings, catalog, contract, observation, result/report and operations test files. - Current E2E types pass: `node scripts/typecheck-test-projects.mjs --low-memory --reuse-declarations tests/runner-e2e`. - Declaration reuse was bound to all11091 matching package/script/server/UI/root inputs, installed lock and TypeScript, plus all source and target generated-output hashes. No source or module-resolution links were copied. All copied output hashes remain unchanged after local checks. - Maintained product-journey discovery passes and includes both lifecycle and settings. The integrated catalog has22 entries, including the separate fresh-cloud onboarding case. - Static preservation checks pass for seven shared registry ASTs after only declared order changes. The settings dispatch body, required seven checks, one-run bound and all dedicated implementation/oracle blobs remain intact. - Historical staging evidence: [QA workflow41689186453](https://github.com/paperclipai/paperclip-cloud/actions/runs/41689186453) passed all seven settings checks and all five phases at source898ae8ab4e49d2408688088ce82e3316b91cc09c. This is the original qualification receipt. It is not a new run of this integrated head. - This PR must earn its own current-head full CI, fresh5/5 review, resolved threads and terminal passing normal check contexts. Those checks are pending. No prior review or check status was copied. - Full workspace checks and a new live QA/model run were not run locally. The coordinator owns current-source acceptance and daily-plan admission. ## Risks - The journey has one paid provider run and needs a fresh dedicated agent. It rejects a campaign that shares this agent with another paid journey. - Browser saves and restoration must persist before navigation. Wrong, missing, foreign or incomplete evidence fails the independent grader. Failed or active work is preserved for normal owned cleanup. - Registration and hash-input order changed during integration, so the definition/source identity changes. The historical QA receipt remains bound to its original source. - The retained earlier review workspace is still held for file preservation. This PR does not recover, reset, replace or destroy it, and submits no turn to it. - No product runtime, permission, model, shared agent, configuration, feature map, QA pin or TestOps-plan changes are included. ## Model Used - OpenAI GPT-6 (Codex), with reasoning, tool use and code execution. The host does not expose the exact deployment identifier or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub issue/PR references; historical QA workflow is linked as qualification evidence) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/evals.md | 2 + tests/runner-e2e/PRODUCT-JOURNEYS.md | 1 + tests/runner-e2e/README.md | 2 + tests/runner-e2e/catalog.test.ts | 1 + tests/runner-e2e/journey-features.ts | 1 + .../product-journeys/AGENT-SETTINGS.md | 64 +++++++ .../product-journeys/agent-flows.ts | 1 + .../agent-settings-evidence.test.ts | 102 ++++++++++ .../agent-settings-evidence.ts | 125 ++++++++++++ .../agent-settings-flow.test.ts | 180 ++++++++++++++++++ .../product-journeys/agent-settings-flow.ts | 174 +++++++++++++++++ .../agent-settings-test-fixture.ts | 40 ++++ tests/runner-e2e/product-journeys/catalog.ts | 3 + tests/runner-e2e/product-journeys/fixtures.ts | 2 +- tests/runner-e2e/product-journeys/launch.ts | 5 + .../product-journeys/success-contract.ts | 2 + tests/runner-e2e/product-journeys/types.ts | 1 + 17 files changed, 705 insertions(+), 1 deletion(-) create mode 100644 tests/runner-e2e/product-journeys/AGENT-SETTINGS.md create mode 100644 tests/runner-e2e/product-journeys/agent-settings-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/agent-settings-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/agent-settings-flow.test.ts create mode 100644 tests/runner-e2e/product-journeys/agent-settings-flow.ts create mode 100644 tests/runner-e2e/product-journeys/agent-settings-test-fixture.ts diff --git a/doc/evals.md b/doc/evals.md index 7d62a82f9f5..20f89339176 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -1,5 +1,7 @@ # Paperclip evaluation guide +The explicit deployment-targeted [saved agent instructions journey](../tests/runner-e2e/product-journeys/AGENT-SETTINGS.md) checks an owned agent’s ordinary UI save, persisted instruction bytes, one current-run output, and restoration. The task request omits its instruction-only reference. It reuses the existing Product E2E target and report path; local controls do not establish a staging pass. + The explicit-only [verified API artifact transfer](../tests/runner-e2e/README.md#verified-api-artifact-transfer) cell checks a native Codex agent on Daytona installing an authorized saved API response and delivering unchanged bytes. The oracle binds the source capture, diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 9bc8ca779ab..80c30828b23 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -43,6 +43,7 @@ never start a server or depend on a tenant filesystem. | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, observe an active native command with current-turn usage, Stop it, verify cancellation and settled accounting, then send one new direction and complete without original-run activity | | `followup-while-running` | Native agent; a maximum of two work runs | Save a follow-up while an independently observed native shell command is still active, then require one attributed final reply with the late requirement after reload; in-turn steering and one normal successor are accepted | | `hire-delegate-reuse` | Native lead with a fresh user chat and scoped hiring permission; Agent Chat enabled; direct hiring policy; seven work runs | Hire one teammate, delegate a saved checklist, reuse the same teammate for a saved review, verify native run and revision authorship, preserve the original, and follow the lead's saved result link after reload | +| `agent-settings-persistence` | Fresh dedicated native agent with the campaign ID in its name; `agent-settings-write`; no previous runs/proof block; sole paid journey; one work run | Save and reload an instruction edit through the browser, verify a task uses the instruction-only reference, reopen its run-authored document, then restore the original bytes. See [AGENT-SETTINGS.md](product-journeys/AGENT-SETTINGS.md) | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | The deployment-targeted Stop journey asks the revision to finish one harmless diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 49c359a300e..49181a8d3ec 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1,5 +1,7 @@ # Paid runner full-stack E2E +The deployment-targeted [saved agent instructions journey](product-journeys/AGENT-SETTINGS.md) verifies browser save/reload, one instruction-dependent native task, independently attributed output, and restoration of the owned agent’s original instructions. Discovery and local grader controls remain separate from QA staging qualification. + The explicit-only `cursor-cloud` suite uses a local controller and hosted Cursor Cloud work. Its four required cells check a downloaded registered artifact, question/answer continuation on the same Cloud agent, UI Stop of an diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 5f08f93b832..93833491ec8 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -147,6 +147,7 @@ describe("runner E2E catalog", () => { "stop-redirect", "followup-while-running", "hire-delegate-reuse", + "agent-settings-persistence", "cloud-onboarding" ]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 18cf44b1f33..ab33d9780cd 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -27,6 +27,7 @@ associate(["product-journeys"], { "question-answer-resume": ["questions-and-approvals/questions", "questions-and-approvals/continuation"], "followup-while-running": ["steering/queue"], "hire-delegate-reuse": ["agents/hire", "documents-artifacts/documents"], + "agent-settings-persistence": ["agents/instructions", "tasks/completion"], "project-task-context": ["projects/lifecycle", "projects/task-context", "tasks/create"], "task-edit-lifecycle": ["tasks/properties", "tasks/completion"], "task-properties": ["tasks/properties"], "task-document-persistence": ["documents-artifacts/documents"], "task-search-filter": ["tasks/organize"], "cloud-onboarding": ["onboarding/first-task"] }); diff --git a/tests/runner-e2e/product-journeys/AGENT-SETTINGS.md b/tests/runner-e2e/product-journeys/AGENT-SETTINGS.md new file mode 100644 index 00000000000..aa848f1daef --- /dev/null +++ b/tests/runner-e2e/product-journeys/AGENT-SETTINGS.md @@ -0,0 +1,64 @@ +# Saved agent instructions + +`agent-settings-persistence` verifies an ordinary instruction edit on an owned +native agent and the result of its next task. It has one provider work run and +requires the `agent-settings-write` target capability. The target provider grants +that capability only for a fresh disposable agent dedicated to this case. Its +exact name must match the prepared context and contain the current campaign ID. +It must have no previous work runs or QA settings proof block. The name is an +additional fixture binding; the target's capability grant establishes ownership. +Target configuration rejects campaigns that combine this case with another paid +journey before acquiring a target. Browser-only journeys may share the campaign. +The journey +does not edit the shared source agent, provider, model, accounts, or feature flags. + +1. Read the original managed instruction entry, exact bytes, content hash, and + content revision token. Require an empty work-run history. + Empty or whitespace-only files use the ordinary editor value because the + read-only page shows an empty-state message. Preserve those exact bytes too. +2. Open Instructions, select **Edit** for the entry, change the **Instruction + file editor**, and use **Save changes**. The new procedure contains a fresh + synthetic reference that does not occur in the task request. + Select the configured entry through the file tree on every open or reload, + expanding its parent folders. An existing `AGENTS.md` must not divert the edit + from a different configured entry. + Confirm persisted bytes and the new save audit before navigating away. The + same confirmation applies when restoring the original instructions. +3. Reload Instructions. Require the exact saved bytes in both the browser and + the normal read API. Verify a new `agent.files_updated` audit event with the + signed-in user, exact owned agent, entry path, and content hash. + Activity reads use both entity filters before the API limit. Unrelated company + history cannot consume that page. A full owned page or foreign entity fails + the coverage check rather than being silently filtered. +4. Create one ordinary task through the shared composer. The agent must use the + saved procedure to produce the `settings-proof` task document. Require the + exact reference, one committed native run on the configured provider, model, + and environment, and current-run document authorship. Pending questions, + checkout, or recovery cannot pass. + The dedicated `nativePhase` column must be `committed`. The grader does not + require `resultJson.finalizationPhase`, which normal bounded run reads can omit. +5. Reload the task and read the saved document. Verify the entry file is still + unchanged after the agent's task. +6. Restore the original bytes through the same Instructions controls. Verify the + restored browser/API content, its original content token, a distinct board + save audit, and no second task run. Return to the task result for the ordinary + authenticated screenshot. + +Current instruction revisions are content tokens. Identical restored bytes have +the original token; the journey does not invent a database parent/child revision +chain. The independently retained board audit establishes each actual save. + +The independent grader rejects wrong or missing settings, leaked task inputs, +foreign agents/tasks/runs/documents, a changed entry, unknown runtime identity, +uncommitted work, extra runs, and an unconfirmed restoration. A failed work run +remains failed. The public flow does not edit instructions during failed or active +work; the target's normal cleanup stops and terminates that owned agent. + +Private evidence consists of `agent-settings-observations.json` and +`observation-read-failures.json` under this case's `private-stories` directory. +The first contains closed IDs, hashes, the synthetic reference, checks, and both +matched board-save receipts. Each receipt includes its audit ID, actor, agent +entity, entry path, content hash, and save source. It +does not contain raw instruction files, account bindings, or provider payloads. +Discovery and local controls do not qualify a staging run. The daily plan may +admit this case only after a genuine staging pass and reviewed source merge. diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index 5986c9ca143..b73a4cda8d5 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -26,6 +26,7 @@ type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } diff --git a/tests/runner-e2e/product-journeys/agent-settings-evidence.test.ts b/tests/runner-e2e/product-journeys/agent-settings-evidence.test.ts new file mode 100644 index 00000000000..9fc728ea592 --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-evidence.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { AGENT_SETTINGS_CHECKS, gradeAgentSettings, settingsProofHeading, settingsTaskPrompt, settingsResultMatches } from "./agent-settings-evidence.js"; +import { settingsEvidenceFixture } from "./agent-settings-test-fixture.js"; +import { agentJourneyRunCountValid, journeyRequiredChecks } from "./success-contract.js"; +import { parseTargetConfig } from "./catalog.js"; + +describe("saved agent settings independent evidence", () => { + it.each([["ask-question", "agent-settings-persistence"], ["agent-settings-persistence", "ask-question"]])( + "rejects a shared paid campaign before target acquisition: %j", (...journeys) => { + expect(() => parseTargetConfig({ version: 1, id: "settings-target", mode: "managed-self-hosted", journeys, + executionEnvironment: "local", baseline: { features: {}, connections: [] }, + limits: { durationMs: 30000, modelBudgetCents: 500, maxRuns: 2 } })).toThrow("agent_settings_requires_dedicated_campaign"); + }); + it("permits browser-only journeys with the dedicated settings campaign", () => { + expect(parseTargetConfig({ version: 1, id: "settings-target", mode: "managed-self-hosted", + journeys: ["task-create", "agent-settings-persistence"], executionEnvironment: "local", + baseline: { features: {}, connections: [] }, limits: { durationMs: 30000, modelBudgetCents: 500, maxRuns: 1 } }).journeys) + .toEqual(["task-create", "agent-settings-persistence"]); + }); + it("requires every independent check and exactly one work run in retained results", () => { + expect(journeyRequiredChecks["agent-settings-persistence"]).toEqual(AGENT_SETTINGS_CHECKS); + expect(agentJourneyRunCountValid("agent-settings-persistence", 1)).toBe(true); + for (const count of [undefined, 0, 2, -1, 1.5, "1"]) { + expect(agentJourneyRunCountValid("agent-settings-persistence", count)).toBe(false); + } + }); + it("accepts the actual content-token contract, board save audits and one bound result", () => { + const f = settingsEvidenceFixture(); + expect(gradeAgentSettings(f.scope, f.evidence)).toEqual(AGENT_SETTINGS_CHECKS.map(id => ({ id, passed: true }))); + expect(f.evidence.restored!.file.revision.id).toBe(f.evidence.original!.file.revision.id); + expect(settingsTaskPrompt).not.toContain(f.scope.reference); + }); + it.each([null, {}, { result: "bounded output" }])("accepts committed current-run evidence when resultJson is %j", resultJson => { + const f = settingsEvidenceFixture(); + Object.assign(f.run, { resultJson }); + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(true); + }); + it("accepts committed current-run evidence when resultJson is omitted", () => { + const f = settingsEvidenceFixture(); + delete (f.run as Record).resultJson; + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(true); + }); + it("rejects an internally consistent baseline containing a previous QA proof block", () => { + const f = settingsEvidenceFixture(undefined, `# Agent\n${settingsProofHeading}\nStale previous reference\n`); + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(false); + }); + const changes: Array<[string, (f: ReturnType) => void]> = [ + ["foreign company", f => { f.evidence.original!.agent.companyId = "foreign"; }], + ["missing campaign-bearing target name", f => { f.scope.agentName = "Agent without campaign name"; f.evidence.original!.agent.name = f.scope.agentName; }], + ["past work", f => { f.evidence.original!.runs.push(f.run); }], + ["wrong instruction entry", f => { f.evidence.saved!.bundle.entryFile = "OTHER.md"; }], + ["false content hash", f => { f.evidence.saved!.file.contentHash = "0".repeat(64); }], + ["wrong saved bytes", f => { f.evidence.saved!.file.content += "wrong"; }], + ["missing save audit", f => { f.evidence.saved!.activity = []; }], + ["unscoped activity response", f => { f.evidence.original!.activity.push({ ...f.firstSave, entityId: "foreign-agent" }); }], + ["full owned activity page", f => { f.evidence.original!.activity = Array.from({ length: 200 }, (_, index) => ({ ...f.firstSave, id: `old-${index}` })); }], + ["foreign board actor", f => { f.firstSave.actorId = "someone-else"; }], + ["foreign audit entity", f => { f.firstSave.entityId = "other-agent"; }], + ["duplicate save evidence", f => { f.evidence.saved!.activity.push({ ...f.firstSave, id: "duplicate" }); }], + ["rendered draft only", f => { f.evidence.reloaded!.file.content = f.original; }], + ["missing rendered content", f => { delete f.evidence.reloaded!.renderedContent; }], + ["prompt leaks reference", f => { f.evidence.task!.issue.description += f.scope.reference; }], + ["foreign task", f => { f.evidence.task!.issue.companyId = "other"; }], + ["wrong assignee", f => { f.evidence.task!.issue.assigneeAgentId = "other"; }], + ["incomplete task", f => { f.evidence.task!.issue.status = "in_progress"; }], + ["active checkout", f => { f.evidence.task!.issue.executionRunId = "active"; }], + ["recovery pending", f => { f.evidence.task!.issue.activeRecoveryAction = {}; }], + ["question pending", f => { f.evidence.task!.interactions = [{ status: "pending" }]; }], + ["extra run", f => { f.evidence.task!.runs.push({ ...f.run, id: "extra" }); }], + ["failed run", f => { f.run.status = "failed"; }], + ["uncommitted run", f => { f.run.nativePhase = "applied"; }], + ["missing committed phase", f => { delete (f.run as Record).nativePhase; }], + ["uncommitted run with omitted result", f => { f.run.nativePhase = "applied"; Object.assign(f.run, { resultJson: null }); }], + ["legacy run", f => { f.run.runtimeMode = "legacy"; }], + ["conflicting issue bindings", f => { Object.assign(f.run, { issueId: "foreign" }); }], + ["wrong provider", f => { f.run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["wrong model", f => { f.run.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["wrong environment", f => { f.run.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["wrong nonce", f => { f.evidence.task!.document.body = JSON.stringify({ reference: "wrong", settingsSaved: true }); }], + ["human-authored output", f => { f.evidence.task!.document.createdByUserId = "user"; }], + ["foreign revision", f => { f.evidence.task!.revision.createdByRunId = "other"; }], + ["task changes instructions", f => { f.evidence.afterWork!.file.content = "changed"; }], + ["wrong reopened revision", f => { f.evidence.task!.reopenedDocument!.latestRevisionId = "old"; }], + ["missing rendered result", f => { delete f.evidence.task!.renderedReference; }], + ["unrestored instructions", f => { f.evidence.restored!.file.content = f.saved; }], + ["missing restore audit", f => { f.evidence.restored!.activity = [f.firstSave]; }], + ["extra work during restore", f => { f.evidence.restored!.runs.push({ ...f.run, id: "later" }); }], + ]; + it.each(changes)("rejects %s", (_name, change) => { + const f = settingsEvidenceFixture(); change(f); + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(false); + }); + it.each(["original", "saved", "reloaded", "afterWork", "restored", "task"] as const)("rejects missing %s evidence", key => { + const f = settingsEvidenceFixture(); delete f.evidence[key]; + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(false); + }); + it("rejects plausible malformed result bodies and accepts one optional JSON fence", () => { + expect(settingsResultMatches('```json\n{"reference":"r","settingsSaved":true}\n```', "r")).toBe(true); + for (const body of [null, "", "not json", "[]", '{"reference":"r"}', '{"reference":"r","settingsSaved":"true"}', + '{"reference":"r","settingsSaved":true,"extra":1}']) expect(settingsResultMatches(body, "r")).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/agent-settings-evidence.ts b/tests/runner-e2e/product-journeys/agent-settings-evidence.ts new file mode 100644 index 00000000000..9d1be4505e5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-evidence.ts @@ -0,0 +1,125 @@ +import { createHash } from "node:crypto"; + +type Row = Record; +export const AGENT_SETTINGS_CHECKS = [ + "agent-settings-browser-save", "agent-settings-reload-persisted", "agent-settings-instruction-only-input", + "agent-settings-result-attributed", "agent-settings-result-reopened", "agent-settings-original-restored", + "configured-model-and-environment", +] as const; +export const settingsDocumentKey = "settings-proof"; +export const settingsProofHeading = "## QA settings persistence proof"; +export const settingsTaskPrompt = "Follow the QA settings proof procedure in your saved agent instructions. Save the requested task document, then finish the task. No other tasks or agents are needed. Do not change your instruction files."; +export const instructionDigest = (content: string) => createHash("sha256").update(content).digest("hex"); +export function settingsInstructionContent(original: string, reference: string) { + return original + `\n\n${settingsProofHeading}\nWhen a task asks for the QA settings proof, save a task document with key ${settingsDocumentKey} and title Settings proof. Its body must be JSON with exactly two fields: reference equal to ${reference}, and settingsSaved equal to true. Keep these instructions unchanged.\n`; +} +export function settingsResultMatches(body: unknown, reference: string) { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const parsed = JSON.parse(json); + return parsed && !Array.isArray(parsed) && Object.keys(parsed).sort().join(",") === "reference,settingsSaved" + && parsed.reference === reference && parsed.settingsSaved === true; + } catch { return false; } +} +export interface AgentSettingsScope { + campaignId: string; companyId: string; userId: string; agentId: string; agentName: string; + provider: string; model: string; environmentId: string; reference: string; +} +export interface SettingsSnapshot { agent: Row; bundle: Row; file: Row; runs: Row[]; activity: Row[]; renderedContent?: string } +export interface SettingsTaskSnapshot { + issue: Row; runs: Row[]; interactions: Row[]; document: Row; revision: Row; + reopenedDocument?: Row; reopenedRevision?: Row; renderedReference?: string; +} +export interface AgentSettingsEvidence { + original?: SettingsSnapshot; saved?: SettingsSnapshot; reloaded?: SettingsSnapshot; + afterWork?: SettingsSnapshot; restored?: SettingsSnapshot; task?: SettingsTaskSnapshot; +} +const present = (value: unknown): value is string => typeof value === "string" && value.length > 0; +const revisionId = (point?: SettingsSnapshot) => point?.file?.revision?.id; +function snapshotBound(point: SettingsSnapshot | undefined, scope: AgentSettingsScope, entry?: string) { + return Boolean(point && point.agent?.id === scope.agentId && point.agent.companyId === scope.companyId + && point.agent.name === scope.agentName && point.agent.adapterType === "paperclip_runner" + && point.agent.adapterConfig?.provider === scope.provider && point.agent.adapterConfig?.model === scope.model + && point.bundle?.agentId === scope.agentId && point.bundle.companyId === scope.companyId + && point.bundle.mode === "managed" && point.bundle.editable === true && present(point.bundle.entryFile) + && (!entry || point.bundle.entryFile === entry) && point.file?.path === point.bundle.entryFile + && point.file.isEntryFile === true && point.file.editable === true && point.file.binary !== true + && typeof point.file.content === "string" && point.file.contentHash === instructionDigest(point.file.content) + && point.file.size === Buffer.byteLength(point.file.content) + && present(revisionId(point)) && point.file.revision.companyId === scope.companyId + && point.file.revision.agentId === scope.agentId && point.file.revision.entryFile === point.bundle.entryFile + && point.file.revision.contentHash === point.file.contentHash + && point.file.revision.byteLength === point.file.size && Array.isArray(point.runs) + && Array.isArray(point.activity) && point.activity.length < 200 + && point.activity.every(row => present(row.id) && row.companyId === scope.companyId + && row.entityType === "agent" && row.entityId === scope.agentId)); +} +export function matchedBoardSaveAudit(point: SettingsSnapshot | undefined, before: SettingsSnapshot | undefined, scope: AgentSettingsScope) { + const old = new Set(before?.activity?.map(row => row.id)); + const added = point?.activity?.filter(row => !old.has(row.id) && row.action === "agent.files_updated" + && row.entityType === "agent" && row.entityId === scope.agentId && row.details?.path === point.bundle.entryFile); + if (!(Array.isArray(added) && added.length === 1 && present(added[0]?.id) + && added[0].companyId === scope.companyId && added[0].actorType === "user" && added[0].actorId === scope.userId + && added[0].agentId === null && added[0].runId === null && added[0].details.contentHash === point!.file.contentHash + && added[0].details.source === "board")) return null; + const row = added[0]; + return { id: row.id, companyId: row.companyId, actorType: row.actorType, actorId: row.actorId, + agentId: row.agentId, runId: row.runId, action: row.action, entityType: row.entityType, entityId: row.entityId, + path: row.details.path, contentHash: row.details.contentHash, source: row.details.source }; +} +function boundIssue(run: Row) { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(value => value != null); + return ids.length && ids.every(present) && new Set(ids).size === 1 ? ids[0] : undefined; +} +export function gradeAgentSettings(scope: AgentSettingsScope, value: AgentSettingsEvidence) { + const { original, saved, reloaded, afterWork, restored, task } = value; + const entry = original?.bundle?.entryFile; + const scoped = [scope.campaignId, scope.companyId, scope.userId, scope.agentId, scope.agentName, + scope.provider, scope.model, scope.environmentId, scope.reference].every(present) + && scope.agentName.includes(scope.campaignId) && snapshotBound(original, scope) + && original!.runs.length === 0 && !original!.file.content.includes(scope.reference) + && !original!.file.content.includes(settingsProofHeading); + const expected = scoped ? settingsInstructionContent(original!.file.content, scope.reference) : undefined; + const save = scoped && snapshotBound(saved, scope, entry) && saved!.file.content === expected + && revisionId(saved) !== revisionId(original) && matchedBoardSaveAudit(saved, original, scope) && saved!.runs.length === 0; + const reload = save && snapshotBound(reloaded, scope, entry) && reloaded!.file.content === expected + && revisionId(reloaded) === revisionId(saved) && reloaded!.renderedContent === expected && reloaded!.runs.length === 0; + const issue = task?.issue, runs = task?.runs; + const inputOnly = reload && issue && present(issue.id) && issue.companyId === scope.companyId + && issue.assigneeAgentId === scope.agentId && issue.assigneeUserId === null && issue.createdByUserId === scope.userId + && issue.createdByAgentId === null && issue.workMode === "standard" && issue.description === settingsTaskPrompt + && !issue.description.includes(scope.reference); + const run = Array.isArray(runs) && runs.length === 1 ? runs[0] : undefined; + const provider = run?.runnerProfileJson?.nativeExecutionInput?.provider; + const configured = Boolean(inputOnly && run && present(run.id) && run.companyId === scope.companyId + && run.agentId === scope.agentId && boundIssue(run) === issue.id && run.status === "succeeded" + && run.runtimeMode === "native" && run.nativePhase === "committed" + && provider?.kind === scope.provider && provider.model === scope.model + && run.contextSnapshot?.paperclipEnvironment?.id === scope.environmentId); + const document = task?.document, revision = task?.revision; + const output = configured && issue && issue.status === "done" && !issue.executionRunId && !issue.scheduledRetry && !issue.activeRecoveryAction + && Array.isArray(task!.interactions) && task!.interactions.every(row => row.status !== "pending") + && document && present(document.id) && document.companyId === scope.companyId && document.issueId === issue.id + && document.key === settingsDocumentKey && document.createdByAgentId === scope.agentId && document.createdByUserId === null + && present(document.latestRevisionId) && settingsResultMatches(document.body, scope.reference) + && revision && revision.id === document.latestRevisionId && revision.documentId === document.id + && revision.createdByAgentId === scope.agentId && revision.createdByUserId === null && revision.createdByRunId === run!.id + && settingsResultMatches(revision.body, scope.reference) + && snapshotBound(afterWork, scope, entry) && afterWork!.file.content === expected && revisionId(afterWork) === revisionId(saved) + && afterWork!.runs.length === 1 && afterWork!.runs[0]?.id === run!.id; + const reopened = task?.reopenedDocument, reopenedRevision = task?.reopenedRevision; + const resultReload = output && issue && reopened && reopened.id === document.id && reopened.companyId === scope.companyId + && reopened.issueId === issue.id && reopened.key === settingsDocumentKey && reopened.latestRevisionId === document.latestRevisionId + && reopened.createdByAgentId === scope.agentId && reopened.createdByUserId === null && settingsResultMatches(reopened.body, scope.reference) + && reopenedRevision && reopenedRevision.id === revision.id && reopenedRevision.documentId === document.id + && reopenedRevision.createdByRunId === run!.id && reopenedRevision.createdByAgentId === scope.agentId + && reopenedRevision.createdByUserId === null && settingsResultMatches(reopenedRevision.body, scope.reference) + && task!.renderedReference === scope.reference; + const restore = resultReload && snapshotBound(restored, scope, entry) && restored!.file.content === original!.file.content + && restored!.renderedContent === original!.file.content && restored!.file.contentHash === original!.file.contentHash + && revisionId(restored) === revisionId(original) && revisionId(restored) !== revisionId(saved) && matchedBoardSaveAudit(restored, afterWork, scope) + && restored!.runs.length === 1 && restored!.runs[0]?.id === run!.id; + return [save, reload, inputOnly, output, resultReload, restore, configured] + .map((passed, index) => ({ id: AGENT_SETTINGS_CHECKS[index]!, passed: Boolean(passed) })); +} diff --git a/tests/runner-e2e/product-journeys/agent-settings-flow.test.ts b/tests/runner-e2e/product-journeys/agent-settings-flow.test.ts new file mode 100644 index 00000000000..4569b771fbf --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-flow.test.ts @@ -0,0 +1,180 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; +import { settingsEvidenceFixture } from "./agent-settings-test-fixture.js"; +import { settingsTaskPrompt } from "./agent-settings-evidence.js"; + +const controls = vi.hoisted(() => ({ createTask: vi.fn(), openDocument: vi.fn() })); +vi.mock("../user-actions.js", () => ({ createTaskThroughUi: controls.createTask })); +vi.mock("./task-document-flow.js", () => ({ openDocumentAuthoring: controls.openDocument })); +vi.mock("@playwright/test", () => ({ expect: (locator: any) => ({ + async toBeVisible() { expect(locator.visible).toBe(true); }, + async toBeEnabled() { expect(locator.enabled()).toBe(true); }, + async toBeDisabled() { expect(locator.enabled()).toBe(false); }, + async toHaveValue(value: string) { expect(locator.value()).toBe(value); }, + async toContainText(value: string) { expect(await locator.textContent()).toContain(value); }, +}) })); +import { runAgentSettings } from "./agent-settings-flow.js"; + +const directories: string[] = []; +afterEach(async () => { await Promise.all(directories.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); }); + +async function fixture(nativeFailure = false, unscopedActivity = false, delayedRestore = false, original?: string, entryFile = "AGENTS.md") { + let f = settingsEvidenceFixture(undefined, original, entryFile), content = f.original, rendered = content, draft = content, dirty = false, dispatched = false; + const originalContent = f.original; + const availablePaths = new Set(f.evidence.original!.bundle.files.map((file: any) => file.path)); + const neighbor = "# AGENTS.md sibling must remain unchanged\n"; + let selectedFile = "AGENTS.md"; + const expandedDirs = new Set(), selectedPaths: string[] = [], savedPaths: string[] = []; + let pendingContent: string | undefined; + const saves: string[] = [], owns: any[] = [], apiWrites: string[] = [], activityReads: string[] = []; + const unrelatedActivity = Array.from({ length: 600 }, (_, index) => ({ id: `unrelated-${index}`, + companyId: f.scope.companyId, entityType: "agent", entityId: "another-agent", details: {} })); + const current = () => content === f.original + ? dispatched ? f.evidence.restored! : f.evidence.original! + : dispatched ? f.evidence.afterWork! : f.evidence.saved!; + const editor = { visible: true, value: () => draft, async inputValue() { return draft; }, async fill(value: string) { draft = value; dirty = true; } }; + const raw = { get visible() { return Boolean(rendered.trim()); }, async textContent() { return rendered; } }; + const edit = { get visible() { return selectedFile === entryFile; }, async click() { draft = content; } }; + const save = { enabled: () => dirty, async click() { + expect(selectedFile).toBe(entryFile); savedPaths.push(selectedFile); + saves.push(draft); dirty = false; + if (delayedRestore && saves.length === 2) pendingContent = draft; + else content = draft; + const nonce = /reference equal to ([a-f0-9]+),/.exec(draft)?.[1]; + if (nonce) f = settingsEvidenceFixture(nonce, originalContent, entryFile); + } }; + const document = { visible: true, async textContent() { return f.evidence.task!.document.body; } }; + const page = { async goto() { + if (pendingContent !== undefined) throw new Error("navigation_before_save_committed"); + selectedFile = "AGENTS.md"; expandedDirs.clear(); rendered = entryFile === "AGENTS.md" ? content : neighbor; + }, async reload() { selectedFile = "AGENTS.md"; expandedDirs.clear(); rendered = entryFile === "AGENTS.md" ? content : neighbor; }, + getByTestId(id: string) { expect(id).toBe("instructions-raw-source"); return raw; }, + getByRole(role: string, options: { name: string }) { + if (role === "button" && options.name === `Edit ${entryFile}`) return edit; + if (role === "button" && options.name === `Stop editing ${entryFile}`) return { async click() {} }; + if (role === "button" && options.name === "Save changes") return save; + if (role === "textbox" && options.name === "Instruction file editor") return editor; + throw new Error("unexpected_locator"); + }, locator(selector: string) { + if (selector === "#document-settings-proof") return document; + const match = /^\[role="treeitem"\]\[data-file-tree-path=("(?:\\.|[^"\\])*")\]$/.exec(selector); + if (!match) throw new Error("unexpected_selector:" + selector); + const filePath: string = JSON.parse(match[1]!); + const directory = entryFile.startsWith(`${filePath}/`); + const parents = filePath.split("/").slice(0, -1); + return { get visible() { return (directory || availablePaths.has(filePath)) + && parents.every((_, index) => expandedDirs.has(parents.slice(0, index + 1).join("/"))); }, + async getAttribute(name: string) { expect(name).toBe("aria-expanded"); return directory ? String(expandedDirs.has(filePath)) : null; }, + async click() { + if (directory) expandedDirs.add(filePath); + else { selectedFile = filePath; selectedPaths.push(filePath); rendered = filePath === entryFile ? content : neighbor; } + } }; + } }; + const api = { origin: "https://qa.example.test", async get(route: string) { + if (route === "/api/agents/agent") return structuredClone(current().agent); + if (route === "/api/agents/agent/instructions-bundle") return structuredClone(current().bundle); + if (route === `/api/agents/agent/instructions-bundle/file?path=${encodeURIComponent(entryFile)}`) { + if (pendingContent !== undefined) { content = pendingContent; pendingContent = undefined; } + return structuredClone(current().file); + } + if (route.includes("/heartbeat-runs?agentId=agent")) return structuredClone(current().runs); + if (route.startsWith("/api/companies/company/activity?")) { + activityReads.push(route); + const query = new URL(route, api.origin).searchParams; + expect(query.get("entityType")).toBe("agent"); expect(query.get("entityId")).toBe("agent"); + expect(query.get("limit")).toBe("200"); + if (unscopedActivity) return [unrelatedActivity[0]]; + return structuredClone([...unrelatedActivity, ...current().activity] + .filter(row => row.entityType === query.get("entityType") && row.entityId === query.get("entityId")) + .slice(0, Number(query.get("limit"))).map(row => ({ ...row, + details: { ...row.details, privateFixturePayload: "must-not-be-retained" } }))); + } + if (route === "/api/issues/task") return structuredClone(f.evidence.task!.issue); + if (route === "/api/issues/task/runs") return [{ runId: "run" }]; + if (route === "/api/issues/task/interactions") return []; + if (route === "/api/heartbeat-runs/run") return { ...structuredClone(f.run), ...(nativeFailure ? { status: "failed" } : {}) }; + if (route === "/api/issues/task/documents/settings-proof") return structuredClone(f.evidence.task!.document); + if (route === "/api/issues/task/documents/settings-proof/revisions/document-revision") return structuredClone(f.evidence.task!.revision); + throw new Error("unexpected_read:" + route); + }, async put() { apiWrites.push("put"); throw new Error("tested save must use browser"); }, + async patch() { apiWrites.push("patch"); throw new Error("tested save must use browser"); } }; + const context = { page, api, company: { id: "company", issuePrefix: "QA" }, user: { id: "user", name: "QA" }, + agent: { id: "agent", name: "Owned agent campaign-1", provider: "opencode", model: "model", runtimeMode: "native" }, + environmentId: "environment", capabilities: ["company", "board-user", "task-write", "agent", "agent-settings-write"] } as unknown as JourneyContext; + controls.createTask.mockReset(); controls.openDocument.mockReset(); + controls.createTask.mockImplementation(async (input: any) => { + expect(saves).toHaveLength(1); expect(input.prompt).toBe(settingsTaskPrompt); + expect(input.prompt).not.toContain(f.scope.reference); + await input.beforeDispatch(); dispatched = true; return { issueId: "task" }; + }); + const session = { async own(resource: any) { owns.push(resource); } } as TargetSession; + const evidence: JourneyEvidence = { outcome: "passed", startingState: "recipe", phases: [], checks: [], resourceDisposition: "retained", privacy: "private" }; + const directory = await mkdtemp(path.join(os.tmpdir(), "qa-settings-flow-")); directories.push(directory); + return { context, session, evidence, directory, saves, savedPaths, selectedPaths, availablePaths, owns, apiWrites, activityReads, unrelatedActivity, original: f.original, + currentContent: () => content, options: { directory, deadlineAt: Date.now() + 30000 } }; +} + +describe("agent settings production-surface orchestration", () => { + it.each(["WORKFLOW.md", "team guide/qa [settings].md", "team/nested/INSTRUCTIONS.md"])( + "selects configured entry %s when AGENTS.md also exists and is initially selected", async entryFile => { + const f = await fixture(false, false, false, undefined, entryFile); + expect(f.availablePaths).toEqual(new Set(["AGENTS.md", entryFile])); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.evidence.checks.every(row => row.passed)).toBe(true); + expect(f.savedPaths).toEqual([entryFile, entryFile]); + expect(f.selectedPaths.length).toBeGreaterThan(2); expect(new Set(f.selectedPaths)).toEqual(new Set([entryFile])); + expect(f.currentContent()).toBe(f.original); expect(f.apiWrites).toEqual([]); + const receipt = JSON.parse(await readFile(path.join(f.directory, "agent-settings-observations.json"), "utf8")); + expect(receipt.saveAudits.saved.path).toBe(entryFile); expect(receipt.saveAudits.restored.path).toBe(entryFile); + }); + it.each(["", " \n\t\n"])("reads and restores the exact empty-file editor value %j", async original => { + const f = await fixture(false, false, false, original); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.saves).toHaveLength(2); expect(f.saves[1]).toBe(original); expect(f.currentContent()).toBe(original); + expect(f.evidence.checks.every(row => row.passed)).toBe(true); + }); + it("saves and restores only through browser controls, then leaves the result ready for screenshot", async () => { + const f = await fixture(); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.evidence.checks).toHaveLength(7); expect(f.evidence.checks.every(row => row.passed)).toBe(true); + expect(f.saves).toHaveLength(2); expect(f.saves[1]).toBe(f.original); expect(f.apiWrites).toEqual([]); + expect(f.owns.map(row => row.id ?? "intent")).toEqual(["intent", "task"]); + const artifact = await readFile(path.join(f.directory, "agent-settings-observations.json"), "utf8"); + expect(artifact).not.toContain(f.original); expect(artifact).not.toContain("adapterConfig"); + expect(artifact).not.toContain("must-not-be-retained"); + const saved = JSON.parse(artifact); expect(saved.checks.every((row: any) => row.passed)).toBe(true); + expect(saved.checkpoints.original.contentHash).toBe(saved.checkpoints.restored.contentHash); + expect(saved.saveAudits).toEqual({ saved: { id: "save", companyId: "company", actorType: "user", actorId: "user", + agentId: null, runId: null, action: "agent.files_updated", entityType: "agent", entityId: "agent", path: "AGENTS.md", + contentHash: saved.checkpoints.saved.contentHash, source: "board" }, restored: { id: "restore", companyId: "company", + actorType: "user", actorId: "user", agentId: null, runId: null, action: "agent.files_updated", entityType: "agent", + entityId: "agent", path: "AGENTS.md", contentHash: saved.checkpoints.restored.contentHash, source: "board" } }); + expect(f.unrelatedActivity).toHaveLength(600); expect(f.activityReads.length).toBeGreaterThan(0); + }); + it("rejects a wrongly unscoped activity response before any settings edit", async () => { + const f = await fixture(false, true); + await expect(runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options)).rejects.toThrow("agent_settings_read_coverage_unavailable"); + expect(f.saves).toEqual([]); expect(f.owns).toEqual([]); + }); + it("confirms a delayed restore audit and bytes before navigating away from the editor", async () => { + const f = await fixture(false, false, true); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.saves).toHaveLength(2); expect(f.currentContent()).toBe(f.original); + expect(f.evidence.checks.every(row => row.passed)).toBe(true); + }); + it("requires scoped settings-write capability before any configuration edit", async () => { + const f = await fixture(); f.context.capabilities = ["company", "board-user", "task-write", "agent"]; + await expect(runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options)).rejects.toThrow("owned_agent_settings_capability_required"); + expect(f.saves).toEqual([]); expect(f.owns).toEqual([]); + }); + it("preserves a native failure and does not restore instructions while failed work may require cleanup", async () => { + const f = await fixture(true); + await expect(runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options)).rejects.toThrow("agent_settings_native_run_failed"); + expect(f.saves).toHaveLength(1); expect(f.currentContent()).not.toBe(f.original); + expect(JSON.parse(await readFile(path.join(f.directory, "agent-settings-observations.json"), "utf8")).checks.every((row: any) => row.passed)).toBe(false); + expect(f.apiWrites).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/agent-settings-flow.ts b/tests/runner-e2e/product-journeys/agent-settings-flow.ts new file mode 100644 index 00000000000..5480ebc163c --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-flow.ts @@ -0,0 +1,174 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi } from "../user-actions.js"; +import { waitFor } from "./agent-flows.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { AGENT_SETTINGS_CHECKS, gradeAgentSettings, instructionDigest, matchedBoardSaveAudit, settingsInstructionContent, + settingsDocumentKey, settingsProofHeading, settingsTaskPrompt, type AgentSettingsEvidence, type AgentSettingsScope, + type SettingsSnapshot } from "./agent-settings-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "agent-settings-persistence"; + +export async function runAgentSettings(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId || !context.capabilities.includes("agent-settings-write")) { + throw new JourneyError("unsupported", "owned_agent_settings_capability_required"); + } + const scope: AgentSettingsScope = { campaignId, companyId: context.company.id, userId: context.user.id, + agentId: agent.id, agentName: agent.name, provider: agent.provider, model: agent.model, + environmentId: context.environmentId, reference: randomBytes(16).toString("hex") }; + const observations: AgentSettingsEvidence = {}; + const read = createObservedReader(context.api, options.deadlineAt), agentRoute = `/api/agents/${agent.id}`; + const agentRunsRoute = `/api/companies/${scope.companyId}/heartbeat-runs?agentId=${agent.id}&limit=100`; + const activityRoute = `/api/companies/${scope.companyId}/activity?entityType=agent&entityId=${agent.id}&limit=200`; + const snapshot = async (): Promise => { + const [current, bundle, runs, activity] = await Promise.all([ + read.get(agentRoute), read.get(`${agentRoute}/instructions-bundle`), + read.get(agentRunsRoute), read.get(activityRoute), + ]); + if (typeof bundle?.entryFile !== "string" || !bundle.entryFile || !Array.isArray(runs) || runs.length >= 100 + || !Array.isArray(activity) || activity.length >= 200 + || activity.some(row => row.companyId !== scope.companyId || row.entityType !== "agent" || row.entityId !== agent.id)) { + throw new JourneyError("failed", "agent_settings_read_coverage_unavailable"); + } + const file = await read.get(`${agentRoute}/instructions-bundle/file?path=${encodeURIComponent(bundle.entryFile)}`); + return { agent: current, bundle, file, runs, activity }; + }; + const openInstructions = async (entry: string, expectedContent: string, reload = false) => { + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/agents/${agent.id}/instructions`, { waitUntil: "domcontentloaded" }); + if (reload) await context.page.reload(); + const segments = entry.split("/"); + const treeRow = (filePath: string) => context.page.locator(`[role="treeitem"][data-file-tree-path=${JSON.stringify(filePath)}]`); + for (let depth = 1; depth < segments.length; depth++) { + const directory = treeRow(segments.slice(0, depth).join("/")); + await expect(directory).toBeVisible(); + if (await directory.getAttribute("aria-expanded") !== "true") await directory.click(); + } + const entryRow = treeRow(entry); + await expect(entryRow).toBeVisible(); + await entryRow.click(); + const edit = context.page.getByRole("button", { name: `Edit ${entry}`, exact: true }); + await expect(edit).toBeVisible(); + if (!expectedContent.trim()) { + await edit.click(); + const editor = context.page.getByRole("textbox", { name: "Instruction file editor", exact: true }); + await expect(editor).toHaveValue(expectedContent); + const renderedContent = await editor.inputValue(); + await context.page.getByRole("button", { name: `Stop editing ${entry}`, exact: true }).click(); + return { edit, renderedContent }; + } + const raw = context.page.getByTestId("instructions-raw-source"); + await expect(raw).toBeVisible(); + const renderedContent = await raw.textContent(); + if (renderedContent !== expectedContent) throw new JourneyError("failed", "agent_settings_rendered_source_mismatch"); + return { edit, renderedContent }; + }; + const saveThroughUi = async (beforePoint: SettingsSnapshot, content: string) => { + const entry = beforePoint.bundle.entryFile, before = beforePoint.file.content; + const { edit } = await openInstructions(entry, before); + await edit.click(); + const editor = context.page.getByRole("textbox", { name: "Instruction file editor", exact: true }); + await expect(editor).toHaveValue(before); + await editor.fill(content); + const save = context.page.getByRole("button", { name: "Save changes", exact: true }); + await expect(save).toBeEnabled(); + await save.click(); + await expect(save).toBeDisabled({ timeout: 30_000 }); + // Disabled can also mean saving. Confirm persisted bytes and the new audit + // before navigating, including during restoration. + return waitFor(options.deadlineAt, snapshot, point => point.file.content === content + && Boolean(matchedBoardSaveAudit(point, beforePoint, scope)), "agent_settings_save_not_persisted"); + }; + const check = (name: typeof AGENT_SETTINGS_CHECKS[number]) => { + const result = gradeAgentSettings(scope, observations).find(row => row.id === name)!; + evidence.checks.push(result); + if (!result.passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + let failed = false; + try { + observations.original = await snapshot(); + const original = observations.original, entry = original.bundle.entryFile; + if (original.agent.id !== agent.id || original.agent.companyId !== scope.companyId || original.agent.name !== agent.name + || !agent.name.includes(campaignId) || original.agent.adapterType !== "paperclip_runner" + || original.agent.adapterConfig?.provider !== agent.provider || original.agent.adapterConfig?.model !== agent.model + || original.bundle.mode !== "managed" || original.bundle.editable !== true || original.file.editable !== true + || typeof original.file.content !== "string" || original.file.content.includes("\r") + || original.file.content.includes(settingsProofHeading) || original.runs.length !== 0) { + throw new JourneyError("failed", "agent_settings_fixture_not_fresh_owned"); + } + const edited = settingsInstructionContent(original.file.content, scope.reference); + observations.saved = await saveThroughUi(original, edited); + check("agent-settings-browser-save"); + const reopenedSource = await openInstructions(entry, edited, true); + observations.reloaded = { ...await snapshot(), renderedContent: reopenedSource.renderedContent }; + check("agent-settings-reload-persisted"); + + const title = `Journey QA agent settings ${campaignId}`; + const task = await createTaskThroughUi({ page: context.page, origin: context.api.origin, issuePrefix: context.company.issuePrefix, + agentName: agent.name, title, prompt: settingsTaskPrompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: task.issueId, journeyId: id }); + const route = `/api/issues/${task.issueId}`, documentRoute = `${route}/documents/${settingsDocumentKey}`; + const state = await waitFor(options.deadlineAt, async () => { + const [issue, summaries, interactions] = await Promise.all([ + read.get(route), read.get(`${route}/runs`), read.get(`${route}/interactions`), + ]); + if (!Array.isArray(summaries) || summaries.length > 1 || !Array.isArray(interactions)) throw new JourneyError("failed", "agent_settings_run_coverage_mismatch"); + const runs = await Promise.all(summaries.map(row => read.get(`/api/heartbeat-runs/${row.runId}`))); + if (runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) throw new JourneyError("failed", "agent_settings_native_run_failed"); + return { issue, runs, interactions }; + }, point => point.issue.status === "done" && point.runs.length === 1 && point.runs[0]!.status === "succeeded" + && point.runs[0]!.nativePhase === "committed" && point.interactions.every(row => row.status !== "pending"), "agent_settings_task_not_completed"); + const document = await read.get(documentRoute); + const revision = await read.get(`${documentRoute}/revisions/${document.latestRevisionId}`); + observations.task = { ...state, document, revision }; + observations.afterWork = await snapshot(); + check("agent-settings-instruction-only-input"); + check("configured-model-and-environment"); + check("agent-settings-result-attributed"); + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/issues/${state.issue.identifier ?? task.issueId}`, { waitUntil: "domcontentloaded" }); + await context.page.reload(); + await openDocumentAuthoring(context); + const displayed = context.page.locator(`#document-${settingsDocumentKey}`); + await expect(displayed).toContainText(scope.reference); + observations.task.reopenedDocument = await read.get(documentRoute); + observations.task.reopenedRevision = await read.get(`${documentRoute}/revisions/${document.latestRevisionId}`); + observations.task.renderedReference = (await displayed.textContent())?.includes(scope.reference) ? scope.reference : undefined; + check("agent-settings-result-reopened"); + + await saveThroughUi(observations.afterWork, original.file.content); + const restoredSource = await openInstructions(entry, original.file.content); + observations.restored = { ...await snapshot(), renderedContent: restoredSource.renderedContent }; + check("agent-settings-original-restored"); + // The shared screenshot policy permits only the task result, not settings. + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/issues/${state.issue.identifier ?? task.issueId}`, { waitUntil: "domcontentloaded" }); + await openDocumentAuthoring(context); + await expect(context.page.locator(`#document-${settingsDocumentKey}`)).toContainText(scope.reference); + } catch (error) { failed = true; throw error; } + finally { + try { + await read.preserve(options.directory); + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + await writeFile(path.join(options.directory, "agent-settings-observations.json"), JSON.stringify({ + companyId: scope.companyId, agentId: scope.agentId, reference: scope.reference, + checkpoints: Object.fromEntries(["original", "saved", "reloaded", "afterWork", "restored"].map(key => { + const point = observations[key as Exclude]; + return [key, point ? { entryFile: point.file.path, contentHash: point.file.contentHash, + independentlyComputedHash: typeof point.file.content === "string" ? instructionDigest(point.file.content) : null, + revisionId: point.file.revision?.id ?? null, runIds: point.runs.map(row => row.id ?? row.runId) } : null]; + })), + issueId: observations.task?.issue.id ?? null, runIds: observations.task?.runs.map(row => row.id) ?? [], + documentId: observations.task?.document.id ?? null, documentRevisionId: observations.task?.revision.id ?? null, + saveAudits: { saved: matchedBoardSaveAudit(observations.saved, observations.original, scope), + restored: matchedBoardSaveAudit(observations.restored, observations.afterWork, scope) }, + checks: gradeAgentSettings(scope, observations), + }, null, 2), { mode: 0o600 }); + } catch (error) { if (!failed) throw error; } + } +} diff --git a/tests/runner-e2e/product-journeys/agent-settings-test-fixture.ts b/tests/runner-e2e/product-journeys/agent-settings-test-fixture.ts new file mode 100644 index 00000000000..1467eeb3db7 --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-test-fixture.ts @@ -0,0 +1,40 @@ +import { instructionDigest, settingsInstructionContent, settingsDocumentKey, settingsTaskPrompt, type AgentSettingsEvidence, type AgentSettingsScope } from "./agent-settings-evidence.js"; + +export function settingsEvidenceFixture(reference = "1a2b3c4d5e6f", original = "# Ordinary instructions\nFollow the assigned task.\n", entryFile = "AGENTS.md") { + const scope: AgentSettingsScope = { campaignId: "campaign-1", companyId: "company", userId: "user", agentId: "agent", + agentName: "Owned agent campaign-1", environmentId: "environment", provider: "opencode", model: "model", reference }; + const saved = settingsInstructionContent(original, scope.reference); + const run = { id: "run", companyId: scope.companyId, agentId: scope.agentId, nativeIssueId: "task", status: "succeeded", + runtimeMode: "native", nativePhase: "committed", resultJson: { finalizationPhase: "committed" }, + contextSnapshot: { issueId: "task", paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } } }; + const snapshot = (content: string, runs: any[] = [], activity: any[] = []) => { + const hash = instructionDigest(content); + return { agent: { id: scope.agentId, companyId: scope.companyId, name: scope.agentName, adapterType: "paperclip_runner", + adapterConfig: { provider: scope.provider, model: scope.model } }, + bundle: { agentId: scope.agentId, companyId: scope.companyId, entryFile, mode: "managed", editable: true, + files: [{ path: entryFile, isEntryFile: true, editable: true }, + ...(entryFile !== "AGENTS.md" ? [{ path: "AGENTS.md", isEntryFile: false, editable: true }] : [])] }, + file: { path: entryFile, content, contentHash: hash, size: Buffer.byteLength(content), isEntryFile: true, editable: true, + revision: { id: `agent-files:${hash}`, companyId: scope.companyId, agentId: scope.agentId, entryFile, + contentHash: hash, byteLength: Buffer.byteLength(content), parentRevisionId: null, baseRevisionId: null, source: "api" } }, + runs, activity, renderedContent: content }; + }; + const audit = (id: string, content: string) => ({ id, companyId: scope.companyId, actorType: "user", actorId: scope.userId, + agentId: null, runId: null, action: "agent.files_updated", entityType: "agent", entityId: scope.agentId, + details: { path: entryFile, contentHash: instructionDigest(content), source: "board" } }); + const firstSave = audit("save", saved), restore = audit("restore", original); + const body = JSON.stringify({ reference: scope.reference, settingsSaved: true }); + const document = { id: "document", companyId: scope.companyId, issueId: "task", key: settingsDocumentKey, + body, latestRevisionId: "document-revision", createdByAgentId: scope.agentId, createdByUserId: null }; + const revision = { id: "document-revision", documentId: document.id, body, createdByRunId: run.id, + createdByAgentId: scope.agentId, createdByUserId: null }; + const evidence: AgentSettingsEvidence = { + original: snapshot(original), saved: snapshot(saved, [], [firstSave]), reloaded: snapshot(saved, [], [firstSave]), + afterWork: snapshot(saved, [run], [firstSave]), restored: snapshot(original, [run], [firstSave, restore]), + task: { issue: { id: "task", companyId: scope.companyId, assigneeAgentId: scope.agentId, assigneeUserId: null, + createdByUserId: scope.userId, createdByAgentId: null, workMode: "standard", description: settingsTaskPrompt, status: "done" }, + runs: [run], interactions: [], document, revision, reopenedDocument: { ...document }, reopenedRevision: { ...revision }, renderedReference: scope.reference }, + }; + return { scope, evidence, original, saved, run, firstSave, restore }; +} diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 7de9ae41786..fe7351e2e45 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -22,6 +22,7 @@ export const productJourneys = [ { id: "stop-redirect", label: "Stop active work and send a new direction once", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/stop-and-pause"], paid: true }, { id: "followup-while-running", label: "Send a follow-up during active work and verify its saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/queue"], paid: true }, { id: "hire-delegate-reuse", label: "Hire, delegate and reuse one teammate", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["agents/hire", "documents-artifacts/documents"], paid: true }, + { id: "agent-settings-persistence", label: "Save a fresh dedicated agent's instructions and verify its first task", requires: ["company", "board-user", "task-write", "agent", "agent-settings-write"], capabilities: ["agents/instructions", "tasks/completion"], paid: true }, { id: "cloud-onboarding", label: "Create a Cloud workspace and complete a first task", requires: ["cloud-account", "fresh-workspace", "provider-key"], capabilities: ["onboarding/first-task"], paid: true }, ] as const; @@ -57,6 +58,8 @@ export function parseTargetConfig(value: unknown): TargetConfig { if (c.adapterModule !== undefined && (typeof c.adapterModule !== "string" || !c.adapterModule)) throw new Error("invalid_adapter_module"); if (c.adapterConfig !== undefined && !object(c.adapterConfig)) throw new Error("invalid_adapter_configuration"); if (c.modelProfileId !== undefined && !["runner-codex", "runner-codex-mini"].includes(c.modelProfileId)) throw new Error("unsupported_model_profile"); + if (c.journeys.includes("agent-settings-persistence") && c.journeys.some(id => id !== "agent-settings-persistence" + && productJourneys.find(journey => journey.id === id)!.paid)) throw new Error("agent_settings_requires_dedicated_campaign"); if (c.mode === "new-cloud" && c.journeys[0] !== "cloud-onboarding") throw new Error("new_cloud_requires_onboarding_first"); if (c.mode === "new-cloud" && c.baseline.connections.length) throw new Error("onboarding_requires_ui_connection_setup"); if (c.mode !== "new-cloud" && c.journeys.includes("cloud-onboarding")) throw new Error("onboarding_requires_fresh_cloud_workspace"); diff --git a/tests/runner-e2e/product-journeys/fixtures.ts b/tests/runner-e2e/product-journeys/fixtures.ts index 3f33df99bac..de4b3d2e055 100644 --- a/tests/runner-e2e/product-journeys/fixtures.ts +++ b/tests/runner-e2e/product-journeys/fixtures.ts @@ -11,7 +11,7 @@ export function journeyFixtures(session: TargetSession, campaignId?: string) { id: "ready-context", async setup() { return ready ??= await (preparation ??= session.prepare()); }, }); - for (const id of ["company", "board-user", "task-write", "agent", "skill-write", "review-service", "inbox-author", "company-access"]) registry.register({ + for (const id of ["company", "board-user", "task-write", "agent", "agent-settings-write", "skill-write", "review-service", "inbox-author", "company-access"]) registry.register({ id, dependencies: ["ready-context"], async setup(resolved) { const context = resolved.get("ready-context") as JourneyContext; diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 705946b8503..a1bd122db02 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -21,6 +21,7 @@ import { runClarificationJourney } from "./clarification-flow.js"; import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; import { runInputFile } from "./input-file-flow.js"; +import { runAgentSettings } from "./agent-settings-flow.js"; import { companyAccessIsolation } from "./company-access-flow.js"; import { inboxTriage } from "./inbox-triage-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; @@ -150,6 +151,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", + "agent-settings-flow.ts", + "agent-settings-evidence.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", @@ -278,6 +281,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.runtimeMode = ready.agent.runtimeMode; if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "agent-settings-persistence") await runAgentSettings(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "routine-scheduled-run") await runScheduledRoutine(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "file-input-answer") await runInputFile(ready, session!, campaignId, evidence, diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 91b14865b2e..d5d9a15b874 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -10,6 +10,7 @@ export const agentJourneyRuns = { "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, + "agent-settings-persistence": 1, "question-answer-resume": 3, "hire-delegate-reuse": 7 } as const; @@ -45,5 +46,6 @@ export const journeyRequiredChecks: Record = { "stopped-run-no-later-events", "stop-preserved-download", "redirect-consumed-once", "user-reply-stored-once", "configured-model-and-environment"], "followup-while-running": ["followup-native-command-active", "followup-stored-once", "followup-consumed-once", "followup-final-answer-persisted", "followup-one-or-two-native-runs", "configured-model-and-environment"], "hire-delegate-reuse": ["one-attributed-native-hire", "first-delegated-task-complete", "checklist-authored-by-hired-run", "same-teammate-reused", "review-authored-by-same-hired-run", "original-checklist-preserved", "hire-user-messages-stored-once", "delegated-result-visible-after-reload", "configured-model-and-environment"], + "agent-settings-persistence": ["agent-settings-browser-save", "agent-settings-reload-persisted", "agent-settings-instruction-only-input", "agent-settings-result-attributed", "agent-settings-result-reopened", "agent-settings-original-restored", "configured-model-and-environment"], "cloud-onboarding": ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"], }; diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 34dbf8ff913..eb79fd03411 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -19,6 +19,7 @@ export type JourneyId = | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" + | "agent-settings-persistence" | "file-input-answer" | "inbox-triage" | "routine-scheduled-run" From ad5cd785b1b5e3be1ac5c2d4736a328bd2ca9478 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Tue, 3 Nov 2026 16:57:44 -0800 Subject: [PATCH 12/28] fix(wake-queue): keep promoted Retry runs scoped to their issue (#18876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The wake queue starts the next run for a task when the task's current run releases its execution lock. > - A board Retry saves a receipt with only `{ issueId, taskRetry }`. This receipt has no saved wake context. > - When the release drain promotes that receipt, the promoted context gets `taskKey` from the payload, but it does not get `issueId` or `taskId`. > - The new run takes the task's execution lock, but the database saves it as a company-scoped run with no issue. > - Low-trust workspace preparation then has no issue reference. The run fails before provider work with "Low-trust execution issue is outside the active trust boundary." > - This pull request restores the issue scope of a payload-only receipt during promotion. > - The benefit is that a Retry on a low-trust task runs in its task boundary and does not fail during setup. ## Linked Issues or Issue Description **What happened?** A user clicked Retry on a low-trust task while a run was busy. When the busy run ended, the release drain promoted the saved Retry receipt. The promoted run had `issue_id = NULL`, `scope_kind = company`, and a context snapshot with only `taskKey`, `wakeReason: task_retry`, `wakeSource`, and `wakeTriggerDetail`. Workspace preparation failed with `low_trust_boundary_mismatch` (HTTP 422) in the setup phase. **Expected behavior** A promoted run for a task must keep that task's issue scope. This is true also when the deferred receipt has no saved wake context. Workspace preparation must then evaluate the task's own trust boundary. **Steps to reproduce** 1. Use a task with a low-trust execution policy whose assignee is an agent. 2. While a run on that task is busy, click Retry on the task. 3. Let the busy run end. The release drain promotes the deferred Retry receipt. 4. The promoted run is company-scoped with no `issueId`. It fails in setup with `low_trust_boundary_mismatch`. **Paperclip version or commit** `master` at 51d1d4ce07abee9807d546b865e3c74fd600d31a. **Agent adapter(s) involved** Not adapter-specific (core bug). ## What Changed - `enrichPromotedWakeContext` (`server/src/modules/wake-queue/domain/context.ts`) now restores `issueId` and `taskId` from the payload when the deferred context does not have them. This is the same rule that the original wake enrichment uses. The function does not overwrite a value that the queued context already has. - I added a domain test for the payload-only receipt and for the no-overwrite rule. - I added a Postgres adapter test (`adapters/promoted-retry-scope.test.ts`). It runs the real release drain with a Retry receipt that has no saved wake context. It checks that the promoted run is issue-scoped and holds the task's execution lock. The database trigger `set_heartbeat_run_scope_kind` derives `issue_id` and `scope_kind` from the restored context `issueId`. ## Verification - Red before the fix: both new tests fail on unchanged `master`. The adapter test shows the incident shape: `issueId: null`, `scopeKind: "company"`, and `taskKey` equal to the issue UUID. - Green after the fix: - `pnpm vitest run server/src/modules/wake-queue/` passes (10 files, 289 tests). - The related suites pass (11 files, 468 tests): `heartbeat/queue`, `heartbeat/run-control`, `heartbeat/run-lifecycle`, `deferred-execution-receipt-work`, `execution-lock-orphan-cleanup`, `heartbeat-comment-wake-batching`, `issue-stale-execution-lock-routes`, `chat-failed-run-retry-boundary`, `task-retry`, `execution-control-work`, and `workspace-recovery`. - Server typecheck (`node server/scripts/typecheck-layers.mjs`) passes. - I did not run a live low-trust Retry on a deployed stack. The adapter test uses the real release drain, the real database trigger, and the persisted receipt shape that `requestTaskRetry` writes. ## Risks - Low risk. The change only adds `issueId` and `taskId` when they are missing and the payload has them. Deferred wakes that already have a queued context do not change. - The release drain still promotes a board Retry receipt directly. It does not go through `resumeTaskRetry`. This PR does not change that path. A promoted Retry therefore does not get `forceFreshSession` or `taskRetryRequestId`. This behavior existed before this PR. It is a possible follow-up, not part of this fix. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), used through Claude Code with tool use (code search, shell, and test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../adapters/promoted-retry-scope.test.ts | 60 +++++++++++++++++++ .../modules/wake-queue/domain/context.test.ts | 22 +++++++ .../src/modules/wake-queue/domain/context.ts | 15 ++++- 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 server/src/modules/wake-queue/adapters/promoted-retry-scope.test.ts diff --git a/server/src/modules/wake-queue/adapters/promoted-retry-scope.test.ts b/server/src/modules/wake-queue/adapters/promoted-retry-scope.test.ts new file mode 100644 index 00000000000..f695fd05839 --- /dev/null +++ b/server/src/modules/wake-queue/adapters/promoted-retry-scope.test.ts @@ -0,0 +1,60 @@ +import { agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { createReleaseIssueExecution } from "../application/use-cases.js"; +import { createPostgresWakeQueueAdapter } from "./postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +// A board Retry receipt stores only `{ issueId, taskRetry }`, with no saved +// wake context. Its promoted run takes the issue's execution lock, so it must +// stay issue-scoped for workspace and trust-boundary preparation. +describe.skipIf(!support.supported)("promoted Retry receipt scope", () => { + let database: Awaited>; + let db: ReturnType; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("wake-promoted-retry-scope-"); + db = createDb(database.connectionString); + }, 60_000); + + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + + it("keeps a promoted Retry receipt without saved wake context scoped to its issue", async () => { + const [company] = await db.insert(companies).values({ name: "Retry scope fixture", issuePrefix: "RSF", + defaultResponsibleUserId: "responsible-user" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Retry agent", adapterType: "process", + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Retry scope fixture issue", + status: "in_review", priority: "medium", assigneeAgentId: agent.id }).returning(); + const [finished] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + invocationSource: "on_demand", status: "succeeded", contextSnapshot: { issueId: issue.id } }).returning(); + await db.update(issues).set({ executionRunId: finished.id }).where(eq(issues.id, issue.id)); + const [wake] = await db.insert(agentWakeupRequests).values({ + companyId: company.id, agentId: agent.id, source: "on_demand", triggerDetail: "manual", reason: "task_retry", + status: "deferred_issue_execution", requestedByActorType: "user", requestedByActorId: "responsible-user", + payload: { issueId: issue.id, + taskRetry: { schema: "paperclip.task-retry.v1", sourceRunId: finished.id, actorId: "responsible-user" } }, + }).returning(); + const release = createReleaseIssueExecution({ + issueLock: createPostgresWakeQueueAdapter(db, { + resolveResponsibleUserId: async () => "responsible-user", + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + }), + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); + + const result = await release({ companyId: company.id, runId: finished.id, now: new Date(), suppressImmediateRecovery: true }); + + expect(result.outcome.kind).toBe("promoted"); + const [promoted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.wakeupRequestId, wake.id)); + expect(promoted).toMatchObject({ scopeKind: "issue", issueId: issue.id, + contextSnapshot: { issueId: issue.id, taskId: issue.id, taskKey: issue.id, wakeReason: "task_retry" } }); + expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0].executionRunId).toBe(promoted.id); + }); +}); diff --git a/server/src/modules/wake-queue/domain/context.test.ts b/server/src/modules/wake-queue/domain/context.test.ts index d4fe457f716..9dc44b8c099 100644 --- a/server/src/modules/wake-queue/domain/context.test.ts +++ b/server/src/modules/wake-queue/domain/context.test.ts @@ -121,4 +121,26 @@ describe("enrichPromotedWakeContext", () => { expect(result.contextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" }); expect(result.contextSnapshot.workspaceRefreshReason).toBe("accepted_plan_confirmation"); }); + + it("restores the issue scope of a payload-only receipt without overwriting a queued context", () => { + const payloadOnly = enrichPromotedWakeContext({ + contextSnapshot: {}, + reason: "task_retry", + source: "on_demand", + triggerDetail: "manual", + payload: { issueId: "issue-1", taskRetry: { schema: "paperclip.task-retry.v1" } }, + }); + expect(payloadOnly.contextSnapshot).toMatchObject({ + issueId: "issue-1", taskId: "issue-1", taskKey: "issue-1", wakeReason: "task_retry", + }); + + const queued = enrichPromotedWakeContext({ + contextSnapshot: { issueId: "issue-1", taskId: "task-1", taskKey: "TASK-1" }, + reason: "issue_commented", + source: "automation", + triggerDetail: null, + payload: { issueId: "issue-2" }, + }); + expect(queued.contextSnapshot).toMatchObject({ issueId: "issue-1", taskId: "task-1", taskKey: "TASK-1" }); + }); }); diff --git a/server/src/modules/wake-queue/domain/context.ts b/server/src/modules/wake-queue/domain/context.ts index 4da2346fa63..a4f0df74e52 100644 --- a/server/src/modules/wake-queue/domain/context.ts +++ b/server/src/modules/wake-queue/domain/context.ts @@ -115,15 +115,16 @@ export type EnrichPromotedWakeContextResult = { /** * Fills the promoted run's context snapshot with the fields the original * wake enrichment always fills, without overwriting a field the deferred - * wake already carried. The deferred wake's own context snapshot was - * already enriched once when it was first queued, so most calls only - * normalize the interaction-continuation keys. + * wake already carried. A deferred wake queued through the wake queue was + * already enriched once, so most calls only normalize the + * interaction-continuation keys. */ export function enrichPromotedWakeContext( input: EnrichPromotedWakeContextInput, ): EnrichPromotedWakeContextResult { const contextSnapshot = { ...input.contextSnapshot }; const { reason, source, triggerDetail, payload } = input; + const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]) ?? readNonEmptyString(payload?.["taskId"]); const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]); const taskKey = deriveTaskKey(contextSnapshot, payload); const wakeCommentIds = mergeWakeCommentIds(contextSnapshot, commentIdFromPayload); @@ -131,6 +132,14 @@ export function enrichPromotedWakeContext( if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) { contextSnapshot.wakeReason = reason; } + // Some durable receipts, such as a board Retry, store only the payload and + // never had a queued context snapshot. Keep their issue scope. + if (!readNonEmptyString(contextSnapshot["issueId"]) && issueIdFromPayload) { + contextSnapshot.issueId = issueIdFromPayload; + } + if (!readNonEmptyString(contextSnapshot["taskId"]) && issueIdFromPayload) { + contextSnapshot.taskId = issueIdFromPayload; + } if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) { contextSnapshot.taskKey = taskKey; } From fbb7055b9020e505cd1bf903b1b1c301ba8f3eef Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:58:31 -0600 Subject: [PATCH 13/28] fix(runner): keep activity FIFOs out of Codex writable roots (#18886) ## Thinking Path > - Paperclip manages agents and their work. > - Runner owns each provider process and its activity channel. > - Codex Plan mode uses a restricted filesystem policy. > - Runner adds its activity FIFO as a writable filesystem root. > - Codex adds protected metadata paths beneath each writable root. > - A FIFO cannot contain those paths, so the Linux helper fails before useful work starts. > - This pull request grants access through the private activity directory and keeps discovery files read-only. ## Linked Issues or Issue Description Refs #18050. The activity channel came from that change. No existing FIFO-mount repair was found. #18853 repairs interrupted preparation recovery; it does not change this provider filesystem policy. **What happened?** A native Codex Plan session can fail while loading AGENTS.md instructions. The filesystem helper tries to inspect `activity.fifo/.aws` and fails with `Not a directory`. This happens before the provider can produce the requested plan. **Expected behavior** The sandbox must load permitted instructions and allow activity-channel writes. Plan mode must still reject workspace writes and private state access. **Steps to reproduce** Use the ignored `read_only_codex_opens_with_runtime_activity` regression in the Runner Rust suite with the source-pinned Linux Codex executable and its matching bubblewrap resource. The fixture calls the actual activity channel producer and policy projection. On the original producer it exits 101 with the FIFO metadata-mount panic. With this change it exits 0. **Paperclip version or commit** Original producer reproduced at 208bda1a0ea36b0500e86606eedc32fc1244aa03. The local native artifact is official Codex 0.160.0 for Linux arm64, which matches the source pin. The failing hosted executable version was not independently available from its normal run records. **Deployment mode** Native Runner with Linux Codex. Verification used an isolated local Linux container with no network, credentials, or model call. ## What Changed - Grant writes to the Runner-created private activity-channel directory instead of using its FIFO as a filesystem root. - Keep the lock, descriptor, and helper read-only. Keep the existing network, workspace, environment, and active-run authority policies. - Add actual channel isolation coverage and a pinned executable regression. The latter also checks FIFO bytes, instruction reads, workspace write denial, private-state and synthetic credential-home denial, descriptor and lock write denial, lock removal denial, and ambient-variable filtering. - Document the native sandbox check and the separate fixture-export path. Export alone does not verify the sandbox. ## Verification - Actual producer plus official Linux Codex 0.160.0 and matching bubblewrap: original producer exits 101 with the FIFO `.aws` panic; changed producer exits 0 with all permission checks and exact FIFO output verified. - `cargo test --manifest-path packages/paperclip-runner/runner/Cargo.toml runtime_notification --lib --offline`: 8 passed, 2 manual sandbox fixtures ignored. - `cargo test --manifest-path packages/paperclip-runner/runner/Cargo.toml codex_provider::tests --lib --offline`: 30 passed. - `cargo build --manifest-path packages/paperclip-runner/runner/Cargo.toml --bin paperclip-runnerd --offline`: passed. - Cargo format and diff checks passed. - A broader optional local Rust library sweep did not pass in the restricted macOS host. Three filesystem-monitor tests failed and another did not finish; the owned sweep was bounded and stopped. A socket-identity fixture also failed with EPERM. A representative monitor failure and the socket failure were repeated with the unchanged original producer. Their failures are retained and are not claimed fixed by this PR. - The existing ignored `publisher_crosses_real_codex_tool_boundary_without_runner_state_access` fixture was not executed successfully on the final head. Its stale channel-directory denial was corrected after Greptile review. The separate actual-producer Linux sandbox fixture verifies exact FIFO bytes and permission denials; it does not call `record-activity`. - Full local pnpm typecheck/test/build was not run for this Rust-only change. Required technical CI passes on fd4da0bf5ef2ea7dfc3da566567444ca3e87d1a2. Greptile is 5/5. The automatic Paperclip Review remains queued; merge is held for that review. No hosted task recovery or provider turn is claimed. ## Risks The writable grant now covers the process-private channel directory. The descriptor remains outside that grant. The lock has an explicit read-only mount inside it. Native sandbox checks verify the lock cannot be written or removed. Existing tests retain authority rotation, stale-context rejection, malformed input bounds, and fixed-path cleanup after endpoint replacement. The retained startup error also mentions a loopback permission error. This change fixes the independently reproduced FIFO-root defect. It does not prove that every hosted namespace capability is healthy. Preconfigured-home and custom policies retain their existing explicit deny rules. ## Model Used OpenAI GPT-6.1 Sol (`gpt-6.1-sol`) in Codex, with source editing, terminal execution, and tool use. The host did not report its context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- packages/paperclip-runner/README.md | 23 +++ .../crates/runner-core/src/codex_provider.rs | 16 +- .../runner-core/src/runtime_notifications.rs | 5 +- .../tests/codex_sandbox.rs | 157 ++++++++++++++++++ 4 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications/tests/codex_sandbox.rs diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index a7e77abe79b..13c354a89c0 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -726,6 +726,29 @@ actual shell discovery, successful durable activity recording, and denial of unr Runner state, descriptor writes and ambient secrets. The normal suite covers bounded framing, concurrent publishers, fixed-path cleanup and attachment fencing. +Codex receives write access to Runner's private activity-channel directory, with +the lock file explicitly read-only. The FIFO itself must not be a writable +filesystem root: Codex adds protected metadata children to writable roots, which +cannot exist beneath a FIFO. The helper and discovery descriptor remain read-only; +the activity grant does not expose Runner state or change workspace/network policy. + +To check restrictive Plan mode without credentials or a model call, run the +actual channel/policy producer against the pinned Linux Codex sandbox: + +```sh +CODEX_NOTIFICATION_SANDBOX_COMMAND=/absolute/path/to/pinned/codex \ +CODEX_NOTIFICATION_SANDBOX_RESOURCES=/absolute/path/to/pinned/executable/resources \ +cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --lib \ + read_only_codex_opens_with_runtime_activity -- --ignored +``` + +This checks instruction reads, FIFO writes, and denial of workspace writes, +unrelated Runner-state reads, descriptor writes, and lock writes/removal. An +isolated non-Linux host can instead set `CODEX_NOTIFICATION_FIXTURE_EXPORT` to a +JSON file and omit `CODEX_NOTIFICATION_SANDBOX_COMMAND`; that only exports a +synthetic producer fixture for a separately controlled Linux sandbox. Exporting +the fixture does not execute or verify the sandbox. + ## Direct live protocol qualification The canonical direct live protocol suite lives in the separate diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index fefedb0501a..1687b1ad23a 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -61,8 +61,13 @@ pub(crate) fn runtime_notification_codex_args( let Some(filesystem) = config["filesystem"].as_inline_table_mut() else { continue; }; - for (path, mode) in paths.iter().zip(["read", "read", "read", "write"]) { - filesystem.insert(path.to_string_lossy().as_ref(), mode.into()); + let Some(channel_directory) = paths[3].parent().filter(|path| path.parent().is_some()) + else { + continue; + }; + filesystem.insert(channel_directory.to_string_lossy().as_ref(), "write".into()); + for path in &paths[..3] { + filesystem.insert(path.to_string_lossy().as_ref(), "read".into()); } extra.extend(["-c".into(), format!("{prefix}{filesystem}")]); } @@ -4522,8 +4527,8 @@ mod tests { let paths = [ PathBuf::from("/runtime/runnerd"), PathBuf::from("/state/private/runtime-notifications.json"), - PathBuf::from("/state/private/runtime-activity.lock"), - PathBuf::from("/tmp/prn-channel"), + PathBuf::from("/tmp/prn-channel/activity.lock"), + PathBuf::from("/tmp/prn-channel/activity.fifo"), ]; let extra = runtime_notification_codex_args(&args, "paperclip-runner-workspace-only", &paths); @@ -4562,7 +4567,7 @@ mod tests { ("/state/private", "none"), ("/runtime/runnerd", "read"), ("/state/private/runtime-notifications.json", "read"), - ("/state/private/runtime-activity.lock", "read"), + ("/tmp/prn-channel/activity.lock", "read"), ("/tmp/prn-channel", "write"), ] { assert_eq!( @@ -4570,6 +4575,7 @@ mod tests { Some(mode) ); } + assert!(filesystem.get("/tmp/prn-channel/activity.fifo").is_none()); } assert!(!serialized.contains("network.")); assert!(!serialized.contains("PAPERCLIP_RUN_ID")); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs index d1722519fb1..5b05061290c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs @@ -666,6 +666,8 @@ mod tests { assert_eq!(fs::read(nested.join("protected")).unwrap(), b"unchanged"); fs::remove_dir_all(pipe.parent().unwrap()).unwrap(); } + include!("runtime_notifications/tests/codex_sandbox.rs"); + #[test] #[ignore = "requires CODEX_NOTIFICATION_SANDBOX_COMMAND and PAPERCLIP_RUNTIME_NOTIFICATION_HELPER"] fn publisher_crosses_real_codex_tool_boundary_without_runner_state_access() { @@ -709,7 +711,8 @@ mod tests { if cat "$1" >/dev/null 2>&1; then exit 71; fi if /bin/sh -c ': >> "$PAPERCLIP_RUNTIME_NOTIFICATION_CONTEXT"' 2>/dev/null; then exit 72; fi if touch "$2/probe" 2>/dev/null; then exit 75; fi -if touch "$3/probe" 2>/dev/null; then exit 76; fi +touch "$3/probe" || exit 76 +rm "$3/probe" || exit 76 if /bin/sh -c ': >> "$1"' ignored "$4" 2>/dev/null; then exit 77; fi context=$("$PAPERCLIP_RUNTIME_NOTIFICATION_HELPER" notification-context) || exit 73 "$PAPERCLIP_RUNTIME_NOTIFICATION_HELPER" record-activity "$context" example.runtime desktop.used || exit 74 diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications/tests/codex_sandbox.rs b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications/tests/codex_sandbox.rs new file mode 100644 index 00000000000..1638b7d84c5 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications/tests/codex_sandbox.rs @@ -0,0 +1,157 @@ +#[test] +fn codex_activity_grant_uses_only_its_own_private_channel_directory() { + let (_directory, config, _store, _state, server) = setup(); + let (_other_directory, _other_config, _other_store, _other_state, other) = setup(); + let channel = server.pipe.0.parent().unwrap(); + let other_channel = other.pipe.0.parent().unwrap(); + assert_ne!(channel, other_channel); + verify_private_directory(channel).unwrap(); + assert!(fs::symlink_metadata(&server.pipe.0) + .unwrap() + .file_type() + .is_fifo()); + let paths = [ + PathBuf::from("/runtime/runnerd"), + context_path(&config.state_dir), + server.pipe.0.with_extension("lock"), + server.pipe.0.clone(), + ]; + let profile = "paperclip-runner-workspace-read-only"; + let args = vec![ + "shell_environment_policy.include_only=[]".into(), + format!("permissions.{profile}.filesystem={{\":root\"=\"none\"}}"), + ]; + let projected = crate::codex_provider::runtime_notification_codex_args(&args, profile, &paths); + let prefix = format!("permissions.{profile}.filesystem="); + let value = projected + .iter() + .find_map(|arg| arg.strip_prefix(&prefix)) + .unwrap(); + let config = format!("filesystem={value}") + .parse::() + .unwrap(); + let filesystem = config["filesystem"].as_inline_table().unwrap(); + assert_eq!( + filesystem + .get(channel.to_str().unwrap()) + .and_then(toml_edit::Value::as_str), + Some("write") + ); + assert!(filesystem.get(other_channel.to_str().unwrap()).is_none()); + assert!(filesystem + .get(paths[1].parent().unwrap().to_str().unwrap()) + .is_none()); +} + +#[test] +#[ignore = "requires source-pinned Linux CODEX_NOTIFICATION_SANDBOX_COMMAND or CODEX_NOTIFICATION_FIXTURE_EXPORT"] +fn read_only_codex_opens_with_runtime_activity_without_widening_file_access() { + use std::process::Command; + + let directory = TestDirectory::create(&std::env::temp_dir()).unwrap(); + let config = config(directory.directory.join("state")); + fs::DirBuilder::new() + .mode(0o700) + .create(&config.state_dir) + .unwrap(); + let server = RuntimeNotifications::bind(&config.state_dir).unwrap(); + let workspace = directory.directory.join("workspace"); + let home = directory.directory.join("home"); + fs::create_dir(&workspace).unwrap(); + fs::create_dir(&home).unwrap(); + fs::write( + home.join("credential-fixture"), + b"synthetic private credential", + ) + .unwrap(); + fs::write( + workspace.join("AGENTS.md"), + b"Read-only fixture instructions.\n", + ) + .unwrap(); + fs::write( + config.state_dir.join("private-state"), + b"must remain inaccessible", + ) + .unwrap(); + let helper = config.state_dir.join("helper"); + fs::write(&helper, b"synthetic executable resource\n").unwrap(); + let paths = [ + helper, + context_path(&config.state_dir), + server.pipe.0.with_extension("lock"), + server.pipe.0.clone(), + ]; + let profile = "paperclip-runner-workspace-read-only"; + let resources = std::env::var("CODEX_NOTIFICATION_SANDBOX_RESOURCES") + .expect("set the source-pinned Codex executable resource directory"); + let resources = serde_json::to_string(&resources).unwrap(); + let args = vec![ + "-c".into(), + "shell_environment_policy.include_only=[\"PATH\"]".into(), + "-c".into(), + format!("permissions.{profile}.filesystem={{\":root\"=\"none\",\":minimal\"=\"read\",\":tmpdir\"=\"none\",{resources}=\"read\",\"{}\"=\"none\",\"{}\"=\"read\"}}", home.display(), workspace.display()), + "-c".into(), + format!("permissions.{profile}.network.enabled=false"), + ]; + let extra = crate::codex_provider::runtime_notification_codex_args(&args, profile, &paths); + let script = r#" +set -eu +test -z "${UNRELATED_SECRET:-}" +test "$(cat "$4/AGENTS.md")" = 'Read-only fixture instructions.' +if touch "$4/forbidden" 2>/dev/null; then exit 71; fi +if cat "$1/private-state" >/dev/null 2>&1; then exit 72; fi +if sh -c ': >> "$1"' ignored "$2" 2>/dev/null; then exit 73; fi +if sh -c ': >> "$1"' ignored "$3" 2>/dev/null; then exit 74; fi +if rm "$3" 2>/dev/null; then exit 75; fi +if cat "$6/credential-fixture" >/dev/null 2>&1; then exit 76; fi +cat "$2" >/dev/null +cat "$3" >/dev/null +printf 'synthetic notification frame\n' > "$5" +"#; + let mut args = args; + args.extend(extra); + args.extend(["sandbox", "-P", profile, "-C"].map(str::to_owned)); + args.push(workspace.to_string_lossy().into_owned()); + args.extend(["--", "/bin/sh", "-c", script, "notification-sandbox"].map(str::to_owned)); + args.extend( + [ + &config.state_dir, + &paths[1], + &paths[2], + &workspace, + &paths[3], + &home, + ] + .map(|path| path.to_string_lossy().into_owned()), + ); + if let Some(path) = std::env::var_os("CODEX_NOTIFICATION_FIXTURE_EXPORT") { + fs::write( + path, + serde_json::to_vec_pretty(&json!({ + "args": args, "stateDirectory": config.state_dir, + "pipe": server.pipe.0, "home": home, "workspace": workspace, + "descriptor": fs::read_to_string(&paths[1]).unwrap(), + })) + .unwrap(), + ) + .unwrap(); + return; + } + let command = std::env::var_os("CODEX_NOTIFICATION_SANDBOX_COMMAND").unwrap(); + let output = Command::new(command) + .args(args) + .current_dir(&workspace) + .env_clear() + .env("PATH", std::env::var_os("PATH").unwrap()) + .env("HOME", &home) + .env("CODEX_HOME", &home) + .env("UNRELATED_SECRET", "must-not-reach-command") + .output() + .unwrap(); + assert!( + output.status.success(), + "source-pinned sandbox failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} From d7283e875772ca8c684dc86e471e6b48d543b7b1 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:20:34 -0600 Subject: [PATCH 14/28] test(e2e): cover chat reload and fresh sessions (#18784) ## Thinking Path > - Paperclip keeps agent work attached to saved company records. > - Agent Chat keeps one conversation for each person and agent. > - Reload must preserve context without starting another turn. > - The ordinary `/new` command must create fresh provider context and preserve history. > - Existing local chat journeys cover these behaviors, but the deployment-targeted daily catalog does not. > - This PR adds one reusable journey with independent saved-state and rendered-reply checks. ## Linked Issues or Issue Description Refs: #2242, #18655. **What existing behavior does this improve?** Product E2E coverage for Agent Chat persistence and session reset on an attached deployment. **Current behavior** The deployment-targeted catalog has no bounded check for recall after reopening and a fresh session in the same saved conversation. **Proposed behavior** Remember a random phrase, reopen and reload the chat, then ask for the phrase without supplying it. Send `/new` through the ordinary composer. Require a new native session and a fresh-context answer while the original chat and history remain saved. ## What Changed - Add the `chat-continuity` flow and independent evidence grader. - Reuse the existing chat sender and run-bound native reply reader. - Require exact board-user message attribution, three successful native provider turns and one synthetic reset row. Count all four persisted run IDs. A reset with provider input or a native session cannot pass as synthetic work. - Use the original bound issue-run summary to prove reset completion when the detail result is null or compact. Preserve the raw detail and require its reset context, company, agent, issue and wake comment to agree. Native provider turns require committed native state and independent output evidence; an optional duplicate result field is not required. - Require an empty agent-filtered run history before the first send. Reused agents are unsupported for this case. Validate every agent-scoped activity row and reject mixed or truncated responses; do not filter global history into a pass. - Require the fresh native detail to contain an explicit null `sessionIdBefore`. Missing or empty admission evidence cannot prove a fresh provider session. Keep optional result blobs optional. - Check provider, model, environment, native session, finalization and reset generation. Reject stale replies, duplicate sends, extra runs, missing history and unexpected work. - Permit authenticated chat screenshots through the existing same-origin and mounted-Layout guard. Authentication and unrelated settings pages remain excluded. - Register the case, exact checks, source hash inputs, discovery and documentation. Feature-map files are unchanged. ## Verification - Normal merge commit `ddc37bb05b87cbc1fa449e4b2fc5a4677f0fe3b6` has exactly two real parents: the staging-qualified chat source `4c6cc9d67d2ae577e3a8ae7856bd6ae22a8489c1` and lifecycle squash `8771606fd87f69ed4057779e3dce3a9bca8088e6` from #18782. Thirteen dedicated case, support and reused runtime/oracle files are byte-for-byte preserved. The eight shared registry placements match the finite integration forecast. The nine required checks and four-run limit are unchanged. - The integrated source passes 99 focused tests across three files and 925 related Product E2E/catalog/native-reply tests across 42 files. Credential-free discovery lists 22 unique journeys, including chat and lifecycle exactly once. The PR diff against the actual lifecycle squash passes its whitespace check and remains the original 17 chat paths. - The current canonical `node scripts/typecheck-test-projects.mjs --low-memory tests/runner-e2e` passes, including Runner TypeScript and server declaration prerequisites. The DB migration checks and build pass against the actual squash's execution-admission schema and migration 0428. No Rust build was run locally. Canonical generated outputs are bound to current tracked inputs and were rechecked after tests. - The first direct typecheck found a stale generated Runner export, `taskDocumentKeySchema`; the current canonical prerequisites resolved it. A frozen offline install lacked a cached locked tarball, and an initial child command reached the global pnpm version switch and failed signature retrieval. Fetching only locked packages and pinning child pnpm to 9.15.4 corrected the setup. The original diagnostics remain retained; no tracked source was changed to bypass them. - The unchanged chat implementation passed a genuine staging campaign, workflow 41725181387, at source `4c6cc9d67d2ae577e3a8ae7856bd6ae22a8489c1`: all nine checks, all five phases, three provider turns plus one synthetic reset, all four usage IDs, and closed cleanup. This evidence remains bound to that original source. The new integration head requires its own current gates and coordinator qualification decision; this PR does not enroll anything in the daily plan. - The earlier staging workflow 41701828968 remains failed. Its nine behavior checks passed, but private collection rejected the valid reset's capture of the old session and cleanup failed. The private adapter fix and evidence-first recovery were handled separately. The original failed grade and archive are preserved. Reset prior-session capture is distinct from the third native turn's explicit-null fresh admission. - Three missing/undefined/empty fresh-admission controls reproduced false passes on `247827a066af01fa583ff030df79b61cb2c87730`, then passed after the strict null fix. Four earlier controls reproduced compact-detail, mixed-activity, missing fresh-agent receipt and unscoped-activity problems. Their original failures remain retained. Positive and wrong outcomes still enforce bound reset summaries, committed native state, configured identity and independent rendered output. Board provisioning uses the acting-agent filter; it does not grandfather earlier work by the test agent. - The first broad support attempt passed 861 tests and failed one report-generator check. A separate reproduction showed `listen EPERM` for the tsx local IPC socket before report output; the unchanged support suite passed with normal local IPC permission. An early flow fixture also returned an object where the events API returns an array; the real event reader rejected it, and only that fixture was corrected. Both original failures remain recorded. - Original commit `d6226b30acaf5a2dad28030f5b295ac1dd119565` failed full CI [41653171240](https://github.com/paperclipai/paperclip/actions/runs/41653171240): a CLI child-server runtime-file wait and two 15-second attention-retention timeouts, followed by database setup/cleanup errors. Their exact causes remain unresolved; the failed jobs and logs were retained and not retried. A later pass does not explain those timeouts. - Strict-admission source `055a7fa99e7cb8aad12b85fccf23880f0dc98a3f` failed full CI [41669527762](https://github.com/paperclipai/paperclip/actions/runs/41669527762): private-invitation and pending-shell access fixtures. The retained assertions match the account-identity fixture repair in #18799, now included through master, together with #18798 and #18650. No old-head job was retried. Subsequent source `4c6cc9d67d2ae577e3a8ae7856bd6ae22a8489c1` passed full CI 41680411215 and current-source review at 5/5. - Current full CI [41743580793, attempt 2](https://github.com/paperclipai/paperclip/actions/runs/41743580793/attempts/2) completed successfully on unchanged `ddc37bb05b87cbc1fa449e4b2fc5a4677f0fe3b6` at 2026-11-04 00:34:52 UTC: 48 successful jobs and one intentional selector skip. Attempt 1 remains failed: three run-bound runners were independently confirmed to have lost Spot capacity, and the two aggregate failures reflected those interrupted lanes. The build and boot health smoke had passed before interruption. Exactly one failed-job recheck was used; no source change or further retry was made. Fresh exact-head Greptile review is 5/5 with no new actionable issues and both review threads resolved. The separate Ponytail assessment says Ship with no must-fix or should-fix item; its optional direct board-row grader control remains documented. Whole-workspace recursive typecheck, full tests and full build were covered by hosted verification and were not repeated locally; the bounded local checks above passed. These gates do not claim that a later master merge tree has been live-qualified. ## Risks The attached target must supply a fresh disposable native agent and durable pre-send ownership for its chat. Its private adapter must collect all four run IDs, distinguish the synthetic reset from the three provider turns, and stop and clean up owned resources after evidence is saved. This case verifies browser reload, reopen and explicit session reset. It does not verify controller restart. Unknown setup or provider cost remains unknown. No shared QA source agent, model setting, production behavior or budget policy is changed by this public PR. ## Model Used OpenAI GPT-6 through Codex with reasoning, code execution and repository tools. The runtime does not expose a more specific model identifier or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/evals.md | 6 +- tests/runner-e2e/PRODUCT-JOURNEYS.md | 1 + tests/runner-e2e/catalog.test.ts | 1 + tests/runner-e2e/journey-features.ts | 1 + .../product-journeys/CHAT-CONTINUITY.md | 62 ++++++ .../product-journeys/agent-flows.ts | 1 + tests/runner-e2e/product-journeys/catalog.ts | 1 + .../chat-continuity-evidence.test.ts | 135 +++++++++++++ .../chat-continuity-evidence.ts | 157 +++++++++++++++ .../chat-continuity-flow.test.ts | 180 ++++++++++++++++++ .../product-journeys/chat-continuity-flow.ts | 164 ++++++++++++++++ .../chat-continuity.test-fixture.ts | 66 +++++++ tests/runner-e2e/product-journeys/launch.ts | 6 + .../product-screenshot.test.ts | 4 +- .../product-journeys/product-screenshot.ts | 2 +- .../product-journeys/success-contract.ts | 2 + tests/runner-e2e/product-journeys/types.ts | 1 + 17 files changed, 785 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/product-journeys/CHAT-CONTINUITY.md create mode 100644 tests/runner-e2e/product-journeys/chat-continuity-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/chat-continuity-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts create mode 100644 tests/runner-e2e/product-journeys/chat-continuity-flow.ts create mode 100644 tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts diff --git a/doc/evals.md b/doc/evals.md index 20f89339176..e2db0f49f9e 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -117,8 +117,10 @@ successor. Registration and calibration are separate from a live staging pass. deployment targets and starting-state recipes. The shared task-properties journey and task-document create/edit/reopen journey use no model; Cloud onboarding supplies reusable verified state. The document journey checks exact -saved content and both revision records through independent API reads. Cloud -fleet operations and trusted CI remain private. Local eval gateway qualification +saved content and both revision records through independent API reads. The explicit `chat-continuity` selection verifies browser reload/reopen, prompt-hidden recall, +ordinary `/new` reset, fresh native context and retained history against an attached target. +It requires a fresh owned agent with no prior work runs and uses three provider turns plus one synthetic reset row; it does not restart a shared stack. +Cloud fleet operations and trusted CI remain private. Local eval gateway qualification does not qualify Cloud routes, and these selections stay outside `--all`. The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 80c30828b23..6fb291d0ac7 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -31,6 +31,7 @@ never start a server or depend on a tenant filesystem. | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | +| `chat-continuity` | Fresh owned native agent with no prior work runs; Agent Chat enabled; fresh owned user chat; three provider turns and one synthetic reset | Recall prompt-hidden context after reopening, then use `/new` to advance the same saved conversation to a fresh native session, preserve history and verify the final reply after reload. See [CHAT-CONTINUITY.md](product-journeys/CHAT-CONTINUITY.md) | | `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | | `task-edit-lifecycle` | Company, signed-in board user, task-write capability; target support for run-owned labels and exact predeclared `taskEdit.revisedTitle` cleanup; zero model runs | Create a self-assigned task, edit its inline header title and label, reload, complete and reopen it; independently verify exact saved state, unchanged context and zero runs. Open a closed Properties panel through its shipped toggle | diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 93833491ec8..4ab07867b7f 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,6 +136,7 @@ describe("runner E2E catalog", () => { "task-create", "project-task-context", "ask-question", + "chat-continuity", "question-answer-resume", "task-properties", "task-document-persistence", diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index ab33d9780cd..b964103c45d 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -26,6 +26,7 @@ associate(["product-journeys"], { "inbox-triage": ["inbox-search/views", "inbox-search/triage"], "task-create": ["tasks/create"], "ask-question": ["questions-and-approvals/questions"], "question-answer-resume": ["questions-and-approvals/questions", "questions-and-approvals/continuation"], "followup-while-running": ["steering/queue"], + "chat-continuity": ["agent-chat/history", "runs-adapters/session"], "hire-delegate-reuse": ["agents/hire", "documents-artifacts/documents"], "agent-settings-persistence": ["agents/instructions", "tasks/completion"], "project-task-context": ["projects/lifecycle", "projects/task-context", "tasks/create"], diff --git a/tests/runner-e2e/product-journeys/CHAT-CONTINUITY.md b/tests/runner-e2e/product-journeys/CHAT-CONTINUITY.md new file mode 100644 index 00000000000..003c2fd19e2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/CHAT-CONTINUITY.md @@ -0,0 +1,62 @@ +# Chat reload and session continuity + +`chat-continuity` is a deployment-targeted Product E2E journey. It uses the +ordinary Agent Chat composer on a fresh, campaign-owned native agent. It does +not restart a persistent Cloud instance or alter a shared source agent. +The agent-filtered heartbeat history must be empty before the first send. +An agent with earlier completed, failed, queued or active work is unsupported; +the fixture does not subtract or hide those runs. A target that reuses one +agent across several journeys must supply a separate fresh agent for this case. + +The first exchange contains a random phrase. After leaving the chat, reopening +it and reloading, the next message asks for that phrase without supplying it. +The independently attributed reply must contain the phrase, and both native +turns must retain the same session identity. + +The user then sends the ordinary `/new` command. Paperclip preserves the same +backing chat and saved history while advancing `conversationSessionGeneration` +from zero to one. The final request does not reveal the remembered phrase. Its +reply must use the fresh-context marker, with a different native session, no +prior session on admission, and no old phrase in the new native task context. +The independent run detail must include its `sessionIdBefore` column explicitly +set to `null`; a missing or empty field is not session-boundary evidence. +Saved history is not removed or rewritten by the fixture. + +Every reply is read from the actual rendered chat after reload using the +existing run-bound native reply oracle. Board-user messages must each persist +exactly once and bind to exactly one observed run. The original comments remain +unchanged through the final reload. The owned agent must create no extra task, +project or teammate. + +The closed accounting contract is **three successful native provider turns +plus one synthetic reset row**, four persisted run IDs in total. The reset must +have its normal reset context and a matching original `/issues/:id/runs` +summary receipt and must not publish an assistant response. That receipt binds +the request's company and issue to the same successful summary run, agent, +context issue and wake comment as the independently read detail. The raw run +detail is not rewritten. Its optional `resultJson` can be absent or compact; +an explicit contradictory reset flag is rejected. Real provider turns require +the authoritative committed `nativePhase`, not an optional duplicate result +field. Their output still requires a bound native-final event or the selected +attributed completion comment and an actual reloaded rendered reply. +The reset is not counted as an additional provider turn. Source model, provider and +managed environment must match all three native turns. The target provider owns +authentication, the disposable agent, pre-send chat ownership, complete run and +usage collection, stopping and evidence-first cleanup. Unknown setup or model +cost remains unknown; a run limit is not proof of an absolute spend limit. + +The company activity request includes the exact owned agent filter before the +server's 500-row limit. Every returned row must match that company and agent, +have its canonical activity identity, and leave the response below that limit. +Mixed or truncated results fail; the client never filters a global response +into an apparent pass. Unrelated company history therefore does not limit this +fresh agent's proof, while any owned task, project or teammate creation fails. +Activity identifies the acting agent, not the created entity. Board provisioning +therefore has no agent actor even when its `entityId` is this new agent. Creation +attributed to the owned agent is rejected, including activity before the first +chat message; the fresh-agent contract does not grandfather that work. + +This case covers browser reload/reopen and explicit session reset. Controller +restart remains covered by the separate existing Agent Chat journeys. Local +oracle and catalog checks do not establish a staging pass; daily admission +requires the actual original staging attempt and complete cleanup evidence. diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index b73a4cda8d5..5fd94765f42 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -24,6 +24,7 @@ type AgentJourney = Exclude; +const allPass = (f: ReturnType) => gradeChatContinuity(f.scope, f.points).every(check => check.passed); + +describe("chat reload and new-session evidence", () => { + it("requires the actual three-turn outcome, one reset and run-bound reloaded replies", () => { + const f = chatContinuityFixture(); + expect(gradeChatContinuity(f.scope, f.points).map(row => row.id)).toEqual(CHAT_CONTINUITY_CHECKS); + expect(allPass(f)).toBe(true); + }); + it("accepts normal compact run details only with bound native-final events and the original reset summary", () => { + expect(allPass(compactChatContinuityFixture())).toBe(true); + }); + it("rejects an unscoped activity response even below the truncation limit", () => { + const f = chatContinuityFixture(); + f.points[5]!.activity.push({ id: "foreign-event", companyId: "foreign", agentId: "other-agent", action: "project.created", + entityType: "project", entityId: "foreign-project", runId: "other-run" }); + expect(allPass(f)).toBe(false); + }); + it("requires the fresh agent's pre-send empty-history receipt", () => { + const f = chatContinuityFixture(); delete f.scope.agentHistory; + expect(allPass(f)).toBe(false); + }); + it("never supplies the old phrase in the recall or fresh-context request", () => { + const s = chatContinuityScenario("random-123"); + expect(s.remember).toContain(s.remembered); + expect(s.recall).not.toContain(s.remembered); + expect(s.fresh).not.toContain(s.remembered); + expect(s.reset).toBe("/new"); + }); + it("registers the exact measured checks and counts all four persisted runs", () => { + expect(journeyRequiredChecks["chat-continuity"]).toEqual(CHAT_CONTINUITY_CHECKS); + const config = { version: 1, id: "qa-chat", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["chat-continuity"], baseline: { features: {}, connections: [] }, + limits: { durationMs: 900_000, maxRuns: 4, modelBudgetCents: 500 } }; + expect(parseTargetConfig(config).limits.maxRuns).toBe(4); + for (const maxRuns of [0, 3, 5]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow("agent_journeys_require_bounded_runs"); + }); + const mutants: ReadonlyArray) => void]> = [ + ["missing point", (f: ReturnType) => { f.points.pop(); }], + ["duplicate phase", (f) => { f.points[5]!.phase = "fresh"; }], + ["foreign chat", (f) => { f.points[1]!.chat.companyId = "foreign"; }], + ["different user", (f) => { f.points[3]!.chat.conversationUserId = "foreign"; }], + ["replaced backing issue", (f) => { f.points[5]!.chat.id = "replacement"; }], + ["active conversation", (f) => { f.points[5]!.chat.conversationState = "running"; }], + ["scheduled retry", (f) => { f.points[5]!.chat.scheduledRetry = { state: "pending" }; }], + ["duplicate user send", (f) => { f.points[5]!.comments.push({ ...f.points[5]!.comments[0], id: "duplicate" }); }], + ["foreign author", (f) => { f.points[5]!.comments[0]!.authorUserId = "foreign"; }], + ["wrong remembered phrase", (f) => { f.points[2]!.reply!.body = "I do not remember"; }], + ["unbound first reply", (f) => { f.points[0]!.reply!.anchor = "unbound"; }], + ["first reply missing after reopen", (f) => { delete f.points[1]!.reply; }], + ["stale recall response", (f) => { f.points[2]!.reply!.runId = "run-0"; }], + ["foreign recall run", (f) => { f.points[2]!.runs[1]!.agentId = "foreign"; }], + ["different provider", (f) => { f.points[2]!.runs[1]!.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["different model", (f) => { f.points[5]!.runs[3]!.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["different environment", (f) => { f.points[5]!.runs[3]!.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["uncommitted native phase", (f) => { f.points[5]!.runs[3]!.nativePhase = "finalizing"; }], + ["missing session proof", (f) => { delete f.points[0]!.runs[0]!.nativeSessionId; }], + ["unintended session replacement on reload", (f) => { f.points[2]!.runs[1]!.nativeSessionId = "replacement"; }], + ["reset generation absent", (f) => { f.points[3]!.chat.conversationSessionGeneration = 0; }], + ["fresh run generation stale", (f) => { f.points[4]!.runs[3]!.contextSnapshot.conversationSessionGeneration = 0; }], + ["fresh turn resumes old session", (f) => { f.points[4]!.runs[3]!.nativeSessionId = "original-session"; }], + ["previous session supplied on fresh admission", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = "original-session"; }], + ["missing fresh-admission session evidence", (f) => { delete f.points[4]!.runs[3]!.sessionIdBefore; }], + ["undefined fresh-admission session evidence", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = undefined; }], + ["empty fresh-admission session evidence", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = ""; }], + ["old phrase leaked in native context", (f) => { f.points[4]!.runs[3]!.contextSnapshot.paperclipTaskMarkdown += f.scenario.remembered; }], + ["old phrase returned after reset", (f) => { f.points[4]!.reply!.body = f.scenario.remembered; }], + ["fresh marker absent", (f) => { f.points[4]!.reply!.body = "I have no phrase"; }], + ["extra run", (f) => { f.points[5]!.runs.push({ ...f.points[5]!.runs[3], id: "extra" }); }], + ["duplicate run ID", (f) => { f.points[5]!.runs[3]!.id = "run-0"; }], + ["missing reset", (f) => { f.points[5]!.runs.splice(2, 1); }], + ["missing synthetic reset receipt", (f) => { f.points[5]!.resetReceipts.splice(0); }], + ["foreign reset summary agent", (f) => { f.points[5]!.resetReceipts[0]!.agentId = "foreign"; }], + ["foreign reset summary company", (f) => { f.points[5]!.resetReceipts[0]!.companyId = "foreign"; }], + ["foreign reset summary issue", (f) => { f.points[5]!.resetReceipts[0]!.contextIssueId = "foreign"; }], + ["wrong reset summary run", (f) => { f.points[5]!.resetReceipts[0]!.runId = "run-0"; }], + ["wrong reset summary wake", (f) => { f.points[5]!.resetReceipts[0]!.wakeCommentId = "user-0"; }], + ["reset summary not successful", (f) => { f.points[5]!.resetReceipts[0]!.status = "running"; }], + ["contradictory reset detail", (f) => { f.points[5]!.runs[2]!.resultJson.conversationReset = false; }], + ["provider work disguised as reset", (f) => { f.points[5]!.runs[2]!.runnerProfileJson = { nativeExecutionInput: { provider: { kind: f.scope.provider, model: f.scope.model } } }; }], + ["native session attached to synthetic reset", (f) => { f.points[5]!.runs[2]!.nativeSessionId = "unexpected-provider-session"; }], + ["reset publishes an answer", (f) => { f.points[5]!.comments.push({ id: "reset-answer", authorAgentId: f.scope.agentId, createdByRunId: "run-2", body: "Hi" }); }], + ["unrequested completion callback", (f) => { f.points[5]!.runs[3]!.contextSnapshot.source = "automation"; }], + ["wrong wake comment", (f) => { f.points[5]!.runs[3]!.contextSnapshot.wakeCommentId = "user-0"; }], + ["run has conflicting issue IDs", (f) => { f.points[5]!.runs[3]!.issueId = "foreign"; }], + ["changed original history", (f) => { f.points[5]!.comments[0]!.body = "changed"; }], + ["missing original history", (f) => { f.points[5]!.comments.splice(1, 1); }], + ["reopen repeats the send", (f) => { f.points[5]!.runs.push({ ...f.points[5]!.runs[3], id: "repeated" }); }], + ["reply not observed after reload", (f) => { f.points[5]!.reply!.observedAfterReload = false; }], + ["hidden final reply", (f) => { f.points[5]!.reply!.visible = false; }], + ["created work", (f) => { f.points[5]!.tasks.push({ id: "unexpected" }); }], + ["created project", (f) => { f.points[5]!.activity.push({ id: "created-project", companyId: f.scope.companyId, agentId: f.scope.agentId, + runId: "run-1", action: "project.created", entityType: "project", entityId: "unexpected" }); }], + ["truncated creation history", (f) => { f.points[5]!.activity = Array.from({ length: 500 }, () => ({})); }], + ["missing run collection", (f) => { f.points[5]!.runs = undefined as unknown as Row[]; }], + ["malformed history row", (f) => { f.points[5]!.comments.push(null as unknown as Row); }], + ]; + it.each(mutants)("rejects %s", (_name, mutate) => { + const f = chatContinuityFixture(); + mutate(f); + expect(allPass(f)).toBe(false); + }); + it("rejects another agent's activity rather than filtering an unscoped response", () => { + const f = chatContinuityFixture(); + f.points[5]!.activity.push({ companyId: f.scope.companyId, agentId: "other-agent", runId: "other-run", action: "project.created" }); + expect(allPass(f)).toBe(false); + }); + it("accepts complete owned activity and rejects unaccounted creation by that same agent", () => { + const f = chatContinuityFixture(); + const row = { id: "owned-event", companyId: f.scope.companyId, agentId: f.scope.agentId, + runId: "run-0", action: "heartbeat.run_completed", entityType: "heartbeat_run", entityId: "run-0" }; + f.points[5]!.activity.push(row); + expect(allPass(f)).toBe(true); + row.action = "project.created"; row.runId = "unaccounted-owned-run"; + expect(allPass(f)).toBe(false); + }); + it("does not treat a compact run as sufficient without actual native output proof", () => { + const f = compactChatContinuityFixture(); + f.points[5]!.events = []; + expect(allPass(f)).toBe(false); + }); + it("retains backward-compatible issue aliases only when they agree", () => { + expect(chatContinuityIssueId({ nativeIssueId: "chat", contextSnapshot: { issueId: "chat" } })).toBe("chat"); + expect(chatContinuityIssueId({ issueId: "foreign", nativeIssueId: "chat" })).toBeUndefined(); + expect(chatContinuityIssueId({})).toBeUndefined(); + }); +}); diff --git a/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts b/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts new file mode 100644 index 00000000000..598d76f3ff5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts @@ -0,0 +1,157 @@ +import { chatMarker } from "../chat-cases.js"; +import { taskReplyBelongsToRun, type TaskReplyObservation } from "../native-task-reply.js"; + +type Row = Record; +export const CHAT_CONTINUITY_CHECKS = [ + "chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", + "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", + "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment", +] as const; + +export function chatContinuityScenario(campaignId: string) { + const remembered = chatMarker("OLDCONTEXT", campaignId), marker = chatMarker("CHAT", campaignId); + return { + remembered, marker, + remember: `For this conversation only, remember the phrase ${remembered}. Just acknowledge briefly; no project or task is needed.`, + recall: "What phrase did I just ask you to remember? Reply with the phrase only. No project or task is needed.", + reset: "/new", + fresh: `Without reading older history or files, if you have a remembered phrase in your current context return it; otherwise reply exactly ${marker}. Do not look it up. No project or task is needed.`, + }; +} + +export interface ChatContinuityScope { + companyId: string; userId: string; agentId: string; agentName: string; + environmentId: string; provider: string; model: string; campaignId: string; + agentHistory?: { source: "company-agent-runs"; companyId: string; agentId: string; limit: 1; runIds: string[]; observedBeforeSend: true }; +} +export interface ChatContinuityResetReceipt { + source: "issue-runs"; companyId: string; issueId: string; + runId: string; agentId: string; status: string; contextIssueId: string; wakeCommentId: string; + resultJson: { conversationReset: true }; +} +export interface ChatContinuityPoint { + phase: "initial" | "returned" | "recalled" | "reset" | "fresh" | "reopened"; + chat: Row; runs: Row[]; comments: Row[]; tasks: Row[]; activity: Row[]; + reply?: TaskReplyObservation & { runId: string; agentId: string; observedAfterReload: boolean }; + events?: Row[]; + resetReceipts: ChatContinuityResetReceipt[]; +} +const nonempty = (value: unknown): value is string => typeof value === "string" && value.length > 0; +export const chatContinuityResetRun = (run: Row) => run.contextSnapshot?.conversationReset === true || run.resultJson?.conversationReset === true; +export const chatContinuityIssueId = (run: Row): string | undefined => { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(nonempty); + return ids.length && new Set(ids).size === 1 ? ids[0] : undefined; +}; +const userBody = (row: Row) => typeof row.body === "string" ? row.body.replaceAll("\\_", "_").trim() : ""; +const sameIds = (left: Row[], right: Row[]) => left.length === right.length && left.every(row => right.some(other => row.id === other.id)); +const commentContent = (row: Row) => [row.id, row.companyId, row.issueId, row.body, row.authorAgentId, + row.authorUserId, row.createdByRunId, row.conversationSessionGeneration]; +const sameRows = (left: Row[], right: Row[]) => left.every(row => right.some(other => other.id === row.id + && JSON.stringify(commentContent(other)) === JSON.stringify(commentContent(row)))); + +export function chatContinuityActivityComplete(scope: Pick, rows: Row[]): boolean { + return Array.isArray(rows) && rows.length < 500 && new Set(rows.map(row => row?.id)).size === rows.length + && rows.every(row => row && nonempty(row.id) && row.companyId === scope.companyId && row.agentId === scope.agentId + && nonempty(row.action) && nonempty(row.entityType) && nonempty(row.entityId)); +} + +export function chatContinuityResetReceiptMatches(scope: Pick, + chatId: string, run: Row, receipt: ChatContinuityResetReceipt): boolean { + const fullResult = run.resultJson; + const optionalFlagValid = fullResult == null || typeof fullResult === "object" && !Array.isArray(fullResult) + && (!Object.hasOwn(fullResult, "conversationReset") || fullResult.conversationReset === true); + return receipt?.source === "issue-runs" && receipt.companyId === scope.companyId && receipt.issueId === chatId + && receipt.runId === run.id && receipt.agentId === scope.agentId && receipt.status === "succeeded" + && receipt.contextIssueId === chatId && nonempty(receipt.wakeCommentId) + && receipt.wakeCommentId === run.contextSnapshot?.wakeCommentId && receipt.resultJson?.conversationReset === true + && run.companyId === scope.companyId && run.agentId === scope.agentId && chatContinuityIssueId(run) === chatId + && run.status === "succeeded" && run.contextSnapshot?.conversationReset === true && optionalFlagValid + && !run.runnerProfileJson?.nativeExecutionInput && !run.nativeSessionId; +} + +export function gradeChatContinuity(scope: ChatContinuityScope, points: ChatContinuityPoint[]) { + const scenario = chatContinuityScenario(scope.campaignId); + const phases = ["initial", "returned", "recalled", "reset", "fresh", "reopened"] as const; + const complete = points.length === phases.length && points.every((point, index) => point?.phase === phases[index] + && point.chat && [point.runs, point.comments, point.tasks, point.activity, point.resetReceipts].every(rows => Array.isArray(rows) + && rows.every(row => row && typeof row === "object" && !Array.isArray(row)))); + if (!complete) return CHAT_CONTINUITY_CHECKS.map(id => ({ id, passed: false })); + const [initial, returned, recalled, reset, fresh, reopened] = complete ? points : []; + const chatId = initial?.chat?.id; + const scoped = complete && [scope.companyId, scope.userId, scope.agentId, scope.agentName, scope.environmentId, + scope.provider, scope.model, scope.campaignId].every(nonempty) && nonempty(chatId) + && scope.agentHistory?.source === "company-agent-runs" && scope.agentHistory.companyId === scope.companyId + && scope.agentHistory.agentId === scope.agentId && scope.agentHistory.limit === 1 && scope.agentHistory.observedBeforeSend === true + && Array.isArray(scope.agentHistory.runIds) && scope.agentHistory.runIds.length === 0 + && points.every(point => point.chat?.id === chatId && point.chat.companyId === scope.companyId + && point.chat.assigneeAgentId === scope.agentId && point.chat.conversationAgentId === scope.agentId + && point.chat.conversationUserId === scope.userId && point.chat.createdByUserId === scope.userId + && point.chat.title === `Chat with ${scope.agentName}` && point.chat.status === "in_review" + && point.chat.conversationState === "waiting" && !point.chat.executionRunId && !point.chat.scheduledRetry && !point.chat.activeRecoveryAction); + const requested = [scenario.remember, scenario.recall, scenario.reset, scenario.fresh]; + const finalComments = reopened?.comments ?? []; + const board = finalComments.filter(row => row.authorUserId === scope.userId && !row.authorAgentId); + const userMessages = scoped && board.length === requested.length && new Set(board.map(row => row.id)).size === requested.length + && requested.every(body => board.filter(row => userBody(row) === body && row.companyId === scope.companyId && row.issueId === chatId).length === 1) + && finalComments.every(row => !row.authorUserId || row.authorUserId === scope.userId); + const runs = reopened?.runs ?? [], providerRuns = runs.filter(run => !chatContinuityResetRun(run)), resetRuns = runs.filter(chatContinuityResetRun); + const runBinding = scoped && points.every(point => point.runs.every(run => nonempty(run.id) && run.companyId === scope.companyId + && run.agentId === scope.agentId && chatContinuityIssueId(run) === chatId && run.status === "succeeded")); + const providerBinding = runBinding && providerRuns.length === 3 && points.flatMap(point => point.runs) + .filter(run => !chatContinuityResetRun(run)).every(run => { + const provider = run.runnerProfileJson?.nativeExecutionInput?.provider; + return run.runtimeMode === "native" && provider?.kind === scope.provider && provider?.model === scope.model + && run.contextSnapshot?.paperclipEnvironment?.id === scope.environmentId + && run.nativePhase === "committed" + && nonempty(run.nativeSessionId); + }); + const expectedCounts = [1, 1, 2, 3, 4, 4]; + const exactTurns = runBinding && providerBinding && resetRuns.length === 1 && runs.length === 4 && new Set(runs.map(run => run.id)).size === 4 + && points.every(point => point.resetReceipts.length === point.runs.filter(chatContinuityResetRun).length + && point.runs.filter(chatContinuityResetRun).every(run => point.resetReceipts + .filter(receipt => chatContinuityResetReceiptMatches(scope, chatId, run, receipt)).length === 1)) + && points.every((point, index) => point.runs.length === expectedCounts[index] + && point.runs.every(run => runs.some(final => final.id === run.id && final.status === run.status))) + && runs.every(run => { + const request = board.find(comment => comment.id === run.contextSnapshot?.wakeCommentId); + return request && run.contextSnapshot?.source === "issue.comment" + && (chatContinuityResetRun(run) ? userBody(request) === scenario.reset : userBody(request) !== scenario.reset); + }) && new Set(runs.map(run => run.contextSnapshot?.wakeCommentId)).size === 4; + const pointReply = (point: ChatContinuityPoint | undefined) => { + const receipt = point?.reply, run = point?.runs.find(run => run.id === receipt?.runId); + return receipt && run && receipt.agentId === scope.agentId && receipt.observedAfterReload === true + && taskReplyBelongsToRun(receipt, run, scope.agentId, point!.comments, point!.events ?? []) ? receipt : undefined; + }; + const originalReply = pointReply(initial), returnedReply = pointReply(returned), recalledReply = pointReply(recalled), freshReply = pointReply(fresh), reopenedReply = pointReply(reopened); + const firstRuns = initial?.runs.filter(run => !chatContinuityResetRun(run)) ?? []; + const recallRuns = recalled?.runs.filter(run => !chatContinuityResetRun(run) && run.id !== firstRuns[0]?.id) ?? []; + const freshRuns = fresh?.runs.filter(run => !chatContinuityResetRun(run) && !recalled?.runs.some(old => old.id === run.id)) ?? []; + const first = firstRuns[0], second = recallRuns[0], last = freshRuns[0], resetRun = resetRuns[0]; + const recall = exactTurns && Boolean(originalReply && returnedReply && recalledReply && originalReply.runId === first?.id + && returnedReply.runId === originalReply.runId && returnedReply.body === originalReply.body && recalledReply.runId === second?.id + && recalledReply.body.includes(scenario.remembered) && first?.nativeSessionId === second?.nativeSessionId + && initial!.chat.conversationSessionGeneration === 0 && returned!.chat.conversationSessionGeneration === 0 + && recalled!.chat.conversationSessionGeneration === 0 && first?.contextSnapshot?.conversationSessionGeneration === 0 + && second?.contextSnapshot?.conversationSessionGeneration === 0 && sameIds(initial!.runs, returned!.runs) + && sameRows(initial!.comments, returned!.comments)); + const boundary = exactTurns && Boolean(first && second && last && resetRun && freshRuns.length === 1 + && reset!.chat.conversationSessionGeneration === 1 && fresh!.chat.conversationSessionGeneration === 1 + && reopened!.chat.conversationSessionGeneration === 1 && last.contextSnapshot?.conversationSessionGeneration === 1 + && resetRun.contextSnapshot?.conversationSessionGeneration === 1 + && Object.hasOwn(last, "sessionIdBefore") && last.sessionIdBefore === null + && last.nativeSessionId !== first.nativeSessionId && last.nativeSessionId !== second.nativeSessionId + && typeof last.contextSnapshot?.paperclipTaskMarkdown === "string" + && !last.contextSnapshot.paperclipTaskMarkdown.includes(scenario.remembered) + && finalComments.every(row => row.createdByRunId !== resetRun.id)); + const freshAnswer = boundary && Boolean(freshReply && freshReply.runId === last?.id + && freshReply.body.trim() === scenario.marker && !freshReply.body.includes(scenario.remembered)); + const history = scoped && Boolean(reopenedReply && freshReply && reopenedReply.runId === freshReply.runId + && reopenedReply.body === freshReply.body && sameIds(fresh!.runs, reopened!.runs) + && sameRows(initial!.comments, reopened!.comments) && sameRows(recalled!.comments, reopened!.comments) + && sameRows(fresh!.comments, reopened!.comments)); + const noWork = scoped && points.every(point => Array.isArray(point.tasks) && point.tasks.length === 0 + && chatContinuityActivityComplete(scope, point.activity) + && !point.activity.some(row => ["issue.created", "issue.child_created", "project.created", "agent.created", "agent.hire_created"].includes(row.action))); + const values = [scoped, userMessages, recall, boundary, freshAnswer, history, noWork, exactTurns, providerBinding]; + return CHAT_CONTINUITY_CHECKS.map((id, index) => ({ id, passed: Boolean(values[index]) })); +} diff --git a/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts b/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts new file mode 100644 index 00000000000..f69c7f69bcf --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; +import { chatContinuityFixture, compactChatContinuityFixture } from "./chat-continuity.test-fixture.js"; + +const mocks = vi.hoisted(() => ({ send: vi.fn(), reply: vi.fn(), capture: vi.fn() })); +vi.mock("../chat-flow.js", () => ({ sendChatMessage: mocks.send })); +vi.mock("../native-task-reply.js", async importOriginal => ({ + ...await importOriginal(), readBoundTaskFinalReply: mocks.reply, +})); +vi.mock("./product-screenshot.js", () => ({ captureProductScreenshot: mocks.capture })); +vi.mock("@playwright/test", () => ({ expect: (locator: { isVisible(): Promise }) => ({ + async toBeVisible() { expect(await locator.isVisible()).toBe(true); }, +}) })); +import { runChatContinuity } from "./chat-continuity-flow.js"; + +const directories: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + vi.resetAllMocks(); + for (const directory of directories.splice(0)) await rm(directory, { recursive: true, force: true }); +}); +async function fixture(options: { preexisting?: boolean; intentFails?: boolean; sendFails?: boolean; foreignRun?: boolean; + compactRuns?: boolean; unrelatedActivity?: number; ignoreActivityScope?: boolean; priorAgentStatus?: string; mixedAgentHistory?: boolean; + provisioningActor?: "board" | "owned-agent" } = {}) { + const f = options.compactRuns ? compactChatContinuityFixture() : chatContinuityFixture(); + const directory = await mkdtemp(path.join(os.tmpdir(), "chat-continuity-flow-")); + directories.push(directory); + let stage = options.preexisting ? 0 : -1; + const messages: string[] = [], actions: string[] = []; + const realTimeout = globalThis.setTimeout; + vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => + realTimeout(callback, delay === 1000 ? 0 : delay, ...args)) as typeof setTimeout); + const current = () => f.points[stage]!; + const companyActivity = [ + ...Array.from({ length: options.unrelatedActivity ?? 0 }, (_, index) => ({ id: `unrelated-${index}`, + companyId: f.scope.companyId, agentId: "other-agent", runId: "other-run", action: "project.created", entityType: "project", entityId: `other-project-${index}` })), + ...(options.provisioningActor ? [{ id: "provisioned-agent", companyId: f.scope.companyId, + agentId: options.provisioningActor === "board" ? null : f.scope.agentId, runId: null, + action: "agent.created", entityType: "agent", entityId: f.scope.agentId }] : []), + ]; + const companyRuns = Array.from({ length: options.unrelatedActivity ?? 0 }, (_, index) => ({ id: `unrelated-run-${index}`, + companyId: f.scope.companyId, agentId: "other-agent", status: "succeeded" })); + if (options.priorAgentStatus) companyRuns.push({ id: "prior-owned-run", companyId: f.scope.companyId, agentId: f.scope.agentId, status: options.priorAgentStatus }); + const get = vi.fn(async (route: string) => { + if (route === "/api/instance/settings/experimental") return { enableAgentChat: true }; + if (route === "/api/companies/company/heartbeat-runs?agentId=agent&limit=1&summary=true") { + actions.push("history"); + if (options.mixedAgentHistory) return [{ id: "foreign-run", companyId: "foreign-company", agentId: "other-agent", status: "running" }]; + return companyRuns.filter(run => run.agentId === f.scope.agentId).slice(0, 1); + } + if (route === "/api/companies/company/chats/agent") return stage < 0 ? null : structuredClone(current().chat); + if (route === "/api/issues/chat/runs") return current().runs.map(run => ({ runId: run.id, agentId: run.agentId, status: run.status, + contextIssueId: run.contextSnapshot.issueId, wakeCommentId: run.contextSnapshot.wakeCommentId, + resultJson: run.contextSnapshot.conversationReset ? { conversationReset: true } : {} })); + if (route === "/api/issues/chat/comments?order=asc") return structuredClone(current().comments); + if (route === "/api/companies/company/issues?view=index&assigneeAgentId=agent&includeAgentCreatedTasks=true&limit=2") return []; + if (route === "/api/companies/company/activity?agentId=agent&limit=500") { + return (options.ignoreActivityScope ? companyActivity : companyActivity.filter(row => row.agentId === f.scope.agentId)).slice(0, 500); + } + if (/\/api\/heartbeat-runs\/run-\d\/events\?/.test(route)) return structuredClone(current().events ?? []); + const id = route.match(/^\/api\/heartbeat-runs\/(run-\d)$/)?.[1]; + if (id) { + const run = structuredClone(current().runs.find(row => row.id === id)!); + if (options.foreignRun) run.nativeIssueId = "foreign"; + return run; + } + throw new Error(`unexpected GET ${route}`); + }); + const page = { goto: vi.fn(async (_url: string) => { actions.push("navigate"); }), reload: vi.fn(async () => { actions.push("reload"); }), + getByTestId: vi.fn(() => ({ isVisible: async () => true })) }; + const own = vi.fn(async (resource: { id?: string }) => { + actions.push(resource.id ? "receipt" : "intent"); + if (options.intentFails && !resource.id) throw new Error("ownership intent failed"); + }); + const context = { page, api: { get, origin: "https://qa.example.test" }, + company: { id: f.scope.companyId, issuePrefix: "QA" }, user: { id: f.scope.userId, name: "QA" }, + environmentId: f.scope.environmentId, agent: { id: f.scope.agentId, name: f.scope.agentName, + provider: f.scope.provider, model: f.scope.model, runtimeMode: "native" } } as unknown as JourneyContext; + const evidence: JourneyEvidence = { outcome: "passed", startingState: "recipe", phases: [], checks: [], resourceDisposition: "released", privacy: "private" }; + mocks.send.mockImplementation(async (_page, message: string) => { + actions.push("send"); messages.push(message); + if (options.sendFails) throw new Error("uncertain send response"); + const index = [f.scenario.remember, f.scenario.recall, f.scenario.reset, f.scenario.fresh].indexOf(message); + expect(index).toBe(messages.length - 1); + stage = [0, 2, 3, 4][index]!; + }); + mocks.reply.mockImplementation(async input => { + await input.page.reload(); + const data = await input.load(); + expect(data.run.id).toBe(input.runId); + const point = f.points.find(point => point.reply?.runId === input.runId)!; + return structuredClone(point.reply!); + }); + mocks.capture.mockResolvedValue(undefined); + return { ...f, context, evidence, own, directory, messages, actions, page, get, companyActivity, + run: () => runChatContinuity(context, { own } as unknown as TargetSession, f.scope.campaignId, evidence, + { directory, deadlineAt: Date.now() + 10_000 }) }; +} +describe("chat continuity journey controls", () => { + it("uses ordinary user messages once, owns the fresh chat before dispatch, and retains the independently graded result", async () => { + const f = await fixture(); + await f.run(); + expect(f.messages).toEqual([f.scenario.remember, f.scenario.recall, "/new", f.scenario.fresh]); + expect(f.actions.indexOf("intent")).toBeLessThan(f.actions.indexOf("send")); + expect(f.actions.indexOf("history")).toBeLessThan(f.actions.indexOf("intent")); + expect(f.own).toHaveBeenCalledTimes(2); + expect(f.own.mock.calls[0]![0]).toMatchObject({ journeyId: "chat-continuity", freshChat: { absent: true, userId: f.scope.userId } }); + expect(f.own.mock.calls[1]![0]).toMatchObject({ id: "chat", journeyId: "chat-continuity" }); + expect(f.evidence.checks).toHaveLength(9); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(mocks.reply).toHaveBeenCalledTimes(5); + expect(mocks.capture).toHaveBeenCalledTimes(6); + expect(f.page.goto.mock.calls.flat().every(url => typeof url !== "string" || url.startsWith("https://qa.example.test/"))).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.map((point: { phase: string }) => point.phase)).toEqual(["initial", "returned", "recalled", "reset", "fresh", "reopened"]); + expect(saved.checks.every((check: { passed: boolean }) => check.passed)).toBe(true); + }); + it("accepts compact details with original reset summaries despite more than 500 unrelated company events", async () => { + const f = await fixture({ compactRuns: true, unrelatedActivity: 750 }); + await f.run(); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.every((point: { activity: unknown[] }) => point.activity.length === 0)).toBe(true); + expect(saved.points[5].resetReceipts[0]).toMatchObject({ source: "issue-runs", runId: "run-2", agentId: f.scope.agentId, + contextIssueId: "chat", wakeCommentId: "user-2", resultJson: { conversationReset: true } }); + expect(saved.points[5].runs[2].resultJson).toBeNull(); + }); + it.each([1, 750])("rejects an ignored agent filter instead of filtering %s unrelated global activity rows", async unrelatedActivity => { + const f = await fixture({ unrelatedActivity, ignoreActivityScope: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_observation_incomplete"); + expect(f.messages).toEqual([f.scenario.remember]); + }); + it("accepts board provisioning whose created entity is the owned agent, rather than treating its entity id as the actor", async () => { + const f = await fixture({ provisioningActor: "board", unrelatedActivity: 750 }); + expect(f.companyActivity.at(-1)).toMatchObject({ action: "agent.created", agentId: null, entityId: f.scope.agentId }); + await f.run(); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.every((point: { activity: unknown[] }) => point.activity.length === 0)).toBe(true); + }); + it("rejects creation attributed to the fresh owned agent even if that activity predates the first send", async () => { + const f = await fixture({ provisioningActor: "owned-agent" }); + await expect(f.run()).rejects.toThrow("chat_continuity_evidence_failed"); + expect(f.evidence.checks.find(check => check.id === "chat-no-unrequested-work")?.passed).toBe(false); + }); + it.each(["succeeded", "running", "queued", "failed"])("rejects prior %s work on a reused agent before any send", async priorAgentStatus => { + const f = await fixture({ priorAgentStatus, unrelatedActivity: 750 }); + await expect(f.run()).rejects.toThrow("fresh_chat_agent_required"); + expect(f.messages).toEqual([]); + expect(f.own).not.toHaveBeenCalled(); + }); + it("rejects mixed or foreign agent history before any send", async () => { + const f = await fixture({ mixedAgentHistory: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_agent_history_scope_mismatch"); + expect(f.messages).toEqual([]); + expect(f.own).not.toHaveBeenCalled(); + }); + it.each(["preexisting", "intentFails"] as const)("does not dispatch model work when %s rejects ownership", async option => { + const f = await fixture({ [option]: true }); + await expect(f.run()).rejects.toThrow(); + expect(f.messages).toEqual([]); + expect(f.evidence.checks.every(check => !check.passed)).toBe(true); + }); + it("never replays an uncertain send", async () => { + const f = await fixture({ sendFails: true }); + await expect(f.run()).rejects.toThrow("uncertain send response"); + expect(f.messages).toEqual([f.scenario.remember]); + expect(mocks.send).toHaveBeenCalledTimes(1); + expect(f.evidence.checks.some(check => !check.passed)).toBe(true); + }); + it("rejects a foreign native run before sending another message", async () => { + const f = await fixture({ foreignRun: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_run_mismatch"); + expect(f.messages).toEqual([f.scenario.remember]); + expect(f.evidence.checks.some(check => !check.passed)).toBe(true); + }); +}); diff --git a/tests/runner-e2e/product-journeys/chat-continuity-flow.ts b/tests/runner-e2e/product-journeys/chat-continuity-flow.ts new file mode 100644 index 00000000000..fe2a2301371 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-flow.ts @@ -0,0 +1,164 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { sendChatMessage } from "../chat-flow.js"; +import { readBoundTaskFinalReply } from "../native-task-reply.js"; +import { collectRunEvents } from "../run-observations.js"; +import { chatContinuityActivityComplete, chatContinuityIssueId, chatContinuityResetReceiptMatches, + chatContinuityResetRun, chatContinuityScenario, gradeChatContinuity, + type ChatContinuityPoint, type ChatContinuityScope, type ChatContinuityResetReceipt } from "./chat-continuity-evidence.js"; +import { createObservedReader } from "./observation-read.js"; +import { captureProductScreenshot } from "./product-screenshot.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +interface Options { deadlineAt: number; directory: string } + +export async function runChatContinuity(context: JourneyContext, session: TargetSession, + campaignId: string, evidence: JourneyEvidence, options: Options) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !agent.model || !agent.provider || !context.environmentId) { + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + } + const scope: ChatContinuityScope = { companyId: context.company.id, userId: context.user.id, + agentId: agent.id, agentName: agent.name, environmentId: context.environmentId, + provider: agent.provider, model: agent.model, campaignId }; + const scenario = chatContinuityScenario(campaignId), reader = createObservedReader(context.api, options.deadlineAt); + const chatPath = `/api/companies/${scope.companyId}/chats/${agent.id}`; + const chatUrl = `${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/chats/${encodeURIComponent(agent.id)}`; + const points: ChatContinuityPoint[] = []; + let chatId: string | undefined, owned = false; + const remaining = () => { + const value = options.deadlineAt - Date.now(); + if (value <= 0) throw new JourneyError("failed", "chat_continuity_deadline"); + return value; + }; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = async () => writeFile(path.join(options.directory, "chat-continuity.json"), + JSON.stringify({ scope, points, checks: gradeChatContinuity(scope, points) }, null, 2), { mode: 0o600 }); + const freshChat = { companyId: scope.companyId, userId: scope.userId, leadAgentId: agent.id, absent: true as const }; + const read = async (): Promise | undefined> => { + const chat = await reader.get(chatPath); + if (!chat) return; + if (!chat.id || chat.companyId !== scope.companyId || chat.assigneeAgentId !== agent.id + || chat.conversationAgentId !== agent.id || chat.conversationUserId !== scope.userId + || chat.createdByUserId !== scope.userId || chat.title !== `Chat with ${agent.name}` || chatId && chat.id !== chatId) { + throw new JourneyError("failed", "chat_continuity_scope_mismatch"); + } + chatId ??= chat.id; + if (!owned) { + await session.own({ kind: "issue", name: chat.title, id: chat.id, journeyId: "chat-continuity", freshChat }); + owned = true; + } + const [summaries, comments, tasks, activity] = await Promise.all([ + reader.get(`/api/issues/${chat.id}/runs`), reader.get(`/api/issues/${chat.id}/comments?order=asc`), + reader.get(`/api/companies/${scope.companyId}/issues?view=index&assigneeAgentId=${agent.id}&includeAgentCreatedTasks=true&limit=2`), + reader.get(`/api/companies/${scope.companyId}/activity?agentId=${agent.id}&limit=500`), + ]); + if (![summaries, comments, tasks, activity].every(Array.isArray) || summaries.length > 4 || !chatContinuityActivityComplete(scope, activity) + || new Set(summaries.map(row => row.runId)).size !== summaries.length) { + throw new JourneyError("failed", "chat_continuity_observation_incomplete"); + } + const runs = await Promise.all(summaries.map(async row => { + if (typeof row.runId !== "string" || !row.runId) throw new JourneyError("failed", "chat_continuity_run_mismatch"); + if (row.agentId !== agent.id || row.contextIssueId !== chat.id) throw new JourneyError("failed", "chat_continuity_summary_scope_mismatch"); + const run = await reader.get(`/api/heartbeat-runs/${row.runId}`); + if (run.id !== row.runId || run.companyId !== scope.companyId || run.agentId !== agent.id + || chatContinuityIssueId(run) !== chat.id || !["queued", "running", "succeeded"].includes(run.status)) { + throw new JourneyError("failed", "chat_continuity_run_mismatch"); + } + return run; + })); + const resetReceipts: ChatContinuityResetReceipt[] = summaries.flatMap(summary => summary.resultJson?.conversationReset === true ? [{ + source: "issue-runs" as const, companyId: scope.companyId, issueId: chat.id, + runId: summary.runId, agentId: summary.agentId, status: summary.status, + contextIssueId: summary.contextIssueId, wakeCommentId: summary.wakeCommentId, resultJson: { conversationReset: true as const }, + }] : []); + return { chat, runs, comments, tasks, activity, resetReceipts }; + }; + const wait = async (providerCount: number, resetCount: number) => { + let stable = ""; + while (remaining() > 0) { + const state = await read(); + if (state) { + const providerRuns = state.runs.filter(run => !chatContinuityResetRun(run)); + const resetRuns = state.runs.filter(chatContinuityResetRun); + if (providerRuns.length > providerCount || resetRuns.length > resetCount) throw new JourneyError("failed", "chat_continuity_extra_run"); + const ready = providerRuns.length === providerCount && resetRuns.length === resetCount + && state.runs.every(run => run.status === "succeeded") + && providerRuns.every(run => run.nativePhase === "committed") + && resetRuns.every(run => state.resetReceipts.filter(receipt => chatContinuityResetReceiptMatches(scope, state.chat.id, run, receipt)).length === 1) + && state.chat.status === "in_review" && state.chat.conversationState === "waiting" + && !state.chat.executionRunId && !state.chat.scheduledRetry && !state.chat.activeRecoveryAction; + const key = ready ? JSON.stringify([state.chat.id, state.chat.conversationSessionGeneration, + state.runs.map(run => [run.id, run.status]), state.comments.map(row => row.id)]) : ""; + if (key && stable === key) return state; + stable = key; + } + await new Promise(resolve => setTimeout(resolve, Math.min(1000, remaining()))); + } + throw new JourneyError("failed", "chat_continuity_not_settled"); + }; + const snapshot = async (phase: ChatContinuityPoint["phase"], state: Awaited>, runId?: string) => { + let point: ChatContinuityPoint = { phase, ...state }; + if (runId) { + const load = async () => { + const latest = await read(); + const run = latest?.runs.find(run => run.id === runId); + if (!latest || !run) throw new JourneyError("failed", "chat_continuity_reply_run_missing"); + const events = await collectRunEvents((afterSeq, limit) => + reader.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + return { run, comments: latest.comments, events }; + }; + const reply = await readBoundTaskFinalReply({ page: context.page, runId, agentId: agent.id, + deadlineAt: options.deadlineAt, knownCompanyPrefixes: [context.company.issuePrefix], load }); + const latest = await read(), runEvidence = await load(); + if (!latest) throw new JourneyError("failed", "chat_continuity_reply_chat_missing"); + point = { phase, ...latest, reply, events: runEvidence.events }; + } + points.push(point); + await retain(); + await captureProductScreenshot(context.page, context.api.origin, path.join(options.directory, `chat-${phase}.png`), options.deadlineAt); + return point; + }; + const newProviderRun = (state: Awaited>, old: Row[] = []) => { + const added = state.runs.filter(run => !chatContinuityResetRun(run) && !old.some(before => before.id === run.id)); + if (added.length !== 1) throw new JourneyError("failed", "chat_continuity_turn_ambiguous"); + return added[0]!; + }; + try { + const settings = await reader.get("/api/instance/settings/experimental"); + if (settings.enableAgentChat !== true) throw new JourneyError("unsupported", "agent_chat_required"); + const history = await reader.get(`/api/companies/${scope.companyId}/heartbeat-runs?agentId=${agent.id}&limit=1&summary=true`); + if (!Array.isArray(history) || history.some(run => !run || run.companyId !== scope.companyId || run.agentId !== agent.id)) { + throw new JourneyError("failed", "chat_continuity_agent_history_scope_mismatch"); + } + if (history.length) throw new JourneyError("unsupported", "fresh_chat_agent_required"); + scope.agentHistory = { source: "company-agent-runs", companyId: scope.companyId, agentId: agent.id, limit: 1, runIds: [], observedBeforeSend: true }; + if (await reader.get(chatPath) !== null) throw new JourneyError("unsupported", "fresh_agent_chat_required"); + await session.own({ kind: "issue", name: `Chat with ${agent.name}`, journeyId: "chat-continuity", freshChat }); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(context.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + await sendChatMessage(context.page, scenario.remember); + const first = await wait(1, 0), initial = await snapshot("initial", first, newProviderRun(first).id); + await context.page.goto(`${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/issues`, { waitUntil: "domcontentloaded", timeout: remaining() }); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await context.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(context.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + await snapshot("returned", await wait(1, 0), initial.reply!.runId); + await sendChatMessage(context.page, scenario.recall); + const second = await wait(2, 0), recalled = await snapshot("recalled", second, newProviderRun(second, initial.runs).id); + await sendChatMessage(context.page, scenario.reset); + await snapshot("reset", await wait(2, 1)); + await sendChatMessage(context.page, scenario.fresh); + const third = await wait(3, 1), freshRun = newProviderRun(third, recalled.runs); + await snapshot("fresh", third, freshRun.id); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await snapshot("reopened", await wait(3, 1), freshRun.id); + } finally { + evidence.checks.push(...gradeChatContinuity(scope, points)); + await retain(); + await reader.preserve(options.directory); + } + if (evidence.checks.some(check => !check.passed)) throw new JourneyError("failed", "chat_continuity_evidence_failed"); +} diff --git a/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts b/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts new file mode 100644 index 00000000000..a236106b597 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts @@ -0,0 +1,66 @@ +import { chatContinuityScenario, type ChatContinuityPoint, type ChatContinuityScope } from "./chat-continuity-evidence.js"; + +type Row = Record; +export function chatContinuityFixture() { + const scope: ChatContinuityScope = { companyId: "company", userId: "board", agentId: "agent", agentName: "Owned agent", + environmentId: "environment", provider: "opencode", model: "configured-model", campaignId: "synthetic-continuity-123", + agentHistory: { source: "company-agent-runs", companyId: "company", agentId: "agent", limit: 1, runIds: [], observedBeforeSend: true } }; + const scenario = chatContinuityScenario(scope.campaignId); + const user = (id: string, body: string, generation: number) => ({ id, body, authorUserId: scope.userId, + authorAgentId: null, companyId: scope.companyId, issueId: "chat", conversationSessionGeneration: generation }); + const answer = (id: string, runId: string, body: string, generation: number) => ({ id, body, + authorAgentId: scope.agentId, authorUserId: null, createdByRunId: runId, + companyId: scope.companyId, issueId: "chat", conversationSessionGeneration: generation }); + const prompts = [scenario.remember, scenario.recall, scenario.reset, scenario.fresh]; + const board = prompts.map((body, index) => user(`user-${index}`, body, index >= 2 ? 1 : 0)); + const replies = [answer("reply-0", "run-0", "Remembered.", 0), answer("reply-1", "run-1", scenario.remembered, 0), + answer("reply-3", "run-3", scenario.marker, 1)]; + const run = (index: number): Row => ({ id: `run-${index}`, companyId: scope.companyId, agentId: scope.agentId, + nativeIssueId: "chat", status: "succeeded", runtimeMode: "native", nativePhase: "committed", + nativeSessionId: index === 3 ? "new-session" : "original-session", sessionIdBefore: null, + contextSnapshot: { issueId: "chat", source: "issue.comment", wakeCommentId: board[index]!.id, + conversationSessionGeneration: index >= 2 ? 1 : 0, paperclipTaskMarkdown: index === 3 ? scenario.fresh : scenario.remember, + paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } }, + resultJson: { finalizationPhase: "committed", presentationDecision: { commentId: `reply-${index}` } } }); + const runs = [run(0), run(1), run(2), run(3)]; + runs[2]!.contextSnapshot.conversationReset = true; + runs[2]!.resultJson = { conversationReset: true }; + delete runs[2]!.runnerProfileJson; + delete runs[2]!.nativeSessionId; + delete runs[2]!.nativePhase; + delete runs[2]!.runtimeMode; + const chat = { id: "chat", companyId: scope.companyId, assigneeAgentId: scope.agentId, conversationAgentId: scope.agentId, + conversationUserId: scope.userId, createdByUserId: scope.userId, title: `Chat with ${scope.agentName}`, + status: "in_review", conversationState: "waiting", conversationSessionGeneration: 0 }; + const comments = [board[0]!, replies[0]!, board[1]!, replies[1]!, board[2]!, board[3]!, replies[2]!]; + const makePoint = (phase: ChatContinuityPoint["phase"], count: number, commentCount: number, generation: number, replyIndex?: number): ChatContinuityPoint => { + const reply = replyIndex === undefined ? undefined : replies[replyIndex]!; + return structuredClone({ phase, chat: { ...chat, conversationSessionGeneration: generation }, runs: runs.slice(0, count), + comments: comments.slice(0, commentCount), tasks: [], activity: [], events: [], resetReceipts: count < 3 ? [] : [{ + source: "issue-runs", companyId: scope.companyId, issueId: "chat", runId: "run-2", agentId: scope.agentId, + status: "succeeded", contextIssueId: "chat", wakeCommentId: "user-2", resultJson: { conversationReset: true }, + }], + ...(reply ? { reply: { runId: reply.createdByRunId, agentId: scope.agentId, observedAfterReload: true, + body: reply.body, anchor: reply.id, visible: true, settled: true, nativeFinal: false } } : {}) }); + }; + const points = [makePoint("initial", 1, 2, 0, 0), makePoint("returned", 1, 2, 0, 0), makePoint("recalled", 2, 4, 0, 1), + makePoint("reset", 3, 5, 1), makePoint("fresh", 4, 7, 1, 2), makePoint("reopened", 4, 7, 1, 2)]; + return { scope, scenario, points }; +} + +export function compactChatContinuityFixture() { + const fixture = chatContinuityFixture(); + for (const point of fixture.points) { + for (const run of point.runs) run.resultJson = null; + if (!point.reply) continue; + point.reply.anchor = `${point.reply.runId}:turn`; + point.reply.nativeFinal = true; + point.events = [{ seq: 1, runId: point.reply.runId, companyId: fixture.scope.companyId, agentId: fixture.scope.agentId, + eventType: "item.completed", protocolSchemaVersion: 1, sourceEventId: "event", sourceInstanceId: "instance", sourceSeq: 1, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: point.reply.runId, sourceKind: "runner", + eventType: "item.completed", sourceEventId: "event", sourceInstanceId: "instance", sourceSeq: 1, + payload: { kind: "agentMessage", channel: "final", text: point.reply.body } } } }]; + } + return fixture; +} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index a1bd122db02..6c7dc9c2930 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -18,6 +18,7 @@ import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; +import { runChatContinuity } from "./chat-continuity-flow.js"; import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; import { runInputFile } from "./input-file-flow.js"; @@ -164,6 +165,9 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", + "chat-continuity-flow.ts", + "chat-continuity-evidence.ts", + "../chat-cases.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", @@ -297,6 +301,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri await mkdir(directory, { recursive: true, mode: 0o700 }); await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); + else if (id === "chat-continuity") await runChatContinuity(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else await runAgentJourney(id, ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { diff --git a/tests/runner-e2e/product-journeys/product-screenshot.test.ts b/tests/runner-e2e/product-journeys/product-screenshot.test.ts index 486b6c19f0e..97b8b5134db 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.test.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.test.ts @@ -13,12 +13,12 @@ function fixture(options: { mounted?: boolean; redirect?: string; initialPath?: return { page, locator, waitFor, screenshot }; } describe("product screenshot boundary", () => { - it.each(["/QA/inbox/unread", "/QA/routines"])("retains an authenticated journey failure on %s", async initialPath => { + it.each(["/QA/inbox/unread", "/QA/routines", "/QA/chats/owned-agent"])("retains an authenticated journey failure on %s", async initialPath => { const f = fixture({ initialPath }); await captureProductScreenshot(f.page, origin, "failed.png"); expect(f.screenshot).toHaveBeenCalledTimes(1); }); - it.each(["/login", "/QA/settings/keys", "/QA/inboxing", "/QA/routines-export"])("excludes unrelated or authentication paths: %s", async initialPath => { + it.each(["/login", "/QA/settings/keys", "/QA/inboxing", "/QA/routines-export", "/QA/chats-export"])("excludes unrelated or authentication paths: %s", async initialPath => { const f = fixture({ initialPath }); await expect(captureProductScreenshot(f.page, origin, "failed.png")).rejects.toThrow("product_screen_unavailable"); expect(f.waitFor).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.ts b/tests/runner-e2e/product-journeys/product-screenshot.ts index cedfe14a17a..c7d352c4102 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.ts @@ -3,7 +3,7 @@ import { JourneyError } from "./types.js"; export function isProductScreenshotUrl(value: string, origin: string): boolean { const url = new URL(value); - return url.origin === origin && /^\/[^/]+\/(issues|search|inbox|routines)(\/|$)/.test(url.pathname); + return url.origin === origin && /^\/[^/]+\/(issues|search|inbox|routines|chats)(\/|$)/.test(url.pathname); } export async function captureProductScreenshot(page: Page, origin: string, file: string, deadlineAt = Date.now() + 10_000) { diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index d5d9a15b874..7369165693a 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -4,6 +4,7 @@ export const agentJourneyRuns = { "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, + "chat-continuity": 4, "plan-revise-accept": 3, "build-revise": 2, "followup-while-running": 2, @@ -34,6 +35,7 @@ export const journeyRequiredChecks: Record = { "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], + "chat-continuity": ["chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment"], "question-answer-resume": ["question-resume.forms", "question-resume.choice.answer", "question-resume.choice.path", "question-resume.text.answer", "question-resume.text.path", "question-resume.complete-evidence", "question-no-premature-output", "question-answers-owned-by-board", "question-document-attributed", "question-document-reloaded", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index eb79fd03411..cc92860f48b 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -9,6 +9,7 @@ export type JourneyId = | "project-task-context" | "task-document-persistence" | "ask-question" + | "chat-continuity" | "cloud-onboarding" | "question-answer-resume" | "task-search-filter" From e2dc2cc0be8dc80a1fe3e88bc04b78d7178fc1e8 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:44:12 -0600 Subject: [PATCH 15/28] fix(native): retire idle shared-sandbox ingress owners before dispatch (#18893) ## Thinking Path > - Paperclip manages agents and their tasks. > - Native tasks can share one reusable sandbox workspace. > - A completed child can keep its provider process warm. > - A parent has its own logical session, but the sandbox has one ingress listener. > - Retirement by task identity leaves the child's listener in place. > - A new parent Runner then fails to bind that listener. > - This pull request retires a proven idle physical owner before the parent's distinct session starts. ## Linked Issues or Issue Description Refs #15353 and #17097 for listener configuration and warm transport rotation. Refs #18682 for owned process retirement. #18860 repairs a different resume-source failure. Searches found no existing repair for retained warm sessions on different tasks that share the same physical ingress allocation. **What happened?** A completed child and a later parent can use the same reusable sandbox. Warm-session retirement compares logical task ownership. It does not find the child's distinct task scope. The child keeps the fixed ingress listener, and the parent's new process fails with `runner_ingress_bind_conflict` / address already in use. **Expected behavior** A parent must start in its own logical session after the proven idle prior listener closes. Active, foreign or unverified owners must remain protected. **Steps to reproduce** Run the `retires the completed child listener` cases in `native-session-executor.test.ts`. The actual executor registers a warm child. PostgreSQL holds its committed, accounted result and clean expired lease. A distinct parent receives the same allocation and current workspace writer. The provider boundary uses real TCP listeners. The original executor reaches the second bind with the child still listening and fails with EADDRINUSE. The changed executor waits for the owned close, then starts the distinct parent session. **Paperclip version or commit** Reproduced on ab2d2098a2764c511978d1c3e7a3fc5dfcef07f4. The branch includes current master 44a4ca1c61 before this PR was opened. **Deployment mode** Native Runner with provider ingress on a reusable sandbox workspace. Local regression uses real PostgreSQL and local TCP with a controlled provider-process boundary. No hosted provider call was made. ## What Changed - Keep physical ingress identity separate from logical task/session ownership. Never attach the parent to the child's session. - Reserve a physical allocation during local admission. Retire a distinct warm owner only after canonical company, agent, workspace, provider, allocation, lease, task and coordinator checks pass. - Require the current remote workspace writer and location to match the admitted run. Require the prior run to be committed and accounted, with its original controller generation and a clean released or expired lease. - Use the existing close, checkpoint and stopped-file collection path. Wait for close before parent dispatch, then recheck canonical authority before staging files or calling the parent backend. Retain the entry when close is uncertain. - Add actual executor/PostgreSQL/TCP safety cases and document the contract. Keep maps unchanged for this bug fix. ## Verification - Untouched production at ab2d2098: actual executor + PostgreSQL + TCP fails EADDRINUSE. Same positive case passes after the change. - Final focused suite: 20 passed. It covers distinct parent session identity; foreign company, allocation and workspace; active or unaccounted owner; changed controller; reassignment; competing lease/writer; stale allocation metadata; Stop before and after claim; workspace-lock race; failed close that leaves the listener and accepted result intact; and controller transfer, Stop or writer change during awaited close. These last cases reject the stale parent before backend invocation after the old listener closes. - Untouched head3425acbe: transferring the incoming coordinator during close still reaches the parent backend. The same actual executor/PostgreSQL/TCP case passes after the post-close authority check. - Broader `native warm session supervision` suite: 160 passed. This run preceded the final two-line logical-session collision guard; the final 20-case suite includes that guard. - `pnpm --filter @paperclipai/server typecheck`: all source and test layers passed, including `src/__tests__`. - `pnpm --filter @paperclipai/server build`: production server and Runner build passed at current head572acfef79. The exact-head build stamp is recorded. - The provider/close boundary is controlled. The fixture seeds canonical committed/accounted finalization data in PostgreSQL. These tests do not claim a live model result, operator ACK, hosted PID-to-listener identity, or recovery of a historical task. - Full repository pnpm checks were not run locally for this server-only slice. Required current-head CI and reviews are pending. ## Risks The new retirement path can close a process from another task. It requires the same canonical company, agent, allocation and workspace, a current exclusive writer, and a completed idle owner. Missing or changed evidence leaves admission blocked. Active owners, changed generations, reassignment and cancellation remain fenced. A failed close retains the owner and saved result. This covers current-controller warm entries with recorded physical identity. Historical processes outside the registry keep the existing exact-owner recovery requirements. No port shuffle, port kill, task capability expansion, new polling loop or invented receipt is added. ## Model Used OpenAI GPT-6.1 Sol (`gpt-6.1-sol`) in Codex, with source editing, terminal execution and tool use. The host did not report its context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/execution-semantics.md | 20 +++ doc/testing-native-remote-continuity.md | 23 +++ .../helpers/native-warm-ingress-fixture.ts | 64 +++++++++ .../native-session-executor.test.ts | 132 ++++++++++++++++++ .../native-runtime/native-session-executor.ts | 48 ++++++- .../native-warm-ingress-ownership.ts | 99 +++++++++++++ 6 files changed, 381 insertions(+), 5 deletions(-) create mode 100644 server/src/__tests__/helpers/native-warm-ingress-fixture.ts create mode 100644 server/src/services/native-runtime/native-warm-ingress-ownership.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 336687a07a6..ec5752baa18 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2621,6 +2621,26 @@ configuration, and controller-owned credential or instruction resources use verified suspension and saved-history resume. This recovery can restart a provider; it does not suspend every healthy warm turn. +A different task can reuse the same physical sandbox after its prior task +finishes. Provider ingress has one listener per allocation. Before a new logical +session starts, Runner retires a locally retained idle session that still owns +that listener. The new run must hold the current native claim, task assignment, +environment lease and remote workspace writer. Company, agent, provider, +allocation, workspace and directory must match. The prior run must be committed +and accounted, with its original controller generation and a clean released or +expired lease. Retirement saves the prior session through its own checkpoint and +file-collection callbacks and waits for close before the new listener binds. +After close, admission rechecks the canonical native claim, task, workspace and +leases before staging files or invoking the new backend. Ownership loss, Stop +or a changed writer during retirement cannot authorize the new execution. +The new task keeps its own logical session and authority. + +Active owners, a changed controller or writer, missing canonical proof and failed +close block admission. A retained entry stays available for verified retirement; +no process is selected by its port number. This rule covers current-controller +warm entries with recorded physical identity. A historical process outside that +registry still needs the existing exact-owner recovery proof. + A fresh native session admitted for the current task may replace a terminal predecessor's goal source epoch when the saved goal is absent or complete. It must hold the task's execution lock and assignee. Its accepted terminal diff --git a/doc/testing-native-remote-continuity.md b/doc/testing-native-remote-continuity.md index bd4fb833711..810c16e9060 100644 --- a/doc/testing-native-remote-continuity.md +++ b/doc/testing-native-remote-continuity.md @@ -540,3 +540,26 @@ explicit incomplete usage report before failure; this preserves known budget exposure without settling ambiguous usage. That required report and its receipt retain the accounting measurements and matching saved prior identity, so an optional diagnostic acknowledgement cannot discard reconciliation evidence. + +## Shared allocation ingress retirement + +The `retires the completed child listener` cases in +`native-session-executor.test.ts` use the actual executor and warm registry, +PostgreSQL ownership rows, and real local TCP listeners. The provider process +boundary is supplied by the fixture. The listeners use an ephemeral test port; +production provider ingress keeps its fixed port. A completed child's listener +must close before a distinct parent session binds in that allocation. The child's +accepted result stays unchanged. + +```sh +pnpm --filter @paperclipai/server exec vitest run \ + src/services/native-runtime/native-session-executor.test.ts \ + -t 'retires the completed child listener' +``` + +Controls cover foreign companies, allocations and workspaces; live or unaccounted +owners; changed controllers, reassignment and competing leases; workspace-lock +and cancellation races; unconfirmed close; and controller, Stop and writer +changes inside awaited close. The latter must prevent parent backend invocation +even when the old listener has closed. These local checks do not identify +a hosted conflicting PID or prove a hosted provider Stop or recovery. diff --git a/server/src/__tests__/helpers/native-warm-ingress-fixture.ts b/server/src/__tests__/helpers/native-warm-ingress-fixture.ts new file mode 100644 index 00000000000..15f904de509 --- /dev/null +++ b/server/src/__tests__/helpers/native-warm-ingress-fixture.ts @@ -0,0 +1,64 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import * as tables from "@paperclipai/db"; +import type { NativeExecutionInputV1, NativeExecutionInputV3 } from "@paperclipai/paperclip-runner"; +import { startRecoveryPostgres } from "./recovery-postgres.js"; +import { prepareNativeHeartbeatRun } from "../../services/native-runtime/prepare-native-run.js"; +import { buildNativeCompletionContract } from "../../services/native-runtime/completion-contracts.js"; +import { nativeRuntimeContextFixture } from "../../services/native-runtime/runtime-context.test-fixture.js"; + +export async function createNativeWarmIngressFixture(base: NativeExecutionInputV1, cwd: string) { + const database = await startRecoveryPostgres("shared-ingress-owner-"); + const db = tables.createDb(database.connectionString); + const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), workspaceId = randomUUID(); + const providerLeaseId = `allocation-${randomUUID()}`; + await db.insert(tables.companies).values({ id: companyId, name: "Shared ingress", issuePrefix: `I${companyId.slice(0, 7)}` }); + await db.insert(tables.agents).values({ id: agentId, companyId, name: "Shared worker", adapterType: "paperclip_runner" }); + await db.insert(tables.environments).values({ id: environmentId, name: "Shared allocation", driver: "sandbox" }); + await db.insert(tables.executionWorkspaces).values({ id: workspaceId, companyId, name: "Shared workspace", mode: "shared_workspace", + strategyType: "project_primary", providerType: "cloud_sandbox", cwd }); + const defaultCompanyId = companyId, defaultAgentId = agentId, defaultWorkspaceId = workspaceId, defaultProviderLeaseId = providerLeaseId; + let epoch = 0; + async function admit(options?: { companyId?: string; agentId?: string; workspaceId?: string; providerLeaseId?: string }) { + const ownerCompanyId = options?.companyId ?? defaultCompanyId, ownerAgentId = options?.agentId ?? defaultAgentId; + const ownerWorkspaceId = options?.workspaceId ?? defaultWorkspaceId, ownerProviderLeaseId = options?.providerLeaseId ?? defaultProviderLeaseId; + const companyId = ownerCompanyId, agentId = ownerAgentId, workspaceId = ownerWorkspaceId, providerLeaseId = ownerProviderLeaseId; + const issueId = randomUUID(), runId = randomUUID(), leaseId = randomUUID(); + const [issue] = await db.insert(tables.issues).values({ id: issueId, companyId, title: "Shared handoff", status: "in_progress", + assigneeAgentId: agentId, executionWorkspaceId: workspaceId }).returning(); + const [run] = await db.insert(tables.heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, + status: "running", runtimeMode: "native", contextSnapshot: { issueId } }).returning(); + await db.update(tables.issues).set({ executionRunId: runId }).where(eq(tables.issues.id, issueId)); + await db.insert(tables.environmentLeases).values({ id: leaseId, companyId, environmentId, executionWorkspaceId: workspaceId, + issueId, heartbeatRunId: runId, provider: "daytona", providerLeaseId, leasePolicy: "reuse_by_environment" }); + await db.insert(tables.nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: leaseId }); + const [contract] = await db.select().from(tables.completionContracts).where(eq(tables.completionContracts.issueId, issueId)); + const execution: NativeExecutionInputV3 = { ...base, schema: "paperclip.native-execution-input.v3", executionMode: "default", planningContext: null, + runtimeContext: nativeRuntimeContextFixture(), binding: { companyId, agentId, issueId, runId, executionWorkspaceId: workspaceId }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: buildNativeCompletionContract(issue, { revision: contract.revision }) }, + workspace: { cwd, repoUrl: null, repoRef: null, branchName: null }, + session: { ...base.session, normalizedSessionId: `logical-${issueId}`, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } } }; + await db.update(tables.executionWorkspaces).set({ metadata: { remoteWorkspace: { + schema: "paperclip.remote-workspace.v1", epoch: ++epoch, snapshotId: null, source: { kind: "empty" }, writerRunId: runId, dirty: true, + location: { environmentId, leaseId, providerLeaseId, remoteCwd: cwd }, + } } }).where(eq(tables.executionWorkspaces.id, workspaceId)); + const target = { kind: "remote" as const, transport: "sandbox" as const, providerKey: "daytona", environmentId, leaseId, remoteCwd: cwd, + effectiveCapabilities: { runnerWebSocketIngress: true } as never, + sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId } }; + return { execution, target, issueId, runId, leaseId }; + } + async function settle(owner: Awaited>, result: Record) { + const resultId = randomUUID(); + await db.insert(tables.nativeRunResults).values({ id: resultId, companyId, issueId: owner.issueId, runId: owner.runId, + completionContractId: owner.execution.completionContract.id, schemaStatus: "accepted", resultJson: result, + serverFingerprint: randomUUID(), canonicalSha256: randomUUID() }); + await db.update(tables.nativeRunFinalizations).set({ phase: "committed", resultId, leaseOwner: null, leaseExpiresAt: null }).where(eq(tables.nativeRunFinalizations.runId, owner.runId)); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), nativePhase: "committed", costAccountingPending: false }).where(eq(tables.heartbeatRuns.id, owner.runId)); + await db.update(tables.issues).set({ status: "done", executionRunId: null, checkoutRunId: null }).where(eq(tables.issues.id, owner.issueId)); + await db.update(tables.environmentLeases).set({ status: "expired", cleanupStatus: "success", releasedAt: new Date() }).where(eq(tables.environmentLeases.id, owner.leaseId)); + return resultId; + } + return { db, companyId, agentId, environmentId, workspaceId, providerLeaseId, admit, settle, cleanup: () => database.cleanup() }; +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index ef43bf02e7f..e3611e34991 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -10011,6 +10011,138 @@ describe("native warm session supervision", () => { expect(state.execute).toHaveBeenCalledTimes(2); }); + it.each(["completed", "different-allocation", "foreign-company", "different-workspace", "live-owner", "unaccounted-owner", "changed-controller", "forged-allocation", "foreign-writer", "foreign-active-lease", "reassigned-child", "parent-already-started", "writer-lock-race", "stop-after-claim", "controller-change-during-close", "stop-during-close", "writer-change-during-close", "retirement-fails", "foreign-logical-session", "stopped-parent"] as const)("retires the completed child listener before a different parent task uses the same sandbox: %s", async scenario => { + const { createServer } = await import("node:net"); + const { createNativeWarmIngressFixture } = await import("../../__tests__/helpers/native-warm-ingress-fixture.js"); + const { eq } = await import("drizzle-orm"); + const tables = await import("@paperclipai/db"); + const f = await createNativeWarmIngressFixture(execution, join(isolatedHome, "shared-workspace")); + await mkdir(join(isolatedHome, "shared-workspace")); + const listener = createServer(); + await new Promise((resolve, reject) => { listener.once("error", reject); listener.listen(0, "127.0.0.1", resolve); }); + const port = (listener.address() as import("node:net").AddressInfo).port; + let failClose = scenario === "retirement-fails"; + let unlockWriter: (() => void) | undefined; + let writerTransaction: Promise | undefined; + let incomingRunId: string | undefined; + const close = vi.fn(async () => { + if (failClose) throw new Error("owned close is unconfirmed"); + if (scenario === "controller-change-during-close" && incomingRunId) await f.db.update(nativeRunFinalizations) + .set({ leaseOwner: "replacement-owner", controllerBootId: "replacement-controller", controllerGeneration: 999 }) + .where(eq(nativeRunFinalizations.runId, incomingRunId)); + if (scenario === "stop-during-close" && incomingRunId) await f.db.update(heartbeatRuns) + .set({ status: "cancelled" }).where(eq(heartbeatRuns.id, incomingRunId)); + if (scenario === "writer-change-during-close" && incomingRunId) { + const [workspace] = await f.db.select().from(tables.executionWorkspaces).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + await f.db.update(tables.executionWorkspaces).set({ metadata: { ...workspace.metadata, + remoteWorkspace: { ...(workspace.metadata!.remoteWorkspace as Record), writerRunId: randomUUID() } } }) + .where(eq(tables.executionWorkspaces.id, f.workspaceId)); + } + if (listener.listening) await new Promise((resolve, reject) => listener.close(error => error ? reject(error) : resolve())); + }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); return result; + }).mockImplementationOnce(async options => { + expect(options.existingSession).toBeUndefined(); + const parentListener = createServer(); + try { + await new Promise((resolve, reject) => { parentListener.once("error", reject); parentListener.listen(scenario === "different-allocation" ? 0 : port, "127.0.0.1", resolve); }); + } finally { + if (parentListener.listening) await new Promise(resolve => parentListener.close(() => resolve())); + } + return result; + }); + try { + const child = await f.admit(); + await executePaperclipNativeSession({ db: f.db, execution: child.execution, runnerInstanceId: "child-runner", useRunnerd: true, + backend: {} as never, runnerExecutionTarget: child.target }); + const resultId = await f.settle(child, result); + const options: Parameters[0] = {}; + if (scenario === "different-allocation") options.providerLeaseId = `different-${f.providerLeaseId}`; + if (scenario === "different-workspace" || scenario === "foreign-company") { + options.workspaceId = randomUUID(); + if (scenario === "foreign-company") { + options.companyId = randomUUID(); options.agentId = randomUUID(); + await f.db.insert(tables.companies).values({ id: options.companyId, name: "Foreign tenant", issuePrefix: `F${options.companyId.slice(0, 7)}` }); + await f.db.insert(tables.agents).values({ id: options.agentId, companyId: options.companyId, name: "Foreign worker", adapterType: "paperclip_runner" }); + } + await f.db.insert(tables.executionWorkspaces).values({ id: options.workspaceId, companyId: options.companyId ?? f.companyId, + name: "Other workspace", mode: "shared_workspace", strategyType: "project_primary", providerType: "cloud_sandbox", cwd: child.execution.workspace.cwd }); + } + const parent = await f.admit(options); + incomingRunId = parent.runId; + if (scenario === "live-owner") await f.db.update(heartbeatRuns).set({ status: "running", nativePhase: "observed", finishedAt: null }).where(eq(heartbeatRuns.id, child.runId)); + if (scenario === "unaccounted-owner") await f.db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, child.runId)); + if (scenario === "changed-controller") await f.db.update(nativeRunFinalizations).set({ controllerBootId: "foreign-controller" }).where(eq(nativeRunFinalizations.runId, child.runId)); + if (scenario === "forged-allocation") await f.db.update(environmentLeases).set({ providerLeaseId: "canonical-other-allocation" }).where(eq(environmentLeases.id, parent.leaseId)); + if (scenario === "foreign-writer") { + const [workspace] = await f.db.select().from(tables.executionWorkspaces).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + await f.db.update(tables.executionWorkspaces).set({ metadata: { ...workspace.metadata, + remoteWorkspace: { ...(workspace.metadata!.remoteWorkspace as Record), writerRunId: randomUUID() } } }).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + } + if (scenario === "reassigned-child") await f.db.update(tables.issues).set({ assigneeAgentId: null }).where(eq(tables.issues.id, child.issueId)); + if (scenario === "parent-already-started") await f.db.update(heartbeatRuns).set({ processPid: 2147483000 }).where(eq(heartbeatRuns.id, parent.runId)); + if (scenario === "foreign-active-lease") { + const issueId = randomUUID(), runId = randomUUID(); + await f.db.insert(tables.issues).values({ id: issueId, companyId: f.companyId, title: "Live sibling", assigneeAgentId: f.agentId, status: "in_progress" }); + await f.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, nativeIssueId: issueId, runtimeMode: "native", status: "running" }); + await f.db.insert(environmentLeases).values({ companyId: f.companyId, environmentId: f.environmentId, executionWorkspaceId: f.workspaceId, + issueId, heartbeatRunId: runId, provider: "daytona", providerLeaseId: f.providerLeaseId, status: "active" }); + } + if (scenario === "foreign-logical-session") parent.execution.session.normalizedSessionId = child.execution.session.normalizedSessionId; + if (scenario === "stopped-parent") await f.db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, parent.runId)); + let claimed!: () => void; + const claimObserved = new Promise(resolve => { claimed = resolve; }); + if (scenario === "writer-lock-race") { + let locked!: () => void; + const lockObserved = new Promise(resolve => { locked = resolve; }); + const release = new Promise(resolve => { unlockWriter = resolve; }); + writerTransaction = f.db.transaction(async tx => { + const [workspace] = await tx.select().from(tables.executionWorkspaces).where(eq(tables.executionWorkspaces.id, f.workspaceId)).for("update"); + locked(); await release; + await tx.update(tables.executionWorkspaces).set({ metadata: { ...workspace.metadata, + remoteWorkspace: { ...(workspace.metadata!.remoteWorkspace as Record), writerRunId: randomUUID() } } }).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + }); + await lockObserved; + } + const start = executePaperclipNativeSession({ db: f.db, execution: parent.execution, runnerInstanceId: "parent-runner", useRunnerd: true, + backend: {} as never, runnerExecutionTarget: parent.target, onExecutionLeaseAcquired: async () => { + claimed(); + if (scenario === "stop-after-claim") await f.db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, parent.runId)); + } }); + const outcome = start.then(value => ({ value }), error => ({ error })); + if (scenario === "writer-lock-race") { + await claimObserved; + expect(close).not.toHaveBeenCalled(); + unlockWriter!(); await writerTransaction; + } + if (scenario === "completed" || scenario === "different-allocation") { + await start; + expect(close).toHaveBeenCalledTimes(scenario === "completed" ? 1 : 0); + expect(state.execute).toHaveBeenCalledTimes(2); + const admittedParent = state.execute.mock.calls[1][0]; + expect(admittedParent.input.binding.issueId).toBe(parent.issueId); + expect(admittedParent.input.session.normalizedSessionId).toBe(parent.execution.session.normalizedSessionId); + expect(admittedParent.input.session.normalizedSessionId).not.toBe(child.execution.session.normalizedSessionId); + expect(admittedParent.existingSession).toBeUndefined(); + } else { + await expect(start).rejects.toThrow(); + expect(close).toHaveBeenCalledTimes(["retirement-fails", "controller-change-during-close", "stop-during-close", "writer-change-during-close"].includes(scenario) ? 1 : 0); + expect(listener.listening).toBe(!["controller-change-during-close", "stop-during-close", "writer-change-during-close"].includes(scenario)); + expect(state.execute).toHaveBeenCalledOnce(); + } + await outcome; + const [retained] = await f.db.select().from(nativeRunResults).where(eq(nativeRunResults.id, resultId)); + expect(retained.resultJson).toEqual(result); + } finally { + failClose = false; + unlockWriter?.(); await writerTransaction; + await closeWarmNativeSessionsForEnvironment({ environmentId: f.environmentId, reason: "shared ingress fixture cleanup" }); + if (listener.listening) await new Promise(resolve => listener.close(() => resolve())); + await f.cleanup(); + } + }, 60_000); + it("claims native execution authority before retiring a superseded warm identity", async () => { const { base, next, run } = fixture("identity-native-claim-order"); const updates: Array<{ table: unknown; values: Record }> = []; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 6bbff7593c7..a2600506bd0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -229,6 +229,7 @@ import { } from "./remote-runner-recovery-diagnostics.js"; import { readRemoteRunnerRecoveryProbe, REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS } from "./remote-runner-recovery-probe.js"; import { inspectRemoteWarmSessionRecovery, isSettledRemoteWarmControlState, type RemoteWarmSessionRecovery } from "./remote-warm-session-recovery.js"; +import { assertNativeWarmIngressRetirement, nativeWarmIngressAllocation, nativeWarmIngressIdentity, type NativeWarmIngressClaim, type NativeWarmIngressOwner } from "./native-warm-ingress-ownership.js"; import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; import { emitStartupReuse, environmentStartupReuse } from "@paperclipai/adapter-utils/startup-reuse"; import { persistActivity, publishActivity } from "../activity-log.js"; @@ -582,6 +583,7 @@ export type NativeInstructionWorkingCopy = { }; type WarmNativeSession = { + ingressOwner?: NativeWarmIngressOwner; onReleased: () => void; sessionCheckpointPersisted?: (snapshot: PersistedNativeSession) => void; pluginTaskId?: string; @@ -1858,6 +1860,7 @@ const sessionToolAuthorityEpochs = new Map(); const initializingSessionToolAuthorities = new Set(); const executingRunnerdSessionScopes = new Map(); const executingNativeOwnerScopes = new Map(); +const executingNativeIngressAllocations = new Map(); export function nativeSessionScopeDigestIsBusy(digest: string): boolean { const scopes = [ @@ -1964,21 +1967,32 @@ async function retireSupersededWarmNativeSessions( ownerScope: string, nextSessionScope: string, nextSessionId: string, + ingress?: { db: Db; claim: NativeWarmIngressClaim }, ): Promise { for (const [scope, entry] of warmNativeSessions) { - if (scope === nextSessionScope || entry.ownerScope !== ownerScope) continue; - if (entry.busy || executingRunnerdSessionScopes.has(scope)) { + if (scope === nextSessionScope) continue; + const logicalOwner = entry.ownerScope === ownerScope; + const priorIngress = entry.ingressOwner; + const physicalOwner = ingress && priorIngress && nativeWarmIngressAllocation(priorIngress.ingress) === nativeWarmIngressAllocation(ingress.claim.ingress); + if (!logicalOwner && !physicalOwner) continue; + if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(scope)) { throw new Error("native_session_supervisor_busy"); } + if (!logicalOwner) { + await assertNativeWarmIngressRetirement(ingress!.db, ingress!.claim, priorIngress!); + if (warmNativeSessions.get(scope) !== entry || entry.ingressOwner !== priorIngress + || entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(scope)) throw new Error("native_session_supervisor_busy"); + } if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); entry.idleTimer = null; entry.busy = true; - entry.closeOnReleaseReason = "warm native session identity changed"; + entry.closeOnReleaseReason = logicalOwner ? "warm native session identity changed" : "warm native ingress requires another session"; try { await closeWarmNativeSession(entry, entry.closeOnReleaseReason, { - discardUnsettledSession: entry.normalizedSessionId !== nextSessionId, + discardUnsettledSession: logicalOwner && entry.normalizedSessionId !== nextSessionId, }); forgetWarmNativeSession(scope, entry); + if (!logicalOwner) await assertNativeWarmIngressRetirement(ingress!.db, ingress!.claim, priorIngress!); } finally { // A failed close retains the owner and prevents launch. A later attempt // must retry retirement, never adopt this partially closed transport. @@ -8310,7 +8324,17 @@ export async function executePaperclipNativeSession(input: { input.runnerExecutionTarget?.environmentId ?? null, ); const ownerToken = Symbol("native execution owner"); + const physicalIngress = input.useRunnerd ? nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget) : undefined; + const physicalIngressKey = physicalIngress ? nativeWarmIngressAllocation(physicalIngress) : undefined; + let ownsPhysicalIngress = false; try { + if (physicalIngressKey) { + const retained = warmNativeSessions.get(nativeSessionScopeKey(input.execution)); + if (retained && retained.issueId !== input.execution.binding.issueId) throw new Error("native_session_supervisor_busy"); + if (executingNativeIngressAllocations.has(physicalIngressKey)) throw new Error("native_session_supervisor_busy"); + executingNativeIngressAllocations.set(physicalIngressKey, ownerToken); + ownsPhysicalIngress = true; + } // Reserve across session identities before retiring or staging anything. if (executingNativeOwnerScopes.has(ownerScope)) { throw new Error("native_session_supervisor_busy"); @@ -8361,7 +8385,8 @@ export async function executePaperclipNativeSession(input: { const workspaceRoot = input.execution.workspace.cwd; if (workspaceRoot === null) throw new Error("native_workspace_required"); const needsWarmRetirement = [...warmNativeSessions].some(([scope, entry]) => - scope !== sessionScopeId && entry.ownerScope === ownerScope); + scope !== sessionScopeId && (entry.ownerScope === ownerScope || physicalIngress && entry.ingressOwner + && nativeWarmIngressAllocation(entry.ingressOwner.ingress) === nativeWarmIngressAllocation(physicalIngress))); const prepareAttachments = async () => { const admissionSha256 = nativeSha256(input.execution); const targetKind = input.runnerExecutionTarget?.kind ?? "local"; @@ -8440,6 +8465,9 @@ export async function executePaperclipNativeSession(input: { startupUnsettled = !(await startup.executionContext.drain()); startup.executionContext.finish(); } + if (ownsPhysicalIngress && physicalIngressKey && executingNativeIngressAllocations.get(physicalIngressKey) === ownerToken) { + executingNativeIngressAllocations.delete(physicalIngressKey); + } if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) { executingNativeOwnerScopes.delete(ownerScope); } @@ -8936,9 +8964,12 @@ async function executePaperclipNativeSessionWithinScope( // Claim current native authority before the old warm checkpoint writer // must prove its same-controller successor. Shared staging stays after // strict predecessor suspension; it cannot run on a failed claim/Stop. + const ingress = nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget); if (input.useRunnerd && !claimedStopCleanup) await retireSupersededWarmNativeSessions( nativeSessionOwnerScope(input.execution, input.runnerExecutionTarget?.environmentId ?? null), nativeSessionScopeKey(input.execution), nativeSessionKey(input.execution), + ingress ? { db: input.db, claim: { ...input.execution.binding, ingress, leaseOwner, + controllerBootId: controller.bootId, controllerGeneration } } : undefined, ); claimedStartupContext?.assertOpen(); if (prepareAfterClaim) input = await prepareAfterClaim(); @@ -10030,11 +10061,18 @@ async function executePaperclipNativeSessionWithinScope( if (existing.session !== session) { existing.sessionCheckpointPersisted = runnerdBackend?.sessionCheckpointPersisted; } + const ingress = input.useRunnerd ? nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget) : undefined; + existing.ingressOwner = ingress ? { ...input.execution.binding, ingress, + controllerBootId: controller.bootId, controllerGeneration } : undefined; existing.session = session; existing.runId = input.execution.binding.runId; existing.issueId = input.execution.binding.issueId; } else warmNativeSessions.set(warmSessionId, { + ingressOwner: (() => { + const ingress = input.useRunnerd ? nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget) : undefined; + return ingress ? { ...input.execution.binding, ingress, controllerBootId: controller.bootId, controllerGeneration } : undefined; + })(), sessionCheckpointPersisted: runnerdBackend?.sessionCheckpointPersisted, onReleased: () => releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, diff --git a/server/src/services/native-runtime/native-warm-ingress-ownership.ts b/server/src/services/native-runtime/native-warm-ingress-ownership.ts new file mode 100644 index 00000000000..8a046006633 --- /dev/null +++ b/server/src/services/native-runtime/native-warm-ingress-ownership.ts @@ -0,0 +1,99 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; + +export type NativeWarmIngressIdentity = { + provider: string; + environmentId: string; + providerLeaseId: string; + leaseId: string; + workspaceId: string; + cwd: string; +}; +export type NativeWarmIngressOwner = { + companyId: string; + agentId: string; + issueId: string; + runId: string; + controllerBootId: string; + controllerGeneration: number; + ingress: NativeWarmIngressIdentity; +}; +export type NativeWarmIngressClaim = NativeWarmIngressOwner & { leaseOwner: string }; + +export function nativeWarmIngressIdentity(execution: NativeExecutionInput, target?: AdapterExecutionTarget | null): NativeWarmIngressIdentity | undefined { + if (target?.kind !== "remote" || target.transport !== "sandbox" || target.effectiveCapabilities?.runnerWebSocketIngress !== true + || !target.providerKey || !target.environmentId || !target.leaseId || !target.sandboxLeaseAcquisition?.providerLeaseId + || !("cwd" in execution.workspace) || !execution.workspace.cwd + || execution.binding.executionWorkspaceId === execution.binding.runId) return undefined; + return { provider: target.providerKey, environmentId: target.environmentId, + providerLeaseId: target.sandboxLeaseAcquisition.providerLeaseId, leaseId: target.leaseId, + workspaceId: execution.binding.executionWorkspaceId, cwd: execution.workspace.cwd }; +} + +export function nativeWarmIngressAllocation(identity: NativeWarmIngressIdentity): string { + return JSON.stringify([identity.provider, identity.environmentId, identity.providerLeaseId]); +} + +const busy = () => new Error("native_session_supervisor_busy"); +export async function assertNativeWarmIngressRetirement(db: Db, current: NativeWarmIngressClaim, previous: NativeWarmIngressOwner): Promise { + if (current.companyId !== previous.companyId || current.agentId !== previous.agentId + || current.ingress.workspaceId !== previous.ingress.workspaceId || current.ingress.cwd !== previous.ingress.cwd + || nativeWarmIngressAllocation(current.ingress) !== nativeWarmIngressAllocation(previous.ingress) + || current.runId === previous.runId) throw busy(); + await db.transaction(async tx => { + const [workspace] = await tx.select().from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, current.companyId), eq(executionWorkspaces.id, current.ingress.workspaceId), + )).for("update"); + const authority = readRemoteWorkspaceAuthority(workspace?.metadata); + if (!workspace || workspace.status !== "active" || authority?.writerRunId !== current.runId + || authority.location?.environmentId !== current.ingress.environmentId + || authority.location.providerLeaseId !== current.ingress.providerLeaseId + || authority.location.leaseId !== current.ingress.leaseId || authority.location.remoteCwd !== current.ingress.cwd) throw busy(); + const runs = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, current.companyId), + inArray(heartbeatRuns.id, [current.runId, previous.runId]))); + const incoming = runs.find(run => run.id === current.runId), prior = runs.find(run => run.id === previous.runId); + const coordinators = await tx.select().from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.companyId, current.companyId), + inArray(nativeRunFinalizations.runId, [current.runId, previous.runId]))); + const incomingCoordinator = coordinators.find(row => row.runId === current.runId), priorCoordinator = coordinators.find(row => row.runId === previous.runId); + const tasks = await tx.select().from(issues).where(and(eq(issues.companyId, current.companyId), inArray(issues.id, [current.issueId, previous.issueId]))); + const incomingTask = tasks.find(task => task.id === current.issueId), priorTask = tasks.find(task => task.id === previous.issueId); + const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, current.companyId), + inArray(environmentLeases.id, [current.ingress.leaseId, previous.ingress.leaseId]))); + const incomingLease = leases.find(lease => lease.id === current.ingress.leaseId), priorLease = leases.find(lease => lease.id === previous.ingress.leaseId); + const now = Date.now(); + if (!incoming || incoming.runtimeMode !== "native" || incoming.status !== "running" || incoming.nativePhase !== "observed" + || incoming.processPid !== null || incoming.processGroupId !== null || incoming.agentId !== current.agentId + || incoming.nativeIssueId !== current.issueId || incoming.resultJson?.startupCancellation || incoming.resultJson?.nativeCancellation + || incomingCoordinator?.issueId !== current.issueId || incomingCoordinator.phase !== "observed" || incomingCoordinator.resultId !== null || incomingCoordinator.leaseOwner !== current.leaseOwner + || incomingCoordinator.controllerBootId !== current.controllerBootId || incomingCoordinator.controllerGeneration !== current.controllerGeneration + || !incomingCoordinator.leaseExpiresAt || incomingCoordinator.leaseExpiresAt.getTime() <= now + || incomingTask?.assigneeAgentId !== current.agentId || incomingTask.executionRunId !== current.runId + || incomingTask.executionWorkspaceId !== current.ingress.workspaceId + || !["in_progress", "in_review"].includes(incomingTask.status) + || (incomingTask.checkoutRunId !== null && incomingTask.checkoutRunId !== current.runId) + || !prior || prior.runtimeMode !== "native" || prior.status !== "succeeded" || !prior.finishedAt || prior.costAccountingPending + || prior.agentId !== previous.agentId || prior.nativeIssueId !== previous.issueId || prior.nativePhase !== "committed" + || priorCoordinator?.issueId !== previous.issueId || priorCoordinator.phase !== "committed" || !priorCoordinator.resultId + || priorCoordinator.leaseOwner !== null || priorCoordinator.leaseExpiresAt !== null + || priorCoordinator.controllerBootId !== previous.controllerBootId || priorCoordinator.controllerGeneration !== previous.controllerGeneration + || !priorTask || priorTask.assigneeAgentId !== previous.agentId || priorTask.executionWorkspaceId !== previous.ingress.workspaceId + || priorTask.executionRunId !== null || priorTask.checkoutRunId !== null) throw busy(); + const leaseMatches = (lease: typeof environmentLeases.$inferSelect | undefined, owner: NativeWarmIngressOwner) => lease + && lease.heartbeatRunId === owner.runId && lease.issueId === owner.issueId + && lease.environmentId === owner.ingress.environmentId && lease.provider === owner.ingress.provider + && lease.providerLeaseId === owner.ingress.providerLeaseId && lease.executionWorkspaceId === owner.ingress.workspaceId; + if (!leaseMatches(incomingLease, current) || incomingLease!.status !== "active" + || (incomingLease!.expiresAt !== null && incomingLease!.expiresAt.getTime() <= now) + || !leaseMatches(priorLease, previous) || !["released", "expired"].includes(priorLease!.status) + || priorLease!.cleanupStatus !== "success") throw busy(); + const otherLeases = await tx.select({ runId: environmentLeases.heartbeatRunId }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, current.companyId), eq(environmentLeases.environmentId, current.ingress.environmentId), + eq(environmentLeases.provider, current.ingress.provider), eq(environmentLeases.providerLeaseId, current.ingress.providerLeaseId), + eq(environmentLeases.status, "active"), + )); + if (otherLeases.some(lease => lease.runId !== current.runId)) throw busy(); + }); +} From 58d1b210984d1c639ee3014a2c314b0ee4e231b9 Mon Sep 17 00:00:00 2001 From: Michael Nguyen <13559011+nguyenm7@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:51:29 -0800 Subject: [PATCH 16/28] fix: separate fresh setup from archived personal connections (#18902) ## Thinking Path > - Paperclip helps people manage AI agents and their service access. > - Each personal connection belongs to one user. > - Setup can recover an archived connection by its name and provider. > - A fresh setup can therefore select another user's old identity and fail its owner check. > - This change creates a separate connection for fresh setup after personal archival. > - Explicit reconnect still requires the existing owner and preserves its history. ## Linked Issues or Issue Description **What happened?** A fresh GitHub setup can return 403 after a personal connection with the same name was archived. The request does not include a reconnect target. Setup still selects the archived identity by name and provider. **Expected behavior** Fresh setup creates a separate personal identity. It leaves the archived connection, credentials, grants, and activity unchanged. Explicit reconnect rejects a different user. **Steps to reproduce** 1. Create a personal GitHub connection in a test company. 2. Archive that connection and its application. 3. As another authenticated member, start fresh personal setup with the same name and provider. 4. Observe the owner-check failure before this change. **Paperclip version or commit** Base: db9be4c5a. Tests use isolated database and HTTP fixtures. **Deployment mode** Authenticated. The legacy owner can be a local board identity. ## What Changed - Exclude implicitly recovered archived personal connections from fresh setup. - Use the existing company-scoped name allocator to create a separate application and connection. - Test three owners: a legacy board identity, another user, and the current user. - Check credential isolation, unchanged archived records and activity, missing-key rejection, and explicit reconnect denial. - Keep managed revival and generic MCP rollback/reconnect fixtures on explicit reconnect targets. - Verify generic personal URL setup creates an independent identity and preserves the archived records. - Document fresh setup and reconnect behavior. ## Verification - Final head: `96184e3179aab2eeb88df87d442691741a4875a2`. Full CI recovered and completed successfully: [run 41752164958, attempt 2](https://github.com/paperclipai/paperclip/actions/runs/41752164958/attempts/2). PR rollup: 53 successful checks, 3 skipped, none failing or pending. Typecheck, build, all server shards, Runner lanes, and E2E gates pass. - Exact-head Greptile: 5/5, zero inline comments or review threads. Independent engineering/security reviews approved the production implementation; engineering approved the test-only follow-up. Browser acceptance review approved with the timing qualification below. - Final-head local generic MCP suite: 84/84 passed (23.26 s); server typecheck passed. Reviewer independently reran the three changed scenarios: 3/3 passed (15.74 s). - Production head `587631230`: local tool-access suite 504/504, recursive typecheck, and full build passed. Broad local checks were not repeated for the test-only delta. Full local `pnpm test:run` remains incomplete after interruption during the first chat shard; no full-local-suite pass is claimed. - Initial full CI at `587631230` failed three generic MCP fixtures that still relied on implicit archived personal recovery. All three reproduced locally (15.01 s). The test-only follow-up gives owner reconnect/rollback scenarios explicit targets and verifies independent fresh URL setup without relaxing their assertions. Corrected server shard 12/12 passes on final head. The unfinished earlier run was cancelled after the follow-up push. - Initial current-head E2E shard 3/8 failed a 5000 ms task-heading wait at `tests/e2e/cloud-auth.spec.ts:46` (40 passed, one failed). Its screenshot/accessibility snapshot contains the expected heading. The single authorized no-code rerun passed 41/41 with zero test retries, including all URL, handoff, and login-form checks. The first-heading assertion took 3774 ms on rerun. Both attempts tested the same merge commit for this PR head. These observations support a timing concern; the original cause remains unknown. The unchanged Cloud-auth path does not invoke archived-connection setup. Recovery is established; an infrastructure cause is not. If it recurs, capture an instrumented first-navigation trace and compare an unchanged baseline under comparable conditions. - PR remains draft. Merge and deployment are outside this request. Synthetic provider/browser fixtures do not prove live GitHub authorization or a deployed fix. ## Risks - Fresh setup after personal archival now creates a new ID and a suffixed name, including for the same owner. Clients must supply an explicit target to reconnect the old identity. - The guard applies to all personal gallery connections, not only GitHub. - No schema migration is required. Explicit owner checks remain in place. - Security review approved the unchanged production implementation. Merge and deployment are outside this change request. ## Model Used OpenAI GPT-6 through Codex. The runtime does not expose a more specific model snapshot or context window. Used reasoning, code editing, and shell tools. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/connections/GITHUB.md | 6 + .../__tests__/generic-mcp-connection.test.ts | 34 ++++- .../src/__tests__/tool-access-service.test.ts | 126 +++++++++++++++++- server/src/services/tool-access.ts | 21 ++- 4 files changed, 171 insertions(+), 16 deletions(-) diff --git a/doc/connections/GITHUB.md b/doc/connections/GITHUB.md index 0f93999ab86..96826dfddfb 100644 --- a/doc/connections/GITHUB.md +++ b/doc/connections/GITHUB.md @@ -77,6 +77,12 @@ repeat an existing instance approval. ## Identity resolution +Fresh setup does not reuse an archived personal connection found by name and +provider. It creates a separate application and connection with an available +name, such as `GitHub (2)`. The archived identity, credentials, grants, and +history stay unchanged. An explicit reconnect or application-targeted setup +continues the existing identity and requires its personal owner. + Every MCP call, `gh` invocation, native Git operation, checkout, health check, and webhook binding uses the same order: diff --git a/server/src/__tests__/generic-mcp-connection.test.ts b/server/src/__tests__/generic-mcp-connection.test.ts index 2978db9bbe1..78bf314dc4f 100644 --- a/server/src/__tests__/generic-mcp-connection.test.ts +++ b/server/src/__tests__/generic-mcp-connection.test.ts @@ -1445,7 +1445,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { ]); }); - it("does not let another user take over an archived personal URL connection", async () => { + it("creates a separate fresh personal URL connection and keeps explicit reconnect owner-bound", async () => { const fixture = installMcpOAuthFixture({ auth: "oauth" }); const company = await createCompany(db); const service = toolAccessService(db); @@ -1456,18 +1456,36 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { await db.insert(companyMemberships).values({ companyId: company.id, principalType: "user", principalId: "other-user", status: "active", membershipRole: "admin", }); + const archivedConnection = await db.select().from(toolConnections).where(eq(toolConnections.id, first.connectionId)); + const archivedApplication = await db.select().from(toolApplications).where(eq(toolApplications.id, first.application.id)); fixture.fetchMock.mockClear(); - await expect(service.connectGalleryApp(company.id, input, { - actorType: "user", actorId: "other-user", - })).rejects.toMatchObject({ status: 403, message: "Only the existing personal identity can reconnect this connection" }); + for (const target of [{ reconnectConnectionId: first.connectionId }, { applicationId: first.application.id }]) { + await expect(service.connectGalleryApp(company.id, { ...input, ...target }, { + actorType: "user", actorId: "other-user", + })).rejects.toMatchObject({ status: 403, message: "Only the existing personal identity can reconnect this connection" }); + } expect(fixture.fetchMock).not.toHaveBeenCalled(); + const fresh = await service.connectGalleryApp(company.id, input, { actorType: "user", actorId: "other-user" }); + expect(fresh.connectionId).not.toBe(first.connectionId); + expect(fresh.application.id).not.toBe(first.application.id); + expect(fresh.auth).toMatchObject({ kind: "oauth" }); + await expect(service.getConnection(fresh.connectionId)).resolves.toMatchObject({ + createdByUserId: "other-user", credentialPolicy: "per_user", credentialSecretRefs: [], + }); + await expect(db.select().from(toolConnections).where(eq(toolConnections.id, first.connectionId))) + .resolves.toEqual(archivedConnection); + await expect(db.select().from(toolApplications).where(eq(toolApplications.id, first.application.id))) + .resolves.toEqual(archivedApplication); + // OAuth grants are created only after authorization; discovery is not a grant. await expect(db.select().from(connectionGrants)).resolves.toHaveLength(0); await expect(service.getConnection(first.connectionId)).resolves.toMatchObject({ status: "archived", createdByUserId: "board-user", }); - const resumed = await service.connectGalleryApp(company.id, input, { actorType: "user", actorId: "board-user" }); + const resumed = await service.connectGalleryApp(company.id, { + ...input, reconnectConnectionId: first.connectionId, + }, { actorType: "user", actorId: "board-user" }); expect(resumed.connectionId).toBe(first.connectionId); expect(resumed.auth).toMatchObject({ kind: "oauth" }); }); @@ -1544,7 +1562,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { return jsonResponse({ jsonrpc: "2.0", id: "paperclip-catalog-refresh", result: { tools: FIXTURE_TOOLS } }); }); const failure = service.connectGalleryApp(company.id, { - link: MCP_URL, name: "Personal retry rollback", grantKind: "user", + link: MCP_URL, name: "Personal retry rollback", grantKind: "user", reconnectConnectionId: first.connectionId, }, actor).then(() => null, (error: unknown) => error); await Promise.race([catalogPending, failure.then((error) => { throw error ?? new Error("Retry finished before the catalog probe"); })]); try { @@ -1587,7 +1605,9 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { FOR EACH ROW EXECUTE FUNCTION test_personal_profile_failure() `); try { - await expect(service.connectGalleryApp(company.id, input, actor)).rejects.toThrow(); + await expect(service.connectGalleryApp(company.id, { + ...input, reconnectConnectionId: first.connectionId, + }, actor)).rejects.toThrow(); await expect(db.select().from(toolCatalogEntries).where(eq(toolCatalogEntries.connectionId, first.connectionId))) .resolves.toEqual(catalogBefore); await expect(db.select().from(toolProfiles).where(eq(toolProfiles.companyId, company.id))).resolves.toHaveLength(0); diff --git a/server/src/__tests__/tool-access-service.test.ts b/server/src/__tests__/tool-access-service.test.ts index 5839a505ec2..166ec8f212a 100644 --- a/server/src/__tests__/tool-access-service.test.ts +++ b/server/src/__tests__/tool-access-service.test.ts @@ -9795,6 +9795,7 @@ describeEmbeddedPostgres("tool access service", () => { const revived = await service.connectGalleryApp( company.id, { + reconnectConnectionId: first.connectionId, galleryKey: "google-drive", connectionMethodKey: "paperclip-write", grantKind: "user", @@ -15997,7 +15998,7 @@ describeEmbeddedPostgres("tool access service", () => { ).rejects.toMatchObject({ status: 404 }); }); - it("reuses a removed gallery app while applying the omitted organization identity default", async () => { + it("creates a separate gallery app for the omitted organization default after personal removal", async () => { const company = await createCompany(db); const service = createTestToolAccessService(db); const actor = { actorType: "user" as const, actorId: "local-board" }; @@ -16022,8 +16023,10 @@ describeEmbeddedPostgres("tool access service", () => { actor, ); - expect(second.application.id).toBe(first.application.id); - expect(second.connectionId).toBe(first.connectionId); + expect(second.application.id).not.toBe(first.application.id); + expect(second.connectionId).not.toBe(first.connectionId); + await expect(service.getConnection(first.connectionId, company.id)) + .resolves.toMatchObject({ status: "archived", credentialPolicy: "per_user" }); expect(second.application.status).toBe("draft"); expect(second.connection.status).toBe("draft"); expect(second.connection.credentialPolicy).toBe("shared"); @@ -16038,8 +16041,8 @@ describeEmbeddedPostgres("tool access service", () => { isDefault: true, }), ]); - await expect(db.select().from(toolApplications)).resolves.toHaveLength(1); - await expect(db.select().from(toolConnections)).resolves.toHaveLength(1); + await expect(db.select().from(toolApplications)).resolves.toHaveLength(2); + await expect(db.select().from(toolConnections)).resolves.toHaveLength(2); }); it("restores grants and credential policy when an identity-changing revival fails", async () => { @@ -18414,6 +18417,119 @@ describeEmbeddedPostgres("tool access service", () => { }); }, 15000); + it.each(["local-board", "another-user", "current-user"])( + "creates an independent fresh GitHub identity after archiving a personal connection owned by %s", + async (ownerUserId) => { + const company = await createCompany(db); + const service = createTestToolAccessService(db); + const currentUserId = "current-user"; + await grantBoardUser(db, company.id, currentUserId, [], "member"); + mockToolsList([readTool("get_me", "Read current user")]); + await withGalleryServerUrl("github", PUBLIC_MCP_FIXTURE_URL, async () => { + const first = await service.connectGalleryApp(company.id, { + galleryKey: "github", + connectionMethodKey: "mcp-key", + name: "GitHub", + grantKind: "user", + credentialValues: { "credentials.authorization": "archived-fixture-key" }, + }, { actorType: "user", actorId: ownerUserId }); + // Model a legacy archived row that still has personal credential refs. + // Fresh setup must leave even this retained material untouched. + await db.update(toolConnections).set({ status: "archived", enabled: false }) + .where(eq(toolConnections.id, first.connectionId)); + await db.update(toolApplications).set({ status: "archived", archivedAt: new Date() }) + .where(eq(toolApplications.id, first.application.id)); + await db.insert(activityLog).values({ + companyId: company.id, actorType: "user", actorId: ownerUserId, + action: "tool_connection.archived", entityType: "tool_connection", + entityId: first.connectionId, + }); + const retained = async () => ({ + applications: await db.select().from(toolApplications) + .where(eq(toolApplications.id, first.application.id)), + connections: await db.select().from(toolConnections) + .where(eq(toolConnections.id, first.connectionId)), + grants: await db.select().from(connectionGrants) + .where(eq(connectionGrants.connectionId, first.connectionId)), + secrets: await db.select().from(companySecrets), + versions: await db.select().from(companySecretVersions), + activity: await db.select().from(activityLog) + .where(eq(activityLog.entityId, first.connectionId)), + }); + const before = await retained(); + const app = createRouteApp(db, + boardSessionActor(company.id, "member", currentUserId), + undefined, { deploymentMode: "authenticated" }); + const freshInput = { + galleryKey: "github", + connectionMethodKey: "mcp-key", + name: "GitHub", + grantKind: "user", + credentialValues: { "credentials.authorization": "fresh-fixture-key" }, + }; + + if (ownerUserId !== currentUserId) { + for (const target of [ + { reconnectConnectionId: first.connectionId }, + { applicationId: first.application.id }, + ]) { + await request(app) + .post(`/api/companies/${company.id}/tools/apps/connect`) + .send({ ...freshInput, ...target }).expect(403); + await expect(service.connectGalleryApp(company.id, { + ...freshInput, grantKind: "user", ...target, + }, { actorType: "user", actorId: currentUserId, actorSource: "session" })) + .rejects.toMatchObject({ + status: 403, + message: "Only the existing personal identity can reconnect this connection", + }); + } + expect(await retained()).toEqual(before); + } + + // An omitted key must not borrow the archived personal credential. + await request(app) + .post(`/api/companies/${company.id}/tools/apps/connect`) + .send({ ...freshInput, credentialValues: {} }).expect(400); + expect(await retained()).toEqual(before); + + const fresh = await request(app) + .post(`/api/companies/${company.id}/tools/apps/connect`) + .send(freshInput).expect(201); + expect(fresh.body.connectionId).not.toBe(first.connectionId); + expect(fresh.body.application.id).not.toBe(first.application.id); + expect(fresh.body.application.name).toBe("GitHub (2)"); + expect(fresh.body.connection).toMatchObject({ + credentialPolicy: "per_user", credentialSecretRefs: [], + }); + const after = await retained(); + expect(after.applications).toEqual(before.applications); + expect(after.connections).toEqual(before.connections); + expect(after.grants).toEqual(before.grants); + expect(after.activity).toEqual(before.activity); + for (const secret of before.secrets) expect(after.secrets).toContainEqual(secret); + for (const version of before.versions) expect(after.versions).toContainEqual(version); + const freshGrants = await db.select().from(connectionGrants) + .where(eq(connectionGrants.connectionId, fresh.body.connectionId)); + expect(freshGrants).toEqual([expect.objectContaining({ + kind: "user", subjectUserId: currentUserId, status: "active", + })]); + const oldSecretIds = before.grants.flatMap((grant) => + grant.credentialSecretRefs.map((ref) => ref.secretId)); + expect(freshGrants[0]!.credentialSecretRefs).toHaveLength(1); + expect(oldSecretIds).toHaveLength(1); + const freshSecretId = freshGrants[0]!.credentialSecretRefs[0]!.secretId; + expect(oldSecretIds).not.toContain(freshSecretId); + expect(after.secrets.find((secret) => secret.id === freshSecretId)) + .toMatchObject({ scope: "user", ownerUserId: currentUserId }); + await expect(db.select().from(activityLog) + .where(and(eq(activityLog.entityId, fresh.body.connectionId), + eq(activityLog.action, "tool_app.connected")))) + .resolves.toEqual([expect.objectContaining({ actorId: currentUserId })]); + }, "mcp-key"); + }, + ); + it("reconnects a personal key on the existing user grant without creating an organization credential", async () => { const company = await createCompany(db); const service = createTestToolAccessService(db); diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 3e3f6292565..5b61834c94c 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -1692,8 +1692,8 @@ export function toolAccessService( // application/connection rows. A fresh gallery click has no // applicationId, so recover that retained identity by its company-unique // name and source instead of inserting a duplicate application that the - // unique index rejects. Active applications are excluded: connecting a - // second account still requires its own name/application identity. + // unique index rejects. Archived personal identities are excluded below. + // Active applications also require a separate application identity. const requestedName = input.name ?? galleryEntry?.name ?? defaultLinkName(input.link ?? ""); const recoverableApplicationStatuses = galleryEntry @@ -1841,6 +1841,19 @@ export function toolAccessService( retainedConnection = undefined; existingApplication = null; } + // A fresh setup must not revive a personal identity found only by name. + // Keep its application, grants, credentials, and history archived, and let + // normal name allocation create a separate application/connection below. + // Explicit reconnects still use the fixed personal-owner check. + if ( + !requestedConnectionId && + !input.applicationId && + retainedConnection?.status === "archived" && + retainedConnection.credentialPolicy === "per_user" + ) { + existingApplication = null; + retainedConnection = undefined; + } let applicationName = existingApplication?.name ?? requestedName; let name = retainedConnection?.name ?? requestedName; if (!existingApplication) { @@ -1876,8 +1889,8 @@ export function toolAccessService( : "organization" : null; // An explicit resume/application reconnect continues the retained identity. - // A fresh gallery connect may still reuse an archived row for stable history - // and company-unique naming, but an explicit Access choice is a new identity + // A fresh gallery connect may still reuse an archived non-personal row for + // stable history, but an explicit Access choice is a new identity // decision and must replace the archived policy. Without this distinction, // removing a dedicated-agent connection and reconnecting the default personal // account leaves `per_agent` behind and the OAuth callback cannot persist its From 057898530c702ec14c1e14d163e96dfb045cd8c0 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:51:37 -0600 Subject: [PATCH 17/28] fix(server): handle wrapped recovery action conflicts (#18885) ## Thinking Path > - Paperclip coordinates durable agent work and recovery. > - Independent workers can find the same task needs recovery. > - The database permits one active recovery action for that task. > - A losing insert must read the action that already committed. > - Drizzle wraps the PostgreSQL unique violation in its cause. > - The current classifier reads only the outer error and misses that conflict. > - This PR recognizes the same allowed conflict through a bounded cause chain. > - Both workers can then return the one durable action through the existing retry path. ## Linked Issues or Issue Description Refs: #18783. **What happened?** The saved-disposition-debt recovery regression failed in CI `41733864320`. Two recovery workers collided on `issue_recovery_actions_active_source_uq`. The fixture captured an unhandled PostgreSQL `23505` error. A separate canceled shard in that run lost its AWS Spot instance; that interruption is a different failure. **Expected behavior** Two service instances that race to create the same active action must return that one action. Unrelated database failures must still propagate. Existing transaction and ownership rules must stay in force. **Steps to reproduce** Hold a PostgreSQL SHARE lock on the recovery-action table. Start the same source-scoped upsert on two independent service instances. Confirm both real INSERTs are waiting in `pg_stat_activity`, then release the lock. The original code persists one action but rejects one caller with a `DrizzleQueryError`. Its outer code is absent; its `PostgresError` cause has SQLSTATE `23505` and the expected active-source constraint. **Paperclip version or commit** Supporting branch starts at master `54782706e1d83d92295246e499627930df03662d`. It now contains a normal merge of master `8771606fd87f69ed4057779e3dce3a9bca8088e6`. The classifier correction stays unchanged. The PR diff against that master still contains only the recovery service and its tests. ## What Changed - Inspect up to eight error causes with cycle detection. - Require SQLSTATE `23505` and one of the same two recovery-action constraint names on the same error. Support the PostgreSQL driver's `constraint_name` field. - Reuse the existing source-scoped retry path and its unchanged three-retry limit. - Add a deterministic real-database collision regression and no-retry controls for unrelated uniqueness, other SQLSTATEs and cyclic errors. - Keep the original recovery regression, assertions and timeouts unchanged. Keep the saved-skill journey separate in #18783. ## Verification - Review follow-up: the real collision test now checks the intended retry merge. The surviving action has attempt count 2, active status, the default unlimited budget, the expected owner, supplied evidence and supplied attempt timestamp. Its wake and monitor fields remain null. Existing identity and convergence assertions run first. - A disposable counterfactual returned the winner unchanged after the lost insert. It passed the original barrier, one-row, no-rejection and same-ID assertions, then failed the new attempt-count check with 1 instead of 2. The counterfactual was not committed. The production service was restored byte-for-byte. The actual service passed all seven focused race, no-retry and sequential controls. The affected canonical test-tree-2 typecheck passed against verified unchanged local declarations. - Parent `4f8abfe4f2244649219ef6ced3534634193f0b53` full CI `41746314291` passed all 48 selected jobs with one intended skip. Its 5/5 summary also had two duplicate unresolved reliability comments about the missing state assertions. This test-only follow-up addresses both. Fresh full CI and current-head review are required again. - The deterministic real-database regression failed before the correction. Both inserts reached the lock and exactly one active action persisted. The losing error was a `DrizzleQueryError` with a `PostgresError` cause, SQLSTATE `23505` and constraint `issue_recovery_actions_active_source_uq`. - **11 focused checks passed across two files on the integrated head** (181.72 seconds). This includes the real-database collision, no-retry controls and all four unchanged recovery producer/dependency regressions, including the original failed real-clock saved-debt case. Both callers return the one persisted action. - Current-head canonical server typecheck passed on Node24.22.2 with all seven layers: below, cycle, above, tests and test-tree-0 through test-tree-2. Canonical shared, SDK, DB and Runner TypeScript prerequisites passed. The first filtered installs lacked CLI and UI imports required by the new test layers. Those module-resolution failures remain recorded as setup failures. A full frozen workspace install with lifecycle scripts disabled supplied the same locked dependencies. No source or type-configuration workaround was used. No Rust or broad workspace build ran locally; full repository typecheck, tests and build remain hosted-CI gates. - The actual supporting master baseline reproduced the same wrapped23505 failure before the correction. Earlier configuration-resolution, local IPC and native-library prerequisites remain separate setup failure evidence. No test failure was reclassified as a pass. - First supporting CI `41739489620` was canceled before its build/test jobs completed. Its later automatic full-CI run `41739747947` failed. The recovery server shard passed. The Runner failure was the Cursor fixture case repaired upstream in #18782, now included by the normal master merge. That CI also observed an attachment-browser timeout, a repository-rehome timeout and a lost-runner connection. Their underlying causes remain unknown. A later pass will establish recovery, not explain those earlier failures. These original results remain retained. - Current `fbd392b9b65dbcf4ac078825096037ed0d490a0b` full CI [41751706634 attempt 2](https://github.com/paperclipai/paperclip/actions/runs/41751706634/attempts/2) passed: 48 successful jobs and one intended skip. Attempt 1 remains retained: ten primary jobs lost their bound Spot capacity and the verify job failed as a dependent. A single ordinary failed-job recheck produced eleven fresh successful executions (ten primaries and verify); timestamps prove 37 successful executions were retained. The skip is not counted as execution. Current Greptile is 5/5 with both addressed reliability threads resolved and no unresolved findings. All 57 current check runs are successful, neutral or skipped, and Snyk is successful. This is CI recovery; it does not explain the separately retained older attachment and repository timeouts. - The original CI failure and both failed saved-skill QA attempts remain unchanged. This PR does not establish a live QA pass or deploy a production change. ## Risks Conflict classification must not turn unrelated errors into retries. The new controls require those errors to propagate after one insert attempt. The traversal is bounded and stops at cycles. Transaction, company scope, ownership, notifications, retry count and database constraints are unchanged. No schema or public API change is required. ## Model Used OpenAI GPT-6 through Codex with reasoning and code execution. The runtime does not expose a more specific model ID or context-window size. These checks make no model or provider calls. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../__tests__/issue-recovery-actions.test.ts | 61 +++++++++++++++++++ server/src/services/issue-recovery-actions.ts | 22 ++++--- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 21553967b5d..85d6bf6afc0 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -90,6 +90,24 @@ function makeRecoveryActionRow(overrides: Record = {}) { const LATEST_RUN_TIMES = { startedAt: new Date(), createdAt: new Date() }; describe("issueRecoveryActionService", () => { + it.each(["unrelated unique constraint", "other SQLSTATE", "cyclic cause"])("does not retry %s errors", async kind => { + const constraint = "issue_recovery_actions_active_source_uq"; + const error: Error & { cause?: unknown } = new Error("Wrapped database failure"); + error.cause = kind === "unrelated unique constraint" ? { code: "23505", constraint_name: "other_unique_constraint" } + : kind === "other SQLSTATE" ? { code: "23503", constraint_name: constraint } : error; + const query = { from() { return this; }, where() { return this; }, orderBy() { return this; }, + limit() { return Promise.resolve([]); } }; + const fakeDb = { + async transaction(fn: (tx: unknown) => Promise): Promise { return fn(fakeDb); }, + select: vi.fn(() => query), + insert: vi.fn(() => ({ values: () => ({ returning: async () => { throw error; } }) })), + }; + await expect(issueRecoveryActionService(fakeDb as never).upsertSourceScoped({ companyId: "company", sourceIssueId: "issue", + kind: "missing_disposition", cause: "saved-debt", fingerprint: "same-source", nextAction: "Restore existing work." })) + .rejects.toBe(error); + expect(fakeDb.select).toHaveBeenCalledOnce(); + expect(fakeDb.insert).toHaveBeenCalledOnce(); + }); it("does not reactivate an action resolved between the active read and update", async () => { const existingRow = makeRecoveryActionRow({ id: "existing-action", attemptCount: 1 }); const createdRow = makeRecoveryActionRow({ id: "new-action", attemptCount: 1 }); @@ -413,6 +431,49 @@ describeEmbeddedPostgres("issue recovery actions", () => { }, ); + it("converges two service instances after their real inserts compete for one active source", async () => { + const { companyId, managerId, sourceIssueId } = await seedCompany(); + const input = { companyId, sourceIssueId, kind: "stranded_assigned_issue" as const, ownerType: "agent" as const, + ownerAgentId: managerId, cause: "stranded_assigned_issue", fingerprint: "concurrent-source", + evidence: { sourceRunId: "concurrent-source-run" }, lastAttemptAt: new Date("2026-11-03T12:00:00Z"), + nextAction: "Restore the existing execution path." }; + let release!: () => void, acquired!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const locked = new Promise(resolve => { acquired = resolve; }); + const lock = db.transaction(async tx => { + await tx.execute(sql`lock table issue_recovery_actions in share mode`); + acquired(); await gate; + }); + await locked; + const outcomes = Promise.allSettled([issueRecoveryActionService(db).upsertSourceScoped(input), + issueRecoveryActionService(db).upsertSourceScoped(input)]); + try { + await vi.waitFor(async () => { + const [waiting] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and state = 'active' and wait_event_type = 'Lock' + and query like 'insert into "issue_recovery_actions"%'`); + expect(waiting.count).toBe(2); + }, { timeout: 5000, interval: 20 }); + } finally { release(); await lock; await outcomes; } + const results = await outcomes; + const active = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, companyId), + eq(issueRecoveryActions.sourceIssueId, sourceIssueId), sql`${issueRecoveryActions.status} in ('active', 'escalated')`)); + expect(active).toHaveLength(1); + expect(results.flatMap(result => result.status === "rejected" ? [{ + name: result.reason?.constructor?.name, code: result.reason?.code, + causeName: result.reason?.cause?.constructor?.name, causeCode: result.reason?.cause?.code, + causeConstraint: result.reason?.cause?.constraint_name, + }] : [])).toEqual([]); + expect(results.map(result => result.status === "fulfilled" ? result.value.id : null)).toEqual([active[0]!.id, active[0]!.id]); + // The lost insert uses the existing upsert merge, just like a second sequential call. + expect(active[0]).toMatchObject({ + status: "active", attemptCount: 2, maxAttempts: null, timeoutAt: null, + ownerType: "agent", ownerAgentId: managerId, ownerUserId: null, + evidence: input.evidence, lastAttemptAt: input.lastAttemptAt, + nextAction: input.nextAction, wakePolicy: null, monitorPolicy: null, + }); + }); + it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => { const { companyId, managerId, sourceIssueId } = await seedCompany(); const svc = issueRecoveryActionService(db); diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index e276dc88f89..ae868406aa6 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -116,19 +116,27 @@ function toReadModel(row: IssueRecoveryActionRow): IssueRecoveryAction { } function isUniqueRecoveryActionConflict(error: unknown) { - const maybe = error as { code?: string; constraint?: string; message?: string } | null; - return Boolean( - maybe && - maybe.code === "23505" && - ( + const seen = new Set(); + let candidate = error; + // Drizzle wraps the driver error in cause. Only this specific uniqueness + // conflict can reuse the existing bounded source-scoped upsert retry. + for (let depth = 0; depth < 8 && candidate && typeof candidate === "object"; depth++) { + if (seen.has(candidate)) break; + seen.add(candidate); + const maybe = candidate as { code?: string; constraint?: string; constraint_name?: string; message?: string; cause?: unknown }; + if (maybe.code === "23505" && ( maybe.constraint === "issue_recovery_actions_active_source_uq" || maybe.constraint === "issue_recovery_actions_active_fingerprint_uq" || + maybe.constraint_name === "issue_recovery_actions_active_source_uq" || + maybe.constraint_name === "issue_recovery_actions_active_fingerprint_uq" || typeof maybe.message === "string" && ( maybe.message.includes("issue_recovery_actions_active_source_uq") || maybe.message.includes("issue_recovery_actions_active_fingerprint_uq") ) - ), - ); + )) return true; + candidate = maybe.cause; + } + return false; } export function issueRecoveryActionService(db: DbOrTransaction) { From 925965a09b8931c1791f2402f3e7595e2845e68e Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Tue, 3 Nov 2026 18:05:40 -0800 Subject: [PATCH 18/28] fix(task-retry): wait out brief chat authorization contention (#18895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - A user can click Retry on a task that is connected to an external chat conversation. > - That Retry prepares the chat retry inside one admission transaction. The chat step takes a per-principal authorization lock with `pg_try_advisory_xact_lock`. > - When another chat transaction holds that lock for a short time, the chat step throws `NativeChatReviewPresentationContentionError`. This error is not an `HttpError`. The route returns HTTP 500, and the error handler reports a server crash. > - The contention is routine and short. The admission transaction does not contact a provider, and it rolls back completely. The owning service can safely try it again. > - This pull request retries the admission a bounded number of times. If the lock stays busy, the service returns a retryable 409 instead of a 500. > - The benefit: the user's click usually succeeds without an error, and Paperclip stops reporting expected lock contention as a crash. ## Linked Issues or Issue Description **What happened?** A hosted staging server reported `NativeChatReviewPresentationContentionError: Chat response authorization is temporarily busy; no provider delivery was attempted` from `POST /api/issues/:id/retry`. The stack is `requestTaskRetry` → `writeExecutionControl` → `prepareFailedChatRunRetry` → `failedChatRetrySource`. The error came from the `chat-identity::` advisory try-lock. The route returned HTTP 500 for each occurrence. **Expected behavior** Short lock contention must not fail a deliberate Retry click. The service must wait it out. If the lock stays busy, the service must return a clear, retryable client error and must not report a server crash. **Steps to reproduce** 1. Bind a task to a restricted chat endpoint and conversation. Make the latest task run a failed run. 2. In a different database transaction, hold `pg_try_advisory_xact_lock(hashtextextended('chat-identity::', 0))`. 3. Call `POST /api/issues/:id/retry`. Before this change, the chat retry step throws `NativeChatReviewPresentationContentionError` and the route returns 500. **Paperclip version or commit** Reproduced on master `8771606fd87f69ed4057779e3dce3a9bca8088e6`. The server release that reported it was `d59791828482f213713ca55000c40e6c9c6c03e5`. **Deployment mode** Hosted staging, Node 24, PostgreSQL. Related: #18845 and #18621 changed nearby Retry admission. They do not handle this contention. ## What Changed - `server/src/services/task-retry.ts`: `requestTaskRetry` now calls the unchanged admission transaction (`admitTaskRetry`) in a bounded loop: 10 attempts, 100 ms apart. It retries only `NativeChatReviewPresentationContentionError` and lock-not-available contention (`isExternalChatWaitAuthorizationContention`, which matches `55P03`). Every other error, including 403/404/409 decisions, propagates unchanged on the first attempt. - If the lock stays busy for all attempts, the service throws `conflict(..., { code: "task_retry_busy", retryable: true })`. The UI's existing 4xx handling resets the Retry button, and the error handler does not report a 4xx as a crash. - `server/src/services/task-retry.test.ts`: three PostgreSQL cases: - Real advisory-lock contention. A second transaction holds the same `chat-identity` lock that production checks, then releases it. Retry succeeds and persists exactly one wake request. A replay with the same idempotency key returns the same request. - Persistent contention returns 409 `task_retry_busy` after exactly 10 attempts and persists no wake request. - A 403 from the chat step does not retry. The transaction body is unchanged. It was moved into `admitTaskRetry` without edits. ## Verification - RED: with the old `task-retry.ts` and the new tests, the contention case fails with the exact production error (`NativeChatReviewPresentationContentionError: Chat response authorization is temporarily busy…`). The persistent case fails because the error is not an `HttpError`. - GREEN: `pnpm exec vitest run src/services/task-retry.test.ts` passes all 39 cases against embedded PostgreSQL. - `pnpm --filter @paperclipai/server typecheck` passes (all layers, including tests). - `src/services/heartbeat/workspace-preparation.test.ts` (another `requestTaskRetry` caller): 165 of 167 cases pass. The 2 failures (`remote_workspace_migration_source_changed` in "resumes an imported path/file-url Git source…") also fail on unchanged master in the same environment, so this change does not cause them. - Runtime assumption (rule `runtime-dependency-boundaries`): the retry is safe only if a failed attempt leaves no persisted state. The contention test uses a real PostgreSQL advisory lock held by a separate connection and checks that only one wake row exists after the failed attempts. Each attempt is a separate transaction, so the loop holds no row lock or advisory lock while it waits. - The full repository `pnpm test:run` and `pnpm build` were not run locally. I rely on CI for those. ## Risks - Low risk. The worst case adds about 1 second of request time when the lock stays busy, and then returns 409 instead of 500. - An attempt can fail after an earlier step in the same transaction (for example, workspace preservation registration). The whole transaction rolls back, so the next attempt repeats those steps from a clean state. Those steps run in the same transaction and do not contact a provider. - If `requestTaskRetry` is called inside an outer transaction, each attempt rolls back only to its savepoint. Locks that the outer transaction holds stay held. In that case the loop ends with the same 409 after its bounded budget. - No schema, API shape, or UI change. ## Model Used Claude Opus 5.5 (`claude-opus-5-5`, Anthropic), through Claude Code with tool use (shell, file editing, test execution) and extended reasoning. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass (focused suites; see Verification for the two pre-existing failures) - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes (no behavior or command documentation changes are needed) - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- server/src/services/task-retry.test.ts | 73 +++++++++++++++++++++++++- server/src/services/task-retry.ts | 35 ++++++++++-- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/server/src/services/task-retry.test.ts b/server/src/services/task-retry.test.ts index 363663258be..a05fd80146c 100644 --- a/server/src/services/task-retry.test.ts +++ b/server/src/services/task-retry.test.ts @@ -1,10 +1,12 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, agentWakeupRequests, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueComments, issues, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db"; +import { agents, agentWakeupRequests, chatConversations, chatEndpoints, companies, createDb, type Db, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueComments, issues, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { priorRecoveryApiReceipt, unresolvedRecoveryToolInvocation } from "./recovery-effect-receipts.js"; import { requestTaskRetry } from "./task-retry.js"; +import { HttpError } from "../errors.js"; +import { NativeChatReviewPresentationContentionError } from "./native-runtime/native-chat-review-presentation.js"; import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js"; import { settleUnrecoverableExecutions } from "./execution-recovery-resolution.js"; import { createSystemRecovery, failSystemRecovery, readSystemRecovery, SystemRecoveryFailure } from "./system-recovery-state.js"; @@ -441,4 +443,73 @@ describe("durable task Retry admission", () => { .toMatchObject({ status: "skipped", error: "task_retry_superseded" }); }); + async function seedChatTask() { + const input = await seed(); + const [issue] = await db.select().from(issues).where(eq(issues.id, input.issueId)); + const [application] = await db.insert(toolApplications).values({ companyId: input.companyId, name: "Slack", type: "chat" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: input.companyId, applicationId: application.id, + name: "Slack", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel" }).returning(); + const [endpoint] = await db.insert(chatEndpoints).values({ companyId: input.companyId, connectionId: connection.id, + provider: "slack", publicId: randomUUID(), assignedAgentId: issue.assigneeAgentId!, status: "active" }).returning(); + await db.insert(chatConversations).values({ companyId: input.companyId, endpointId: endpoint.id, issueId: input.issueId, + externalConversationId: "channel", externalThreadId: randomUUID(), externalLabel: "#retry", state: "waiting" }); + return input; + } + const chatIdentityLock = (companyId: string, principalId: string) => + sql`pg_try_advisory_xact_lock(hashtextextended(${`chat-identity:${companyId}:${principalId}`}, 0))`; + + it("waits out brief chat authorization contention without a second request", async () => { + const input = await seedChatTask(); + const principalId = randomUUID(), actionId = randomUUID(); + let calls = 0, locked!: () => void, release!: () => void; + const held = new Promise(resolve => { locked = resolve; }); + const mayRelease = new Promise(resolve => { release = resolve; }); + // Another chat transaction owns this principal's authorization lock. + const holder = db.transaction(async tx => { + expect((await tx.execute<{ locked: boolean }>(sql`select ${chatIdentityLock(input.companyId, principalId)} as locked`))[0].locked).toBe(true); + locked(); await mayRelease; + }); + await held; + const prepareChatRetry = async (tx: Db) => { + calls += 1; + if (calls === 2) release(); + // Mirror the production authorization check, including its rollback. + const [row] = await tx.execute<{ locked: boolean }>(sql`select ${chatIdentityLock(input.companyId, principalId)} as locked`); + if (!row?.locked) throw new NativeChatReviewPresentationContentionError(); + return { actionId }; + }; + try { + const receipt = await requestTaskRetry({ ...input, prepareChatRetry }); + expect(receipt).toMatchObject({ accepted: true, chatActionId: actionId }); + expect(calls).toBeGreaterThan(1); + const rows = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, input.companyId)); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: receipt.requestId, status: "deferred_issue_execution", + payload: { taskRetry: { chatActionId: actionId } } }); + expect((await requestTaskRetry({ ...input, prepareChatRetry })).requestId).toBe(receipt.requestId); + } finally { release(); await holder; } + }); + + it("returns a retryable conflict when chat authorization stays busy", async () => { + const input = await seedChatTask(); + let calls = 0; + const failure = await requestTaskRetry({ ...input, prepareChatRetry: async () => { + calls += 1; + throw new NativeChatReviewPresentationContentionError(); + } }).catch(error => error); + expect(failure).toBeInstanceOf(HttpError); + expect(failure).toMatchObject({ status: 409, details: { code: "task_retry_busy", retryable: true } }); + expect(calls).toBe(10); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, input.companyId))).toHaveLength(0); + }); + + it("does not retry an authorization denial", async () => { + const input = await seedChatTask(); + let calls = 0; + await expect(requestTaskRetry({ ...input, prepareChatRetry: async () => { + calls += 1; + throw new HttpError(403, "denied"); + } })).rejects.toMatchObject({ status: 403 }); + expect(calls).toBe(1); + }); }); diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index 31180616cbb..930e753529c 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -21,6 +21,8 @@ import { deferredChatInputPredicate, resumeDeferredChatWake } from "./durable-ch import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; import { hasPendingRunCleanup } from "./execution-blocker.js"; import { PersistedExecutionWaitRefusal } from "./deferred-execution-wait.js"; +import { NativeChatReviewPresentationContentionError } from "./native-runtime/native-chat-review-presentation.js"; +import { isExternalChatWaitAuthorizationContention } from "./native-runtime/chat-attachment-reuse.js"; export async function taskRetryMutationAllowed(db: Db, input: { companyId: string; issueId: string; actorId: string; actorSource: TaskRetryIntent["actorSource"]; @@ -206,14 +208,39 @@ export async function retireSupersededTaskRetry(db: Db, input: { return retired.length > 0; } -/** Persist a deliberate click under dispatch's issue-first lock order. This - * transaction never contacts a provider. The normal wake queue owns execution. */ -export async function requestTaskRetry(input: { +type TaskRetryRequest = { db: Db; companyId: string; issueId: string; actorId: string; idempotencyKey: string; observedRunId: string | null; actorSource?: string; prepareChatRetry?: (tx: Db, scope: { companyId: string; issueId: string; agentId: string; failedRunId: string; initiatedByUserId: string; retryIntentId?: string }) => Promise<{ actionId: string }>; -}) { +}; + +const TASK_RETRY_CONTENTION_ATTEMPTS = 10; +const TASK_RETRY_CONTENTION_DELAY_MS = 100; + +function isTaskRetryContention(error: unknown) { + return error instanceof NativeChatReviewPresentationContentionError || isExternalChatWaitAuthorizationContention(error); +} + +/** Persist a deliberate click under dispatch's issue-first lock order. This + * transaction never contacts a provider. The normal wake queue owns execution. */ +export async function requestTaskRetry(input: TaskRetryRequest) { + for (let attempt = 1; ; attempt += 1) { + try { + return await admitTaskRetry(input); + } catch (error) { + if (!isTaskRetryContention(error)) throw error; + // Chat authorization for the same principal is briefly locked. The whole + // admission rolled back, so a fresh attempt rereads every fact. + if (attempt >= TASK_RETRY_CONTENTION_ATTEMPTS) + throw conflict("Retry is busy because chat authorization is in progress. Try again.", + { code: "task_retry_busy", retryable: true }); + } + await new Promise(resolve => setTimeout(resolve, TASK_RETRY_CONTENTION_DELAY_MS)); + } +} + +function admitTaskRetry(input: TaskRetryRequest) { return writeExecutionControl(input.db, DELIVERY_QUEUES.workspaceRecovery, async tx => { const db = tx as unknown as Db; const [issue] = await tx.select().from(issues).where(and(eq(issues.companyId, input.companyId), From a194166017597860a0e53c0c1314e95c597d75cc Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:17:08 -0600 Subject: [PATCH 19/28] perf(ci): repair dependency caches and parallelize transport fixtures (#18883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Required CI checks protect changes to its native Runner and control plane. > - Recent Runner jobs restored a Rust cache but still rebuilt dependencies for 74–92 seconds. > - New workspace install hooks disabled the materialized JavaScript dependency cache. > - A single transport file also kept 326 test cases on one worker. > - This pull request corrects the cache inputs, replays known install hooks, and separates independent transport test groups. > - The same checks can finish sooner without removing test cases or widening runner access. ## Linked Issues or Issue Description Refs #18794 and #18544. Related dependency-action version update: #17698; this change fixes the cache input contract used by the currently pinned action. **What happened?** Recent PR jobs such as #18783 and #18825 restored an exact Rust dependency cache from October 16. The virtual workspace's development profile changed on October 30. Rust Cache 2.9.2 hashes member manifests but omits that virtual manifest. Rebuilding cannot replace an immutable exact cache entry. The JavaScript cache also rejected the new server and Runner Copilot postinstall hooks. **Expected behavior** Changing a Cargo profile selects a new dependency cache. An exact JavaScript dependency restore still performs required workspace install hooks. Independent transport fixtures use the existing file workers while retaining all assertions and real deadlines. **Steps to reproduce** Inspect the Rust dependency cache hit and following `build:rust` in the recent Runner package jobs. Run `node scripts/ci-dependency-cache.mjs` on the previous revision to see `cacheable: false`. Run the original transport file to measure the serial test work. ## What Changed - Add the virtual workspace manifest and ancestor Cargo configuration to all shared Rust dependency keys. Keep reader/writer identities aligned and preserve master-only cache writes and CODEOWNERS-only AWS routing. - Recognize only the exact reviewed root, server, and Runner postinstall commands. Include the server/Runner installer scripts in the dependency key. Replay workspace postinstall hooks after a materialized restore. Unknown lifecycle hooks still disable reuse. - Move attachment recovery, warm attachment, live adoption, and suspension fixtures into separate files. Keep the large event-backlog cases in the original file to avoid starting another expensive fixture group at the same time. Share five stateless test helpers outside production source. - Extend the opt-in `profile-ci` benchmark with a PR-scoped Rust cache producer and a fresh-host consumer. Require exact dependency/workspace hits and reject any Cargo unit that is rebuilt. Retain the JSON build evidence. Existing setup qualification checks real JavaScript cache restores. - Schedule the ten measured files over 50 seconds first, in approximate duration order. Package CI uses up to four workers, bounded by available CPUs; local runs retain Vitest defaults. The AWS runner has four CPUs and 16 GiB RAM; the three-worker run averaged 44% CPU and peaked at 15% memory. Preserve Vitest selection and shard membership and default ordering for other files. Job count and AWS authorization are unchanged. - Scope the optional paired server benchmark to changes in its measured inputs. Cache-only changes qualify exact restores and run the shared package tests using restored dependencies. - Update the cache and test documentation and the existing native cache contract guard. ## Verification - All 586 `.github/scripts/tests/*.test.mjs` policy tests passed, including cache trust and runner routing guards. Actionlint passed for all three changed workflows. - All 169 original top-level transport test definitions remain byte-identical. The original file passed all 326 cases in 317.99 seconds on this macOS host. A clean preliminary split passed the exact same named cases in 183.01 seconds with the same four-worker cap. - Preserve the first failed experiment: its parallel run overlapped the policy suite and timed out in the 1,024-event settled backlog case at 60 seconds; 325 cases passed. A clean repeat passed, but that does not establish the earlier failure's sole cause. The final layout keeps the backlog cases in the original file. The final layout passed the same 326 cases in 186.35 seconds (41.4% faster than the 317.99-second baseline). The original test bodies and all timeouts remain unchanged. - Fresh-host Rust qualification passed: all 308 compilation units were fresh, and Cargo completed in 1.59 seconds on a new runner ([final-head run](https://github.com/paperclipai/paperclip/actions/runs/41745082837/job/126296908816)). JavaScript setup dropped from 46 seconds cold to 23 seconds with exact dependency/build restores; all 1,052 shared-package cases passed against the restored tree ([final-head job](https://github.com/paperclipai/paperclip/actions/runs/41745082837/job/126295937797)). The previous qualification also passed (43 to 19 seconds) and confirmed all three postinstall hooks replayed. These are qualification results, not an end-to-end PR timing claim. - Initial full AWS package run passed 4,907 cases with 10 existing skips. The original-file remainder fell to 118.7 seconds, but the 100-second deadline file started last, leaving the full package at 417.43 seconds. Starting only three long files first passed 4,911 cases (including four new sequencer guards) in 422.54 seconds, so reordering alone did not improve full-package time. Summed file execution was 1,202 seconds across three busy workers. The final four-worker configuration passed **4,912 cases, with the same 10 existing skips**, in **324.99 seconds**, a **23.1% reduction** from 422.54 seconds. The five added cases guard ordering, selection, sharding, and path handling. Peak memory was 17.95% of the 16 GiB runner; average CPU was 49.14% ([final-head package job](https://github.com/paperclipai/paperclip/actions/runs/41745082974/job/126296339480)). - Local workspace typecheck and build passed. The complete local `pnpm test:run` passed all six chat partitions but reported failures in one existing OAuth redirect case and one gateway authority case in the general-server group. Both focused repetitions passed (three OAuth cases and one gateway case), and a complete repeat of both files passed all 593 cases in 42.89 seconds; that does not explain the original failures. The broad local run was stopped after 61 minutes once full final-head CI had passed; its remaining general/workspace/serialized work was not completed locally, and it is not reported as a pass. Original failure logs were retained. Full final-head CI provides the complete suite validation. Sequencer selection/shard guards and the 11-case transport suspension smoke passed. The corrected release workflow guard passed all 71 tests. - The first CI run failed only the policy guard that still expected the old native cache key; downstream aggregate checks also failed. That guard now checks the new profile/config prefix and existing host/target/protocol suffix. A second policy run found that the new sequencer test imported Vitest in the dependency-free policy lane; it now runs in the Runner package lane, where all four cases pass. Greptile reviewed final head `8011f84189` at 5/5 with no actionable findings and confirmed this issue fixed. Final-head policy, typecheck, build, and review passed. At 23:52:46 UTC on November 3, several AWS jobs were terminated together: the Runner package process received SIGTERM and browser logs recorded a runner shutdown signal. The failed/cancelled jobs passed on the same-head retry, including required `ci / verify`; no assertion or timeout was changed in response. The final concurrency adjustment at `fe5a69bb0b` also received Greptile 5/5 with no actionable findings. Final-head required CI, including `ci / verify`, passed ([aggregate](https://github.com/paperclipai/paperclip/actions/runs/41745082974/job/126300554711)). Local timings exclude install, build, cache transfer, and CI queue time. ## Risks A new Rust key needs one successful canonical master build before ordinary PRs can hit it. The fresh-host qualification uses separate PR-scoped entries. Parallel test files can increase I/O contention; case execution inside each file, timeouts, and assertions are unchanged. Two extracted suites exceed 1,000 lines because they retain existing multi-stage recovery fixtures unchanged; one parameterized fixture alone is 1,059 lines. No production execution logic, lockfile, credentials, runner trust gates, or required check is removed. ## Model Used OpenAI GPT-6 through Codex, with reasoning, repository editing, command execution, and GitHub inspection. The host does not expose a more precise model snapshot or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass — targeted checks passed; the broad-run failures and repetitions are documented above - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../actions/setup-ci-dependencies/action.yml | 8 + .../tests/ci-dependency-cache.test.mjs | 43 +- .../tests/pr-runner-rust-cache.test.mjs | 18 +- .github/workflows/ci-work-benchmark.yml | 141 + .github/workflows/pr-trusted.yml | 8 +- .github/workflows/release-verify.yml | 6 +- doc/DEVELOPING.md | 18 +- doc/RELEASE-AUTOMATION-SETUP.md | 38 +- .../scripts/vitest-sequencer.mjs | 27 + .../scripts/vitest-sequencer.test.mjs | 61 + .../runnerd-codex-transport-adoption.test.ts | 556 +++ ...odex-transport-attachment-recovery.test.ts | 1077 +++++ ...runnerd-codex-transport-suspension.test.ts | 769 +++ ...rd-codex-transport-warm-attachment.test.ts | 1649 +++++++ .../src/live/runnerd-codex-transport.test.ts | 4160 +---------------- .../test/support/runnerd-transport.ts | 88 + packages/paperclip-runner/vitest.config.ts | 5 + .../release-verify-workflow.test.mjs | 4 +- scripts/ci-dependency-cache.mjs | 12 +- 19 files changed, 4539 insertions(+), 4149 deletions(-) create mode 100644 packages/paperclip-runner/scripts/vitest-sequencer.mjs create mode 100644 packages/paperclip-runner/scripts/vitest-sequencer.test.mjs create mode 100644 packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts create mode 100644 packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts create mode 100644 packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts create mode 100644 packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts create mode 100644 packages/paperclip-runner/test/support/runnerd-transport.ts diff --git a/.github/actions/setup-ci-dependencies/action.yml b/.github/actions/setup-ci-dependencies/action.yml index 3367001b4b5..0a78e542c91 100644 --- a/.github/actions/setup-ci-dependencies/action.yml +++ b/.github/actions/setup-ci-dependencies/action.yml @@ -65,6 +65,14 @@ runs: pnpm install --frozen-lockfile fi + - name: Replay workspace install hooks after restore + if: steps.installed.outputs.cache-hit == 'true' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + pnpm -r --if-present run postinstall + pnpm run postinstall + - name: Restore certified shared build outputs (read only) if: inputs.restore-cache == 'true' && steps.inputs.outputs.cacheable == 'true' id: builds diff --git a/.github/scripts/tests/ci-dependency-cache.test.mjs b/.github/scripts/tests/ci-dependency-cache.test.mjs index 1b79b00c401..36195ad32e9 100644 --- a/.github/scripts/tests/ci-dependency-cache.test.mjs +++ b/.github/scripts/tests/ci-dependency-cache.test.mjs @@ -71,6 +71,36 @@ test('unmodelled workspace lifecycle scripts disable materialized reuse', () => } }); +test('known Copilot hooks permit caching, track their helpers, and still run on restore', () => { + const files = { ...entries, + 'server/package.json': JSON.stringify({ scripts: { postinstall: 'node scripts/install-copilot-assets.mjs' } }), + 'packages/paperclip-runner/package.json': JSON.stringify({ scripts: { postinstall: 'node scripts/install-copilot-assets.mjs' } }), + 'server/scripts/install-copilot-assets.mjs': 'server bootstrap', + 'packages/paperclip-runner/scripts/install-copilot-assets.mjs': 'runner bootstrap', + 'packages/paperclip-runner/scripts/materialize-copilot-binary.mjs': 'native materializer', + 'packages/paperclip-runner/scripts/copilot-inner-distribution.mjs': 'closure pins', + }; + const compute = (input) => cachePlan({ files: Object.keys(input), + read: file => input[file] == null ? null : Buffer.from(input[file]), scope, + packageDirs: ['.', 'server', 'packages/paperclip-runner'], + }); + const original = compute(files); + assert.equal(original.cacheable, true); + for (const file of Object.keys(files).filter(file => /^(server|packages\/paperclip-runner)\/scripts\//.test(file))) { + for (const replacement of ['changed', null]) { + assert.notEqual(compute({ ...files, [file]: replacement }).dependencyKey, original.dependencyKey, file); + } + } + assert.equal(cachePlan({ files: ['elsewhere/package.json'], + read: () => Buffer.from(files['server/package.json']), scope, packageDirs: ['elsewhere'], + }).cacheable, false, 'the command is allowed only in its reviewed owners'); + const action = readFileSync(new URL('../../actions/setup-ci-dependencies/action.yml', import.meta.url), 'utf8'); + const replay = action.split(' - name: Replay workspace install hooks after restore')[1].split(' - name:')[0]; + assert.match(replay, /if: steps\.installed\.outputs\.cache-hit == 'true'/); + assert.match(replay, /pnpm -r --if-present run postinstall\s+pnpm run postinstall/); + assert.ok(action.indexOf('Replay workspace install hooks') < action.indexOf('Validate workspace links')); +}); + test('cache restores are exact and only a frozen canonical master push can publish entries', () => { const action = readFileSync(new URL('../../actions/setup-ci-dependencies/action.yml', import.meta.url), 'utf8'); const installed = action.slice(action.indexOf(' - name: Restore installed'), action.indexOf(' - name: Locate pnpm')); @@ -97,6 +127,15 @@ test('paired test timings remain available when lifecycle hooks disable material assert.match(step(name), /if: steps.qualification.outputs.cacheable == 'true'/); } assert.match(step('Explain disabled materialized reuse'), /cacheable != 'true'/); - assert.doesNotMatch(step('Prepare unchanged baseline'), /if:/); - assert.doesNotMatch(step('Compare complete commands and executed case inventories'), /if:/); + for (const name of ['Prepare unchanged baseline', 'Compare complete commands and executed case inventories']) { + assert.match(step(name), /if: steps\.test_scope\.outputs\.run == 'true'/); + assert.doesNotMatch(step(name), /if:.*cacheable/); + } + const scope = step('Select changed server test work'); + assert.match(scope, /git diff --name-only HEAD\^1 HEAD --/); + for (const file of ['scripts/benchmark-ci-work.mjs', 'scripts/run-vitest-stable.mjs', + 'server/src/__tests__/setup-supertest.ts', 'server/src/__tests__/workspace-runtime.test.ts', + 'server/src/services/workspace-git-snapshot-streaming.test.ts']) assert.ok(scope.includes(file)); + assert.doesNotMatch(scope, /scripts\/ci-dependency-cache|\.github\/actions\/setup-ci-dependencies/); + assert.match(step('Exercise restored dependencies for cache-only changes'), /pnpm exec vitest run --project @paperclipai\/shared/); }); diff --git a/.github/scripts/tests/pr-runner-rust-cache.test.mjs b/.github/scripts/tests/pr-runner-rust-cache.test.mjs index 6089bcec638..1136e5cf150 100644 --- a/.github/scripts/tests/pr-runner-rust-cache.test.mjs +++ b/.github/scripts/tests/pr-runner-rust-cache.test.mjs @@ -48,11 +48,27 @@ test("every rust-cache restore in the PR workflow belongs to a guarded reader jo const keyInputs = [ /uses: Swatinem\/rust-cache@([0-9a-f]{40}) # v[0-9.]+/, /workspaces: (\$\{\{ steps\.runner_rust_workspace\.outputs\.path \}\} -> target)/, - /shared-key: (release-runner-v2)/, + /shared-key: (release-runner-v2-.+)/, /cache-workspace-crates: (false)/, /cache-bin: (false)/, ]; +test("virtual workspace profiles and ancestor Cargo configuration partition every dependency cache", () => { + for (const [name, workflow] of [["PR", prWorkflow], ["release", releaseWorkflow]]) { + const keys = [...workflow.matchAll(/^\s+shared-key: (.+)$/gm)].map(([, key]) => key); + assert.ok(keys.length > 0); + for (const key of keys) { + assert.ok(key.includes("hashFiles('packages/paperclip-runner/runner/Cargo.toml'"), name); + for (const directory of ['.', 'packages', 'packages/paperclip-runner', 'packages/paperclip-runner/runner']) { + const prefix = directory === '.' ? '' : `${directory}/`; + for (const file of ['config', 'config.toml']) { + assert.ok(key.includes(`'${prefix}.cargo/${file}'`), `${name}: ${prefix}.cargo/${file}`); + } + } + } + } +}); + test("every PR reader restores the Rust cache under the same key the master push writes", () => { for (const [name, pr] of readers) { for (const pattern of keyInputs) { diff --git a/.github/workflows/ci-work-benchmark.yml b/.github/workflows/ci-work-benchmark.yml index fe7ad35782d..96dffab12d5 100644 --- a/.github/workflows/ci-work-benchmark.yml +++ b/.github/workflows/ci-work-benchmark.yml @@ -7,6 +7,9 @@ on: - .github/actions/setup-ci-dependencies/** - .github/workflows/ci-work-benchmark.yml - scripts/ci-dependency-cache.mjs + - .github/workflows/pr-trusted.yml + - .github/workflows/release-verify.yml + - scripts/runner-debug-crate-cache.mjs - scripts/benchmark-ci-work.mjs - scripts/run-vitest-stable.mjs - server/src/__tests__/setup-supertest.ts @@ -39,6 +42,21 @@ jobs: fetch-depth: 2 persist-credentials: false + - name: Select changed server test work + id: test_scope + run: | + changed=$(git diff --name-only HEAD^1 HEAD -- \ + scripts/benchmark-ci-work.mjs \ + scripts/run-vitest-stable.mjs \ + server/src/__tests__/setup-supertest.ts \ + server/src/__tests__/workspace-runtime.test.ts \ + server/src/services/workspace-git-snapshot-streaming.test.ts) + if [ -n "$changed" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: @@ -123,6 +141,7 @@ jobs: echo "Materialized dependency reuse is disabled by an unmodelled install hook. Retaining cold setup and comparing test commands only." >> "$GITHUB_STEP_SUMMARY" - name: Prepare unchanged baseline + if: steps.test_scope.outputs.run == 'true' run: | base_sha=$(git rev-parse HEAD^1) git worktree add --detach ../baseline "$base_sha" @@ -135,12 +154,17 @@ jobs: pnpm --filter @paperclipai/plugin-sdk ensure-build-deps - name: Compare complete commands and executed case inventories + if: steps.test_scope.outputs.run == 'true' env: BENCHMARK_OUTPUT: ${{ runner.temp }}/ci-work-benchmark run: | node scripts/benchmark-ci-work.mjs ../baseline "$BENCHMARK_OUTPUT" cat "$BENCHMARK_OUTPUT/summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Exercise restored dependencies for cache-only changes + if: steps.test_scope.outputs.run != 'true' && steps.qualification.outputs.cacheable == 'true' + run: pnpm exec vitest run --project @paperclipai/shared + - name: Retain timings, inventories and failure logs if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a @@ -148,3 +172,120 @@ jobs: name: ci-work-benchmark path: ${{ runner.temp }}/ci-work-benchmark retention-days: 14 + + rust_prime: + if: >- + contains(github.event.pull_request.labels.*.name, 'profile-ci') && + (github.event.action != 'labeled' || github.event.label.name == 'profile-ci') + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Select the pinned compiler and workspace path + id: workspace + working-directory: packages/paperclip-runner + run: | + toolchain=$(rustup show active-toolchain | awk '{print $1}') + echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV" + rustup toolchain list | awk '{print $1}' | grep -vx "$toolchain" | xargs -r -n1 rustup toolchain uninstall || true + pinned="$HOME/paperclip-runner-rust" + ln -s "$GITHUB_WORKSPACE/packages/paperclip-runner/runner" "$pinned" + echo "path=$pinned" >> "$GITHUB_OUTPUT" + - name: Qualify the dependency cache + id: dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ steps.workspace.outputs.path }} -> target + shared-key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} + cache-workspace-crates: false + cache-bin: false + save-if: 'true' + - name: Build the exact debug units + working-directory: packages/paperclip-runner + run: cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins + - name: Compute workspace artifact identity + id: debug + run: node scripts/runner-debug-crate-cache.mjs plan + - name: Save workspace artifacts in this PR only + id: artifacts + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.debug.outputs.key }} + path: ${{ steps.debug.outputs.paths }} + + rust_restore: + needs: rust_prime + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Select the pinned compiler and workspace path + id: workspace + working-directory: packages/paperclip-runner + run: | + toolchain=$(rustup show active-toolchain | awk '{print $1}') + echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV" + rustup toolchain list | awk '{print $1}' | grep -vx "$toolchain" | xargs -r -n1 rustup toolchain uninstall || true + pinned="$HOME/paperclip-runner-rust" + ln -s "$GITHUB_WORKSPACE/packages/paperclip-runner/runner" "$pinned" + echo "path=$pinned" >> "$GITHUB_OUTPUT" + - name: Qualify the dependency cache + id: dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ steps.workspace.outputs.path }} -> target + shared-key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} + cache-workspace-crates: false + cache-bin: false + save-if: 'false' + - name: Compute workspace artifact identity + id: debug + run: node scripts/runner-debug-crate-cache.mjs plan + - name: Restore workspace artifacts in this PR only + id: artifacts + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.debug.outputs.key }} + path: ${{ steps.debug.outputs.paths }} + - name: Verify fresh-host cache reuse + env: + DEPENDENCIES_HIT: ${{ steps.dependencies.outputs.cache-hit }} + ARTIFACTS_HIT: ${{ steps.artifacts.outputs.cache-hit }} + run: | + test "$DEPENDENCIES_HIT" = true + test "$ARTIFACTS_HIT" = true + node scripts/runner-debug-crate-cache.mjs mark-current + cd packages/paperclip-runner + started=$(date +%s) + cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins --message-format=json > "$RUNNER_TEMP/rust-restored-build.jsonl" + echo "Restored Rust build: $(($(date +%s) - started)) seconds." >> "$GITHUB_STEP_SUMMARY" + node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + const artifacts = readFileSync(`${process.env.RUNNER_TEMP}/rust-restored-build.jsonl`, 'utf8') + .trim().split('\n').map(line => JSON.parse(line)).filter(row => row.reason === 'compiler-artifact'); + if (!artifacts.length || artifacts.some(row => !row.fresh)) { + throw new Error(`Cache hit rebuilt units: ${JSON.stringify(artifacts.filter(row => !row.fresh).map(row => row.package_id))}`); + } + console.log(`All ${artifacts.length} Cargo units were fresh on a new host.`); + NODE + - name: Retain Cargo reuse evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: rust-cache-qualification + path: ${{ runner.temp }}/rust-restored-build.jsonl + retention-days: 14 diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 455512ee2de..420f1d98259 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -370,7 +370,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -784,7 +784,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -1035,7 +1035,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -1376,7 +1376,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index a8a29e433b5..b0319840ec1 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -92,7 +92,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-typecheck-v2 + shared-key: release-typecheck-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Rebuild workspace code and rerun every check. Cache only compiled # dependencies; never restore installed executables from cargo/bin. cache-workspace-crates: false @@ -398,7 +398,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Rebuild workspace code and rerun every check. Cache only compiled # dependencies; never restore installed executables from cargo/bin. cache-workspace-crates: false @@ -680,7 +680,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 with: workspaces: ${{ steps.native_rust_workspace.outputs.path }} -> target - shared-key: release-native-v2-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} + shared-key: release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }}-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} cache-workspace-crates: false cache-bin: false cache-on-failure: false diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 8976eccb990..ec278667695 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1095,6 +1095,22 @@ verifying Vite/HMR behavior. For normal issue work, start with the smallest targeted check that proves the change. Reserve repo-wide typecheck/build/test runs for PR-ready handoff or changes broad enough that narrow checks do not cover the risk. +The Runner transport suite keeps attachment recovery, warm attachment, live +adoption, and suspension in separate test files. Vitest can schedule these +independent fixtures on its existing file workers; cases within each file stay +serial. The package sequencer starts the ten files measured above 50 seconds in +the November 4, 2026 AWS run first, ordered by approximate duration. This includes +transport deadlines, settlement, provider launch, recovery, and streaming waits, +so fresh CI runners do not have to infer duration from file size. Package CI uses +up to four workers, bounded by available CPUs; local runs retain Vitest defaults. +Vitest's file selection and shard membership are unchanged. Real lease/deadline +waits and assertions are unchanged. Run the complete +family, including the separate deadline and settlement regressions, with: + +```sh +pnpm --filter @paperclipai/paperclip-runner exec vitest run runnerd-codex-transport +``` + #### Embedded Postgres template cache To run the suite with one owned PostgreSQL server and a separate disposable @@ -3187,7 +3203,7 @@ Run the credential-free real-process restart suite with: ```sh pnpm --filter @paperclipai/paperclip-runner build:runner-binaries pnpm exec vitest run server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts -pnpm --filter @paperclipai/paperclip-runner exec vitest run src/live/runnerd-codex-transport.test.ts -t 'adopts a live runner' +pnpm --filter @paperclipai/paperclip-runner exec vitest run src/live/runnerd-codex-transport-adoption.test.ts -t 'adopts a live runner' ``` The suite uses isolated PostgreSQL state, isolated `PAPERCLIP_HOME` roots, real diff --git a/doc/RELEASE-AUTOMATION-SETUP.md b/doc/RELEASE-AUTOMATION-SETUP.md index 324a6adb07e..021188f2e53 100644 --- a/doc/RELEASE-AUTOMATION-SETUP.md +++ b/doc/RELEASE-AUTOMATION-SETUP.md @@ -368,7 +368,14 @@ readiness can succeed. A failed lane does not cancel the other lane. Both lanes restore Cargo dependencies with the pinned Rust Cache action. The compiler comes from the Runner package's `rust-toolchain.toml` before the action computes its key. Compiler and Cargo metadata changes select a new cache. The -`release-runner-v2` shared key avoids separate copies for these lanes. +`release-runner-v2` shared key avoids separate copies for these lanes. Its suffix +hashes the virtual workspace's `runner/Cargo.toml` and ancestor Cargo configuration. +Rust Cache discovers member manifests through Cargo metadata, which omits this +virtual manifest. Without the suffix, changing `[profile.dev]` can restore an +exact but obsolete dependency cache on every run. GitHub caches are immutable, +so the successful rebuild cannot replace that old entry. The profile/configuration +hash also partitions fallback restores. Every reader and writer uses the same +inputs, including the separate typecheck and cross-target caches. Only the Rust lane saves this cache. After verification it also runs `build:rust` to warm the debug dependencies used by the protocol lane; its own tests already warm release dependencies. The cache writer is shorter than the protocol lane. @@ -385,13 +392,36 @@ paths and the PR lanes can restore master's entry. The guard tests under Rust cache step in both workflows. Workspace crates and installed Cargo binaries are excluded from this entry. -Every run runs all assigned checks, including on a cache hit. Only an -own-repository master-push run verifying that push's exact SHA can restore the -cache, and only a successful Rust lane saves it. PR, tag, and manual candidate +Every run runs all assigned checks, including on a cache hit. In release +verification, only an own-repository master-push run verifying that push's exact +SHA can restore the cache, and only a successful Rust lane saves it. The PR +workflow restores master's cache read only. Tag and manual candidate release verification compile without this cache. A miss or eviction costs compilation time but does not change the checks. To discard old dependency caches, increment the shared-key version and let the next successful master verification warm it. +### Installed JavaScript dependencies + +`setup-ci-dependencies` can restore exact `node_modules` trees and separately +certified shared/SDK build outputs. `scripts/ci-dependency-cache.mjs` hashes +manifests, resolution, patches, install scripts, and the runtime/checkout layout. +The root workspace-link hook and the server/Runner Copilot asset hooks are known +postinstall commands. An exact dependency restore replays these hooks before +validating workspace links and build outputs. Copilot defers installation in an +unbuilt source checkout; its normal build installs and verifies the assets. +Provider assets are not included in the dependency cache. Any unrecognized +workspace lifecycle hook still disables materialized reuse. Only a frozen install +from the canonical master push can publish the shared entries. + +For CI cache changes, the `profile-ci` label runs the paired setup benchmark +and exercises restored dependencies with the shared-package tests. The longer +server test comparison runs when its measured scripts or fixtures change; +cache-only changes do not require unchanged test commands to run faster. +It also builds Rust units on one GitHub-hosted runner and restores the dependency +and workspace caches on a second runner with a fresh checkout. Both restores +must be exact, and Cargo must report every unit as fresh. These qualification +caches are scoped to the PR and run; they cannot populate master's caches. + ### Runner release crate cache The dependency entry leaves the Runner's own crate out, and a fresh checkout diff --git a/packages/paperclip-runner/scripts/vitest-sequencer.mjs b/packages/paperclip-runner/scripts/vitest-sequencer.mjs new file mode 100644 index 00000000000..c8069693070 --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-sequencer.mjs @@ -0,0 +1,27 @@ +import { BaseSequencer } from "vitest/node"; + +const slowFiles = [ + "runnerd-codex-transport.test.ts", + "runnerd-codex-transport-settlement.test.ts", + "cursor-cloud-runner-launch.test.ts", + "runnerd-codex-transport-deadlines.test.ts", + "opencode-server-driver.test.ts", + "acpx-controller-recovery.test.ts", + "runnerd-codex-transport-warm-attachment.test.ts", + "runnerd-opencode-streaming-tools.test.ts", + "live-session.test.ts", + "runnerd-codex-transport-adoption.test.ts", +]; + +const priority = (file) => { + const name = file.moduleId.replaceAll("\\", "/").split("/").at(-1); + const index = slowFiles.indexOf(name); + return index < 0 ? slowFiles.length : index; +}; + +export class RunnerSequencer extends BaseSequencer { + async sort(files) { + const sorted = await super.sort(files); + return sorted.sort((a, b) => priority(a) - priority(b)); + } +} diff --git a/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs b/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs new file mode 100644 index 00000000000..781fe81d8d1 --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; +import { BaseSequencer } from "vitest/node"; +import { RunnerSequencer } from "./vitest-sequencer.mjs"; + +const project = { name: "runner", config: { isolate: true, sequence: { groupOrder: 0 } } }; +const files = [ + "large.test.ts", + "runnerd-codex-transport.test.ts", + "small.test.ts", + "runnerd-codex-transport-settlement.test.ts", + "runnerd-codex-transport-deadlines.test.ts", +].map((name) => ({ moduleId: `/repo/src/live/${name}`, project })); +const ctx = { + config: { root: "/repo", shard: { index: 1, count: 2 } }, + cache: { + getFileTestResults: () => undefined, + getFileStats: (key) => ({ size: 100 - files.findIndex((file) => key.endsWith(file.moduleId.slice(6))) }), + }, +}; + +test("starts real-wait transport files first without changing test membership", async () => { + const sorted = await new RunnerSequencer(ctx).sort(files); + assert.deepEqual(sorted, [files[1], files[3], files[4], files[0], files[2]]); + assert.equal(new Set(sorted).size, files.length); + assert.deepEqual(await new BaseSequencer(ctx).sort(files), files); +}); + +test("selected subsets stay selected and retain default ordering", async () => { + const selected = [files[2], files[0]]; + assert.deepEqual(await new RunnerSequencer(ctx).sort(selected), await new BaseSequencer(ctx).sort(selected)); + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[4]]), [files[4]]); + assert.deepEqual(await new RunnerSequencer(ctx).sort([]), []); +}); + +test("keeps Vitest shard membership and schedules waits within each shard", async () => { + const all = []; + for (const index of [1, 2]) { + const shardCtx = { ...ctx, config: { ...ctx.config, shard: { index, count: 2 } } }; + const sequencer = new RunnerSequencer(shardCtx); + const shard = await sequencer.shard(files); + assert.deepEqual(shard, await new BaseSequencer(shardCtx).shard(files)); + const sorted = await sequencer.sort(shard); + assert.deepEqual(new Set(sorted), new Set(shard)); + all.push(...sorted); + } + assert.equal(all.length, files.length); + assert.deepEqual(new Set(all), new Set(files)); +}); + +test("recognizes Windows paths", async () => { + const deadline = { ...files[4], moduleId: "C:\\repo\\src\\live\\runnerd-codex-transport-deadlines.test.ts" }; + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[0], deadline]), [deadline, files[0]]); +}); + +test("starts measured slow provider files before short files without adding work", async () => { + const streaming = { ...files[0], moduleId: "/repo/src/live/runnerd-opencode-streaming-tools.test.ts" }; + const launch = { ...files[0], moduleId: "/repo/src/control-plane/cursor-cloud-runner-launch.test.ts" }; + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[0], streaming, files[4], launch]), + [launch, files[4], streaming, files[0]]); +}); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts new file mode 100644 index 00000000000..424db66444b --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts @@ -0,0 +1,556 @@ +import { + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +async function verifyLiveRunnerAdoption( + mismatchedCheckpoint: boolean, + mismatchedArtifact = false, + goalMidTurn = false, + detachBeforeCleanup = false, + startWithoutCheckpoint = false, + preparedTask = false, + adoptionAuthorization: "allow" | "deny" | "legacy" | "legacy-missing" | "missing-binding" = "allow", + toolHandlerOrder?: "before" | "after" | "completion-before" | "completion-after", + uncertainAtClose = false, + semanticRecoveryMode?: "denied-before-cancel" | "cleanup-only", + stopExitProofFault?: "missing" | "false", +) { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); + const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; + const completionPath = join(stateDirectory, "pending-completion.json"); + if (pendingCompletion) await writeFile(completionPath, JSON.stringify({ + schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed the synthetic work", + completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [], + })); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + const checkpoint = vi.fn(async () => undefined); + server.on("upgrade", (request, socket, head) => { + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Expected adoption test listener"); + const registration = vi.fn(async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + ...(mismatchedArtifact || uncertainAtClose || semanticRecoveryMode === "cleanup-only" ? { checkpoint } : {}), + release: async () => { + if (authority === next) authority = null; + }, + }; + }); + const identity = { + runnerInstanceId: "runner-live-adopt", + environmentLeaseId: "lease-live-adopt", + runId: "run-live-adopt", + normalizedSessionId: "session-live-adopt", + turnId: "turn-live-adopt", + itemId: "item-live-adopt", + }; + const sharedOptions = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, ...(pendingCompletion ? ["--terminal-result-file", completionPath] : toolHandlerOrder ? ["--emit-tool-call"] : []), ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), + stateDirectory, + prpIdentity: identity, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + controlPlaneRegistration: registration, + }; + const prepareLaunch = vi.fn(async (requirements: RunnerProviderLaunchRequirements) => ({ + configurationId: "adopted-preparation", requirementsDigest: runnerLaunchRequirementsDigest(requirements), + provider: createRunnerProviderLaunch(requirements, { options: { codexArgs: sharedOptions.codexArgs }, + root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), + codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "" }), + })); + const options = { ...sharedOptions, ...(preparedTask ? { + prepareLaunch, runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), + runnerProcessLauncher: (spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: { ...process.env, ...spec.environment }, + detached: process.platform !== "win32", stdio: "ignore" }); + return { child, processGroupId: child.pid ?? null, completion: new Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }>((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); + }) }; + }, + } : {}) }; + const first = createCapabilityRunnerdCodexTransport(options); + const pendingTool = vi.fn(() => new Promise>(() => undefined)); + if (toolHandlerOrder) first.transport.setServerRequestHandler(pendingTool); + let releaseRecovery!: () => void; + const recoveryGate = new Promise(resolve => { releaseRecovery = resolve; }); + const recoveryDenied = new Error("execution authority revoked after Stop"); + const recoveredTool = vi.fn(async () => { + if (semanticRecoveryMode) { + // Model server authority denying replay after Stop cleared the task lock. + // The gate holds that denial until reattachment has returned its session, + // before the caller invokes the eventual onSession cancellation callback. + await recoveryGate; + throw recoveryDenied; + } + return { result: { receipt: "known-outcome" }, isError: false }; + }); + const redispatchedTool = vi.fn(async () => ({ success: true, contentItems: [] })); + let runnerPid: number | null = null; + let runnerStartedAt: string | null = null; + let adopted: ReturnType | null = + null; + let restoreIntervalSpies: () => void = () => undefined; + try { + let opened: Record; + try { + opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: toolHandlerOrder && !pendingCompletion ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : codexSemanticToolSpecs(), + }); + } catch (error) { + const stderr = await readFile( + join(stateDirectory, "diagnostics", "runnerd.stderr.log"), + "utf8", + ).catch(() => ""); + throw new Error( + `${String(error)}\n${JSON.stringify(first.evidence())}${stderr ? `\n${stderr}` : ""}`, + ); + } + runnerPid = first.evidence().runnerPid; + runnerStartedAt = first.transport.processInfo().startedAt; + expect(runnerPid).toEqual(expect.any(Number)); + + if (goalMidTurn) { + await first.transport.request("thread/goal/set", { objective: "Recover a live goal", status: "active" }); + for await (const event of first.transport.notifications()) { + if (event.method === "turn/started") break; + } + } + + if (toolHandlerOrder) { + await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); + await vi.waitFor(() => expect(pendingTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); + } + await first.detachControllerForRestart(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + if (goalMidTurn) { + await writeFile(join(stateDirectory, "emit-goal-item"), "emit"); + // runnerd need not poll the provider into its PRP outbox while disconnected. + // Wait for flushed provider output, not a platform-dependent final poll + // racing the disconnect. Adoption must still bind that buffered item. + await vi.waitFor(async () => { + expect(await readFile(join(stateDirectory, "emit-goal-item.sent"), "utf8")).toBe("sent"); + }, { timeout: 5_000 }); + } + + const controlPlaneStatePath = join( + stateDirectory, + "control-plane", + "control-plane-state.json", + ); + const compactProviderIdentityEvents = async () => { + const controlPlaneState = JSON.parse( + await readFile(controlPlaneStatePath, "utf8"), + ) as { committedEvents: Array<{ eventType: string }> }; + controlPlaneState.committedEvents = + controlPlaneState.committedEvents.filter( + (event) => + event.eventType !== "harness.ready" && + event.eventType !== "session.started" && + event.eventType !== "session.resumed", + ); + await writeFile( + controlPlaneStatePath, + `${JSON.stringify(controlPlaneState, null, 2)}\n`, + { mode: 0o600 }, + ); + }; + if (!startWithoutCheckpoint) await compactProviderIdentityEvents(); + + if (preparedTask) { + if (adoptionAuthorization !== "legacy") await rm(join(stateDirectory, "launch-preparation.json")); + prepareLaunch.mockClear(); + registration.mockClear(); + if (adoptionAuthorization === "missing-binding") { + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + delete retained.preparedLaunchConfigurations; + await writeFile(controlPlaneStatePath, JSON.stringify(retained), { mode: 0o600 }); + } + } + const authorizeRunnerAdoption = vi.fn(async () => { + expect(registration).not.toHaveBeenCalled(); + if (adoptionAuthorization === "deny") throw new Error("project grant revoked"); + }); + const duplicateLauncher = vi.fn(() => { + throw new Error("duplicate runner spawn attempted"); + }); + const openedThread = opened.thread as Record; + const signal = vi.fn(() => true); + const probeError = new Error("fixture remote process identity is unavailable"); + let rejectLiveness = false; + const intervals = uncertainAtClose ? vi.spyOn(globalThis, "setInterval") : null; + const clearedIntervals = uncertainAtClose ? vi.spyOn(globalThis, "clearInterval") : null; + restoreIntervalSpies = () => { intervals?.mockRestore(); clearedIntervals?.mockRestore(); }; + adopted = createCapabilityRunnerdCodexTransport({ + ...options, + ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), + ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), + ...(semanticRecoveryMode === "cleanup-only" ? { recoveryPurpose: "stop_cleanup" as const } : {}), + // Hash different stable bytes without replacing the real runner artifact + // used by concurrent tests. Adoption must never execute this path. + ...(mismatchedArtifact + ? { + runnerBinary: resolve(import.meta.dirname, "../../package.json"), + runnerReconnectGraceMs: 150, + } + : {}), + ...(toolHandlerOrder && !pendingCompletion ? { recoverSemanticToolInput: recoveredTool } : {}), + resumeDynamicTools: pendingCompletion ? codexSemanticToolSpecs() : toolHandlerOrder ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : [], + resumeProviderSession: startWithoutCheckpoint ? undefined : { + driverSessionId: String(openedThread.id), + providerSessionId: mismatchedCheckpoint + ? "wrong-provider-session" + : String(openedThread.sessionId), + }, + runnerProcessLauncher: duplicateLauncher, + adoptExistingRunner: { + pid: runnerPid!, + processGroupId: runnerPid, + startedAt: runnerStartedAt, + signal, + isAlive: () => { + if (rejectLiveness) return Promise.reject(probeError); + try { + process.kill(runnerPid!, 0); + return true; + } catch { + return false; + } + }, + }, + }); + if (preparedTask && ["deny", "legacy-missing", "missing-binding"].includes(adoptionAuthorization)) { + await expect(adopted.transport.request("thread/read", {})).rejects.toThrow( + adoptionAuthorization === "deny" ? "project grant revoked" : adoptionAuthorization === "legacy-missing" + ? "runner_launch_recovery_requirements_unavailable" : "runner_launch_recovery_configuration_unavailable"); + expect(registration).not.toHaveBeenCalled(); + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(signal).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(authorizeRunnerAdoption).toHaveBeenCalledTimes(adoptionAuthorization === "deny" ? 1 : 0); + return; + } + if (mismatchedArtifact) { + await expect( + adopted.transport.request("thread/read", {}), + ).rejects.toThrow("native_adopted_runner_authentication_timeout"); + expect(authority?.activeRunnerConnectionCount()).toBe(0); + await expect( + adopted.transport.request("turn/start", { + input: [{ type: "text", text: "must not be dispatched" }], + }), + ).rejects.toThrow("native_adopted_runner_authentication_timeout"); + await adopted.transport.close(); + expect(signal).not.toHaveBeenCalled(); + expect(checkpoint).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + const retained = JSON.parse( + await readFile(controlPlaneStatePath, "utf8"), + ) as { + identity: unknown; + commands: Array<{ type: string; status: string }>; + }; + expect(retained.identity).toEqual(identity); + expect( + retained.commands.some( + (command) => + command.type === "runner.drain" && command.status === "pending", + ), + ).toBe(true); + expect(retained.commands.map((command) => command.type)).not.toEqual( + expect.arrayContaining(["runner.suspend"]), + ); + expect(retained.commands.map((command) => command.type)).not.toEqual( + expect.arrayContaining(["turn.stop"]), + ); + return; + } + if (mismatchedCheckpoint) { + await expect( + adopted.transport.request("thread/read", {}), + ).rejects.toThrow("native_adopted_provider_identity_mismatch"); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + return; + } + if (toolHandlerOrder?.endsWith("before")) adopted.transport.setServerRequestHandler(redispatchedTool); + await expect(adopted.transport.request(startWithoutCheckpoint ? "thread/start" : "thread/read", {})).resolves.toEqual( + expect.objectContaining({ + thread: expect.objectContaining({ id: "codex-thread-1" }), + }), + ); + if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); + if (semanticRecoveryMode) { + const retained = authority as DurablePrpControlPlane | null; + expect(retained).not.toBeNull(); + expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); + expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); + if (semanticRecoveryMode === "denied-before-cancel") { + await vi.waitFor(() => expect(recoveredTool).toHaveBeenCalledOnce()); + // This is a real Core replay callback, not an artificial transport error. + // Deliver its denial before giving the caller its cancellation turn. + releaseRecovery(); + await vi.waitFor(async () => { + await expect(adopted!.transport.request("thread/read", {})).rejects.toBe(recoveryDenied); + }); + await expect(adopted.transport.request("turn/interrupt", {})).rejects.toBe(recoveryDenied); + expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); + } else { + // Let all startup/handler microtasks and a real reconciliation read run. + // Cleanup must not enter replay even while its authority would deny it. + await adopted.transport.request("thread/read", {}); + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).not.toHaveBeenCalled(); + releaseRecovery(); + await expect(adopted.transport.request("turn/interrupt", {})).resolves.toEqual({}); + expect(retained!.store.state.commands.filter(command => command.type === "turn.interrupt")).toMatchObject([{ status: "completed" }]); + } + expect(redispatchedTool).not.toHaveBeenCalled(); + expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); + expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(duplicateLauncher).not.toHaveBeenCalled(); + if (semanticRecoveryMode === "cleanup-only") { + const faultedStopProof = vi.fn(); + if (stopExitProofFault) { + const queue = retained!.queueCommand.bind(retained!); + vi.spyOn(retained!, "queueCommand").mockImplementation((type, ...args) => { + const command = queue(type, ...args); + if (type === "turn.stop") { + let result = command.result; + Object.defineProperty(command, "result", { + enumerable: true, + configurable: true, + get: () => result, + set: value => { + const receipt = value as { result?: Record } | null; + if (receipt?.result && typeof receipt.result === "object") { + faultedStopProof(receipt.result.providerExitConfirmed); + const stopped = { ...receipt.result }; + if (stopExitProofFault === "missing") delete stopped.providerExitConfirmed; + else stopped.providerExitConfirmed = false; + result = { ...receipt, result: stopped }; + } else result = value; + }, + }); + } + return command; + }); + await expect(adopted.transport.close()).rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); + expect(faultedStopProof).toHaveBeenCalledWith(true); + expect(checkpoint).toHaveBeenCalledExactlyOnceWith("unsettled"); + } else { + await expect(adopted.transport.close()).resolves.toBeUndefined(); + expect(checkpoint).toHaveBeenCalledExactlyOnceWith("settled"); + expect(retained!.store.state.commands.filter(command => command.type === "turn.stop")).toMatchObject([{ + status: "completed", result: { result: { providerExitConfirmed: true } }, + }]); + } + expect(retained!.semanticToolResultsSettled()).toBe(false); + expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); + expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); + if (!stopExitProofFault) { + expect(retained!.store.state.commands.filter(command => command.type === "runner.suspend")).toMatchObject([{ status: "completed" }]); + const runnerState = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(runnerState).toMatchObject({ ...identity, lifecycle: "suspended" }); + } + const providerState = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); + expect(providerState).toMatchObject({ activeProviderTurnId: null, pendingEvents: [], queuedEvents: [] }); + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).not.toHaveBeenCalled(); + } + return; + } + if (toolHandlerOrder) { + await vi.waitFor(() => { + expect(pendingCompletion ? redispatchedTool : recoveredTool).toHaveBeenCalledOnce(); + const result = authority!.store.state.commands.find(command => command.type === "semantic_tool.result"); + expect(result?.status).toBe("completed"); + }, { timeout: 5_000 }); + if (pendingCompletion) { + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).toHaveBeenCalledWith(expect.objectContaining({ params: expect.objectContaining({ tool: "paperclip_finish" }) })); + } else expect(redispatchedTool).not.toHaveBeenCalled(); + } + expect(adopted.evidence().runnerPid).toBe(runnerPid); + expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); + if (startWithoutCheckpoint) { + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + for (const type of ["run.prepare", "session.open"]) { + expect(retained.commands.filter((command: { type: string }) => command.type === type)).toHaveLength(1); + } + } + if (goalMidTurn) { + const observed = await Promise.race([ + (async () => { + for await (const notification of adopted!.transport.notifications()) { + if (notification.method === "item/started") return notification; + } + throw new Error("recovered goal item was lost"); + })(), + new Promise((_, reject) => setTimeout(() => reject(new Error("recovered item timed out")), 5_000)), + ]); + expect(observed.params).toMatchObject({ threadId: "codex-thread-1", turnId: "provider-goal-turn-1" }); + } + expect(duplicateLauncher).not.toHaveBeenCalled(); + if (preparedTask) { + if (adoptionAuthorization === "legacy") { + expect(prepareLaunch).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ identity }), expect.objectContaining({ configurationId: "adopted-preparation" })); + expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); + } else { + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(authorizeRunnerAdoption).toHaveBeenCalledExactlyOnceWith("adopted-preparation", identity); + } + expect((await adopted.transport.request("thread/read", {})).thread).toMatchObject({ cwd: tmpdir() }); + } + if (uncertainAtClose) { + const before = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + const retainedAuthority = authority as DurablePrpControlPlane | null; + rejectLiveness = true; + await expect(adopted.transport.close()).rejects.toBe(probeError); + expect(retainedAuthority?.activeRunnerConnectionCount()).toBe(0); + expect(intervals!.mock.results.length).toBeGreaterThan(0); + for (const interval of intervals!.mock.results) if (interval.type === "return") expect(clearedIntervals).toHaveBeenCalledWith(interval.value); + const notificationsClosed = (async () => { for await (const _event of adopted!.transport.notifications()) {} return true; })(); + let queueDeadline: ReturnType | undefined; + try { + expect(await Promise.race([notificationsClosed, new Promise(resolve => { queueDeadline = setTimeout(() => resolve(false), 250); })])).toBe(true); + } finally { clearTimeout(queueDeadline); } + await expect(adopted.transport.close()).rejects.toBe(probeError); + expect(adopted.transport.processInfo()).toMatchObject({ exited: false }); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(signal).not.toHaveBeenCalled(); + expect(checkpoint).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + expect(retained.identity).toEqual(before.identity); + expect(retained.commands).toEqual(before.commands); + return; + } + expect(adopted.evidence().diagnostics).toContain( + `adopted runner ${runnerPid} authenticated to its durable PRP authority`, + ); + if (!startWithoutCheckpoint) { + expect(adopted.evidence().diagnostics).toContain( + "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", + ); + expect(adopted.evidence().diagnostics).toContain( + "confirmed adopted provider identity against authenticated recovery session.snapshot", + ); + } + if (detachBeforeCleanup) { + await adopted.detachControllerForRestart(); + await adopted.transport.close("old controller finalizer"); + expect(signal).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + const commandTypes = retained.commands.map((command: { type: string }) => command.type); + expect(commandTypes).not.toContain("turn.stop"); + expect(commandTypes).not.toContain("runner.suspend"); + } + } finally { + releaseRecovery(); + await adopted?.transport.close().catch(() => undefined); + if (uncertainAtClose) { + const providerPid = adopted?.evidence().codexPid; + if (providerPid) { + try { process.kill(providerPid, "SIGKILL"); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } + } + await (authority as DurablePrpControlPlane | null)?.stop(); + } + restoreIntervalSpies(); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // The adopted runner normally exits after its durable suspend command. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +} + +it( + "adopts a live runner on the same durable authority without spawning a duplicate", + () => verifyLiveRunnerAdoption(false), + 30_000, +); + +it("adopts a prepared runner after losing its controller launch cache without preparing resources again", + () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); + +it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", + mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); + +it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); + +it( + "blocks adopted runner artifact drift without duplicate launch, checkpoint replacement, or process signals", + () => verifyLiveRunnerAdoption(false, true), + 15_000, +); + +it( + "rejects a live runner whose provider identity mismatches the compacted checkpoint", + () => verifyLiveRunnerAdoption(true), + 30_000, +); + +it("binds buffered mid-goal items only after the authenticated recovery snapshot", () => verifyLiveRunnerAdoption(false, false, true), 30_000); + +it("keeps an adopted runner alive when the detached controller finalizer closes", () => verifyLiveRunnerAdoption(false, false, true, true), 30_000); + +it("adopts an opening session without a checkpoint instead of bootstrapping a duplicate provider", () => verifyLiveRunnerAdoption(false, false, false, false, true), 30_000); + +it.each(["before", "after"] as const)("settles a saved tool outcome with the handler installed %s recovery startup", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); + +it.each(["completion-before", "completion-after"] as const)("recovers a pending finish without an ordinary-tool lookup (%s)", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); + +it("exposes semantic recovery before cancellation when the authority has been revoked", () => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "denied-before-cancel"), 30_000); + +it.each(["before", "after", "completion-before", "completion-after"] as const)("allows cleanup-only adoption to cancel without replaying a pending semantic input (%s)", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order, false, "cleanup-only"), 30_000); + +it.each(["missing", "false"] as const)("refuses cleanup settlement when the exact stop receipt has %s provider exit proof", fault => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "cleanup-only", fault), 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts new file mode 100644 index 00000000000..72b95a8206c --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts @@ -0,0 +1,1077 @@ +import { + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type PreparedRunnerProviderLaunch, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; +import { cp, lstat, mkdtemp, readFile, readdir, readlink, rm, writeFile } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +it.each([ + ...[ + "before-result", + "after-result", + "after-activation", + "before-confirmation", + "after-confirmation", + ].flatMap((lossPoint) => + [false, true].map((routed) => ({ + lossPoint, + routed, + recoveryFault: "none", + })), + ), + ...["endpoint", "missing-capability", "listen", "malformed-core"].map( + (recoveryFault) => ({ + lossPoint: "after-result", + routed: true, + recoveryFault, + }), + ), + ...["before_bootstrap", "before_spawn", "before_authentication"].map( + (stage) => ({ + lossPoint: "after-result", + routed: true, + recoveryFault: `authorize_${stage}`, + }), + ), + ...[ + "attach-wait", + ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER + ? ["attach-capability"] + : []), + ].map((recoveryFault) => ({ + lossPoint: "before-result", + routed: true, + recoveryFault, + })), + ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER + ? [ + { + lossPoint: "after-result", + routed: false, + recoveryFault: "legacy-parser", + }, + ] + : []), + ...["before-result", "after-result", "after-activation", "after-confirmation"].map(lossPoint => ({ + lossPoint, routed: true, recoveryFault: "none", executionPrepared: true, + })), + { + lossPoint: "after-result", + routed: true, + recoveryFault: "none", + externalArtifact: true, + }, + { + lossPoint: "after-confirmation", + routed: true, + recoveryFault: "none", + ordinaryFollowup: true, + }, + ...[ + "missing_snapshot", + "rejected_snapshot", + "wrong_thread", + "callback_failure", + "callback_async_failure", + ].map((ordinaryFollowup) => ({ + lossPoint: "after-confirmation", + routed: true, + recoveryFault: "none", + ordinaryFollowup, + })), +])( + "recovers a warm attachment with a fresh controller and runner ($lossPoint, routed=$routed, fault=$recoveryFault, followup=$ordinaryFollowup, external=$externalArtifact, prepared=$executionPrepared)", + async (testCase) => { + const { lossPoint, routed, recoveryFault } = testCase; + const externalArtifact = "externalArtifact" in testCase && testCase.externalArtifact; + const executionPrepared = "executionPrepared" in testCase && testCase.executionPrepared; + const configurations = new Map(); + const ordinaryFollowup = + "ordinaryFollowup" in testCase && testCase.ordinaryFollowup; + const snapshotFault = + typeof ordinaryFollowup === "string" && + !ordinaryFollowup.startsWith("callback_"); + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-attach-restart-"), + ); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const handles: ReturnType[] = []; + const routes = new Map< + string, + { core: DurablePrpControlPlane; generation: symbol } + >(); + const routeCalls: string[] = []; + let recovering = false; + let recoveryClaimCurrent = true; + let recoveryFenceActive = true; + const completionSnapshotIds: string[] = []; + let rejectHeldCompletion: ((error: Error) => void) | undefined; + let markHeldCompletionEntered!: () => void; + const heldCompletionEntered = new Promise((resolve) => { + markHeldCompletionEntered = resolve; + }); + let snapshotObserverSpy: ReturnType | undefined; + const routeServer = createServer((_request, response) => + response.writeHead(404).end(), + ); + routeServer.on("upgrade", (request, socket, head) => { + const entry = routes.get(request.url ?? ""); + if (!entry) { + socket.destroy(); + return; + } + entry.core.handleUpgrade(request, socket, request.url!, head); + }); + if (routed) + await new Promise((resolveListen) => + routeServer.listen(0, "127.0.0.1", resolveListen), + ); + const routeAddress = routeServer.address(); + const routePort = + routeAddress && typeof routeAddress === "object" ? routeAddress.port : 0; + const registration = async ( + core: DurablePrpControlPlane, + identity = core.store.state.identity, + ) => { + if ( + recovering && + ordinaryFollowup && + recoveryFenceActive && + !recoveryClaimCurrent + ) { + throw new Error("fixture old recovery claim is no longer current"); + } + const path = `/api/runner/v1/connect/${identity.runId}`; + const generation = Symbol(); + routes.set(path, { core, generation }); + routeCalls.push(identity.runId); + return { + connection: + recovering && recoveryFault === "listen" + ? { + mode: "listen" as const, + listenAddress: "0.0.0.0" as const, + listenPort: routePort, + listenPath: path, + } + : { + mode: "connect" as const, + connectUrl: `ws://127.0.0.1:${routePort}${path}${recovering && recoveryFault === "endpoint" ? "/changed" : ""}`, + }, + release: () => { + if (routes.get(path)?.generation === generation) routes.delete(path); + }, + }; + }; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore(options); + cores.push(core); + if (recovering && snapshotFault) { + const getCommand = core.getCommand.bind(core); + snapshotObserverSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((id) => { + const command = getCommand(id); + if ( + command?.type !== "session.snapshot" || + command.status !== "completed" + ) + return command; + const observed = structuredClone(command); + if (ordinaryFollowup === "rejected_snapshot") { + observed.status = "failed"; + observed.result = { + result: { message: "fixture snapshot observation rejected" }, + }; + } else if (ordinaryFollowup === "wrong_thread") { + observed.result = { + ...observed.result, + result: { + ...(observed.result?.result as Record), + driverSessionId: "foreign-thread", + providerSessionId: "foreign-thread", + }, + }; + } else observed.result = { result: { status: "session_open" } }; + return observed; + }); + } + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch({ ...options, + ...(executionPrepared ? { processLauncher: undefined } : {}), + ...(externalArtifact && recovering ? { runnerBinaryPath: defaultCapabilityRunnerdBinary() } : {}), + }); + handles.push(handle); + return handle; + }); + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const stopOwnedProvider = async (pid: number) => { + // Exact runner completion can race the OS reaping its already-signalled + // provider. Absence, not the outcome of a redundant signal, is required. + try { + await vi.waitFor(() => expect(dead(-pid)).toBe(true), { + timeout: 500, + interval: 10, + }); + } catch { + try { + process.kill(-pid, "SIGKILL"); + } catch (error) { + if ( + !["ESRCH", "EPERM"].includes( + String((error as NodeJS.ErrnoException).code), + ) + ) + throw error; + } + await vi.waitFor(() => expect(dead(-pid)).toBe(true), { + timeout: 2_000, + interval: 10, + }); + } + expect(dead(pid)).toBe(true); + }; + const fingerprintTree = async (root: string): Promise => { + const rows: unknown[] = []; + const visit = async (path: string, relative: string) => { + const metadata = await lstat(path); + rows.push({ + path: relative, + inode: metadata.ino, + mode: metadata.mode, + mtimeMs: metadata.mtimeMs, + digest: metadata.isFile() + ? createHash("sha256") + .update(await readFile(path)) + .digest("hex") + : metadata.isSymbolicLink() + ? createHash("sha256") + .update(await readlink(path)) + .digest("hex") + : null, + }); + if (metadata.isDirectory()) + for (const child of (await readdir(path)).sort()) + await visit(join(path, child), `${relative}/${child}`); + }; + await visit(root, "."); + return rows; + }; + const within = async (promise: Promise, timeout = 5_000) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error("warm restart fixture timed out")), + timeout, + ); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + }; + const options = { + runnerBinary: + recoveryFault === "attach-capability" + ? process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER! + : (process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + defaultCapabilityRunnerdBinary()), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 2_000, + prpIdentity: (() => { + const runId = randomUUID(); + return { + runnerInstanceId: randomUUID(), + environmentLeaseId: + process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_SCOPE === + "transient" + ? runId + : randomUUID(), + runId, + normalizedSessionId: randomUUID(), + turnId: `turn-${runId}`, + itemId: `item-${runId}`, + }; + })(), + ...(routed + ? { + controlPlaneRegistration: registration, + warmTransitionRegistrationMode: "routed_connect" as const, + } + : {}), + ...(executionPrepared ? { + runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), + readRunnerState: async () => JSON.parse(await readFile(join(stateDirectory, "runner/runner-state.json"), "utf8")), + // The fixture's spawn spy supplies the real local process owner. + runnerProcessLauncher: () => { throw new Error("fixture process owner was bypassed"); }, + prepareLaunch: async (requirements: RunnerProviderLaunchRequirements, recovery?: PreparedRunnerProviderLaunch) => { + const existing = configurations.get(requirements.identity.runId); + if (existing) { + expect(requirements).toEqual(existing.requirements); + if (recovering) expect(recovery).toEqual(existing.prepared); + return existing.prepared; + } + const prepared = { configurationId: `prepared-${requirements.identity.runId}`, + requirementsDigest: runnerLaunchRequirementsDigest(requirements), + provider: createRunnerProviderLaunch(requirements, { + options: { codexArgs: fakeCodexArgs(stateDirectory, "--call-log", callsPath, "--record-process-start") }, + root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), + codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "", + }), + }; + configurations.set(requirements.identity.runId, { requirements: structuredClone(requirements), prepared }); + return prepared; + }, + } : {}), + }; + const first = createCapabilityRunnerdCodexTransport(options); + let resumed: + ReturnType | undefined; + const providerPids = new Set(); + let commitSpy: ReturnType | undefined; + let commandObserverSpy: ReturnType | undefined; + let detached: Promise | undefined; + let cleanupProven = false; + const legacyProbeDirectories: string[] = []; + try { + const opened = await within( + first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ); + const openedThread = opened.thread as { id: string; sessionId: string }; + const thread = { id: openedThread.id, sessionId: openedThread.sessionId }; + const firstThreadSnapshot = await within( + first.transport.request("thread/read", {}), + ); + const firstEvidence = first.evidence(); + const cleanRunnerStateBytes = await readFile( + join(stateDirectory, "runner/runner-state.json"), + ); + const runnerProcessStartedAt = + process.platform === "darwin" + ? new Date( + execFileSync( + "ps", + ["-o", "lstart=", "-p", String(handles[0]!.child.pid)], + { encoding: "utf8", timeout: 1_500 }, + ).trim(), + ).toISOString() + : null; + if ((first.evidence().codexPid ?? 0) > 0) + providerPids.add(first.evidence().codexPid!); + const core = cores[0]!; + const oldIdentity = structuredClone(core.store.state.identity); + const nextRunId = randomUUID(); + const desired = { + ...oldIdentity, + runId: nextRunId, + turnId: `turn-${nextRunId}`, + itemId: `item-${nextRunId}`, + }; + if ( + recoveryFault === "attach-capability" || + recoveryFault === "attach-wait" + ) { + if (recoveryFault === "attach-wait") { + const getCommand = core.getCommand.bind(core); + commandObserverSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((id) => { + if ( + core.store.state.commands.find( + (entry) => entry.commandId === id, + )?.type === "run.attach" + ) { + throw new Error("fixture result observer unavailable"); + } + return getCommand(id); + }); + } + const bootstrapCount = core.store.state.freshBootstraps; + await expect( + first.transport.attachRun!({ + runId: desired.runId, + turnId: desired.turnId, + itemId: desired.itemId, + }), + ).rejects.toThrow( + recoveryFault === "attach-capability" + ? "capability is required" + : "result observer unavailable", + ); + expect(routeCalls).toEqual([oldIdentity.runId, desired.runId]); + expect([...routes.keys()]).toEqual([ + `/api/runner/v1/connect/${oldIdentity.runId}`, + ]); + expect(core.store.state.freshBootstraps).toBe(bootstrapCount); + expect(handles).toHaveLength(1); + if (recoveryFault === "attach-capability") + expect( + core.store.state.commands.some( + (entry) => entry.type === "run.attach", + ), + ).toBe(false); + return; + } + const commit = core.store.commit.bind(core.store); + let lossObserved = false; + let signalLoss!: () => void; + const loss = new Promise((resolveLoss) => { + signalLoss = resolveLoss; + }); + commitSpy = vi + .spyOn(core.store, "commit") + .mockImplementation((candidate) => { + // The selected process-crash boundary stays unavailable until the + // exact owned runner is joined. A later replay must not silently + // settle this fixture before its retained pair is exported. + if (lossObserved) + throw new Error( + "fixture controller persistence unavailable after loss", + ); + const phase = candidate.warmTransition?.phase; + const target = + lossPoint === "after-activation" ? "activated" : "prepared"; + const atBoundary = lossPoint.includes("confirmation") + ? candidate.completedWarmTransition !== undefined && + candidate.warmTransition === undefined + : phase === target; + if (!lossObserved && atBoundary) { + lossObserved = true; + if ( + lossPoint !== "before-result" && + lossPoint !== "before-confirmation" + ) + commit(candidate); + // A dead controller cannot accept a reconnect through the second + // route installed for the in-flight authority rotation. Detaching + // only the current route leaves that overlapping fixture listener + // alive; completed-receipt replay needs no further store commit. + for (const [path, entry] of routes) { + if (entry.core === core) routes.delete(path); + } + core.disconnectActiveRunner(); + detached = first.transport.detachControllerForRestart!(); + signalLoss(); + // A committed confirmation is still before its ACK. Returning from + // this hook lets the current frame handler send that ACK even after + // close begins, collapsing the intended crash window on fast peers. + throw new Error("fixture interrupted exact result commit"); + } + commit(candidate); + }); + const attachment = first.transport.attachRun!({ + runId: desired.runId, + turnId: desired.turnId, + itemId: desired.itemId, + }); + void attachment.catch(() => undefined); + await within(loss); + await within(detached!); + expect(lossObserved).toBe(true); + const runnerPath = join(stateDirectory, "runner/runner-state.json"); + const runner = JSON.parse(await readFile(runnerPath, "utf8")); + expect(runner.warmTransition.phase).toBe( + lossPoint === "after-activation" || lossPoint.includes("confirmation") + ? "activating" + : "prepared", + ); + const persisted = JSON.parse(await readFile(core.store.path, "utf8")); + expect( + persisted.commands.find( + (entry: { type: string }) => entry.type === "run.attach", + )?.status, + ).toBe( + lossPoint === "before-result" + ? "pending" + : lossPoint === "after-result" + ? "completed" + : undefined, + ); + // Stop only this fixture's exact owned process handles. No stored receipt + // or PID absence is used as authority to terminate an unknown owner. + await durableControlPlane + .waitForProcess(handles[0]!, 100) + .catch(() => undefined); + await within(handles[0]!.completion); + for (const pid of providerPids) { + await stopOwnedProvider(pid); + } + await within(attachment.catch(() => undefined)); + const joinedRunner = JSON.parse(await readFile(runnerPath, "utf8")); + const joinedCore = JSON.parse(await readFile(core.store.path, "utf8")); + expect(joinedRunner.warmTransition).toEqual(runner.warmTransition); + expect(joinedCore.schema).toBe(persisted.schema); + expect(joinedCore.warmTransition).toEqual(persisted.warmTransition); + expect(joinedCore.completedWarmTransition).toEqual( + persisted.completedWarmTransition, + ); + commitSpy.mockRestore(); + const fixtureOutput = + process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_DIRECTORY; + if (fixtureOutput && recoveryFault === "none") { + const retainedArtifact = join(fixtureOutput, "paperclip-runnerd"); + await cp(options.runnerBinary, retainedArtifact, { force: false }); + expect( + createHash("sha256") + .update(await readFile(retainedArtifact)) + .digest("hex"), + ).toBe( + createHash("sha256") + .update(await readFile(options.runnerBinary)) + .digest("hex"), + ); + const retained = join( + fixtureOutput, + `${lossPoint}-${routed ? "routed" : "local"}`, + ); + await cp(stateDirectory, retained, { + recursive: true, + errorOnExist: true, + force: false, + }); + await writeFile( + join(retained, "transition-fixture-metadata.json"), + JSON.stringify( + { + schema: "paperclip.test.warm-transition-fixture.v1", + oldIdentity, + newIdentity: desired, + lossPoint, + routed, + runner: { + pid: handles[0]!.child.pid, + processGroupId: handles[0]!.processGroupId, + startedAt: runnerProcessStartedAt, + spawnObservedAt: handles[0]!.startedAt, + completion: await handles[0]!.completion, + processAbsent: dead(handles[0]!.child.pid!), + groupAbsent: dead(-handles[0]!.processGroupId!), + }, + providers: [...providerPids].map((pid) => ({ + pid, + processGroupId: pid, + startedAt: firstEvidence.providerProcessStartedAt, + processAbsent: dead(pid), + groupAbsent: dead(-pid), + })), + artifact: { + path: retainedArtifact, + version: runner.warmTransition.receipt.runnerVersion, + digest: runner.warmTransition.receipt.runnerDigest, + }, + thread, + firstThreadSnapshot, + firstEvidence, + }, + null, + 2, + ), + { mode: 0o600 }, + ); + } + routes.clear(); + const routeCallCount = routeCalls.length; + const beforeCalls = (await readFile(callsPath, "utf8")) + .trim() + .split(/\r?\n/); + expect( + beforeCalls.filter((call) => call === "process-start"), + ).toHaveLength(1); + if (recoveryFault === "legacy-parser") { + const legacyBinary = + process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER!; + const legacyDigest = `sha256:${createHash("sha256") + .update(await readFile(legacyBinary)) + .digest("hex")}`; + for (const [mode, bytes] of [ + ["pending", await readFile(runnerPath)], + ["clean", cleanRunnerStateBytes], + ] as const) { + const probe = await mkdtemp( + join(tmpdir(), "runnerd-legacy-schema-probe-"), + ); + legacyProbeDirectories.push(probe); + await writeFile(join(probe, "runner-state.json"), bytes, { + mode: 0o600, + }); + const handle = durableControlPlane.spawnRunner({ + connection: runner.warmTransition.receipt.connection, + stateDirectory: probe, + identity: oldIdentity, + runnerBinaryPath: legacyBinary, + runnerVersion: "0.3.0", + runnerDigest: legacyDigest, + ticket: "bootstrap_legacy_parser_probe", + maxOutboxBytes: runner.maxOutboxBytes, + p0ReserveBytes: runner.p0ReserveBytes, + maxRuntimeMs: 200, + reconnectGraceMs: 200, + }); + const result = await within(handle.completion, 5_000); + expect(result.code).not.toBe(0); + if (mode === "pending") { + expect(result.stderr).toContain( + "durable state binding does not match", + ); + expect(await readFile(join(probe, "runner-state.json"))).toEqual( + bytes, + ); + } else { + expect(result.stderr).not.toContain( + "durable state binding does not match", + ); + expect( + JSON.parse( + await readFile(join(probe, "runner-state.json"), "utf8"), + ).nextSourceSeq, + ).toBeGreaterThan(JSON.parse(bytes.toString("utf8")).nextSourceSeq); + } + } + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + return; + } + recovering = true; + if (recoveryFault === "malformed-core") + await writeFile(core.store.path, "{", { mode: 0o600 }); + const beforeRecoveryBytes = await Promise.all( + [core.store.path, runnerPath].map((path) => readFile(path)), + ); + const beforeRecoveryTree = + recoveryFault === "none" ? null : await fingerprintTree(stateDirectory); + const authorizationStages: string[] = []; + let releaseHeldAuthorization: (() => void) | undefined; + let markHeldAuthorizationEntered!: () => void; + const heldAuthorizationEntered = new Promise((resolve) => { + markHeldAuthorizationEntered = resolve; + }); + const authorizationFault = recoveryFault.startsWith("authorize_"); + const failedAuthorizationStage = recoveryFault.slice("authorize_".length); + const beforeProviderBytes = await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + ); + resumed = createCapabilityRunnerdCodexTransport({ + ...options, + ...(externalArtifact ? { + runnerBinary: "/controller-missing-runner", + runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), + } : {}), + ...(ordinaryFollowup + ? { + authorizeWarmTransitionRecovery: async () => { + if (!recoveryClaimCurrent) + throw new Error( + "fixture old recovery claim is no longer current", + ); + }, + onWarmTransitionRecoveryCompleted: (completion: { + transitionId: string; + }) => { + expect(completion.transitionId).toBe( + runner.warmTransition.receipt.transitionId, + ); + const completedSnapshot = cores[1]!.store.state.commands + .filter((command) => command.type === "session.snapshot") + .at(-1)!; + expect(completedSnapshot.status).toBe("completed"); + expect(cores[1]!.store.state.identity).toEqual(desired); + expect(cores[1]!.store.state.warmTransition).toBeUndefined(); + completionSnapshotIds.push(completedSnapshot.commandId); + if ( + ordinaryFollowup === "callback_failure" && + completionSnapshotIds.length === 1 + ) { + throw new Error( + "fixture recovery completion callback failed", + ); + } + if ( + ordinaryFollowup === "callback_async_failure" && + completionSnapshotIds.length === 1 + ) { + return new Promise((_resolve, reject) => { + rejectHeldCompletion = reject; + markHeldCompletionEntered(); + }); + } + recoveryFenceActive = false; + }, + } + : {}), + ...(authorizationFault + ? { + authorizeWarmTransitionRecovery: async (stage: string) => { + authorizationStages.push(stage); + if ( + stage === "before_bootstrap" && + failedAuthorizationStage === "before_bootstrap" + ) { + await new Promise((resolveGate) => { + releaseHeldAuthorization = resolveGate; + markHeldAuthorizationEntered(); + }); + } + if (stage === failedAuthorizationStage) + throw new Error("fixture recovery authority revoked"); + }, + } + : {}), + ...(recoveryFault === "missing-capability" + ? { warmTransitionRegistrationMode: undefined } + : {}), + ...(recoveryFault !== "none" + ? { + environment: { + CODEX_API_KEY: "synthetic-refused-route-credential", + }, + } + : {}), + prpIdentity: desired, + resumeProviderSession: { + driverSessionId: thread.id, + providerSessionId: thread.sessionId, + }, + }); + if (recoveryFault !== "none") { + const recoveryRequest = within( + resumed.transport.request("thread/read", {}), + ); + void recoveryRequest.catch(() => undefined); + if ( + authorizationFault && + failedAuthorizationStage === "before_bootstrap" + ) { + await within(Promise.race([ + heldAuthorizationEntered, + recoveryRequest.then(() => { + throw new Error("warm restart authorization callback was not entered"); + }), + ])); + expect(releaseHeldAuthorization).toBeTypeOf("function"); + const queuedBefore = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + try { + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must not queue before bootstrap" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.attachRun!({ + runId: "must-not-attach", + turnId: "must-not-attach", + itemId: "must-not-attach", + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.resolveRuntimeRequest!({ + requestId: "must-not-resolve", + turnId: desired.turnId, + resolution: { action: "cancel" }, + }), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map( + (command) => command.commandId, + ), + ).toEqual(queuedBefore); + expect(handles).toHaveLength(1); + } finally { + releaseHeldAuthorization!(); + } + } + const failedRecovery = expect(recoveryRequest).rejects; + if (authorizationFault) { + await failedRecovery.toThrow( + failedAuthorizationStage === "before_authentication" + ? "native_runner_warm_transition_recovery_pending" + : "fixture recovery authority revoked", + ); + const expectedStages = [ + "before_bootstrap", + "before_spawn", + "before_authentication", + ]; + expect([...new Set(authorizationStages)]).toEqual( + expectedStages.slice( + 0, + expectedStages.indexOf(failedAuthorizationStage) + 1, + ), + ); + expect(handles).toHaveLength( + failedAuthorizationStage === "before_authentication" ? 2 : 1, + ); + expect( + await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + ), + ).toEqual(beforeProviderBytes); + const refusedRunner = JSON.parse(await readFile(runnerPath, "utf8")); + expect(refusedRunner.warmTransition).toEqual(runner.warmTransition); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + if (failedAuthorizationStage !== "before_authentication") + expect(await readFile(runnerPath)).toEqual(beforeRecoveryBytes[1]); + if (failedAuthorizationStage === "before_bootstrap") + expect(await readFile(core.store.path)).toEqual( + beforeRecoveryBytes[0], + ); + await within(resumed.transport.close()).catch(() => undefined); + expect([...routes.keys()]).toEqual([]); + return; + } + if (recoveryFault === "malformed-core") await failedRecovery.toThrow(); + else + await failedRecovery.toThrow( + recoveryFault === "missing-capability" + ? "requires_exact_owned_endpoint" + : "registered_endpoint_mismatch", + ); + expect(handles).toHaveLength(1); + expect( + await Promise.all( + [core.store.path, runnerPath].map((path) => readFile(path)), + ), + ).toEqual(beforeRecoveryBytes); + expect(await fingerprintTree(stateDirectory)).toEqual( + beforeRecoveryTree, + ); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + expect([...routes.keys()]).toEqual([]); + return; + } + if ( + snapshotFault || + ordinaryFollowup === "callback_failure" || + ordinaryFollowup === "callback_async_failure" + ) { + const firstRead = within(resumed.transport.request("thread/read", {})); + void firstRead.catch(() => undefined); + if (ordinaryFollowup === "callback_async_failure") { + await within(Promise.race([ + heldCompletionEntered, + firstRead.then(() => { + throw new Error("warm restart completion callback was not entered"); + }), + ])); + expect(rejectHeldCompletion).toBeTypeOf("function"); + try { + expect(recoveryFenceActive).toBe(true); + const queuedBefore = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must not pass held completion" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.request("thread/resume", {}), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map( + (command) => command.commandId, + ), + ).toEqual(queuedBefore); + } finally { + rejectHeldCompletion!( + new Error("fixture recovery completion callback failed"), + ); + } + } + await expect(firstRead).rejects.toThrow( + snapshotFault + ? ordinaryFollowup === "rejected_snapshot" + ? "snapshot observation rejected" + : "completion_unproven" + : "recovery completion callback failed", + ); + expect(recoveryFenceActive).toBe(true); + expect(completionSnapshotIds).toHaveLength(snapshotFault ? 0 : 1); + const beforeDeniedWork = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + const beforeDeniedProviderCalls = await readFile(callsPath, "utf8"); + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must remain fenced" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.request("thread/resume", {}), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.attachRun!({ + runId: "must-not-attach", + turnId: "must-not-attach", + itemId: "must-not-attach", + }), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map((command) => command.commandId), + ).toEqual(beforeDeniedWork); + expect(await readFile(callsPath, "utf8")).toBe( + beforeDeniedProviderCalls, + ); + snapshotObserverSpy?.mockRestore(); + const providerCallsBeforeRetry = await readFile(callsPath, "utf8"); + expect( + (await within(resumed.transport.request("thread/read", {}))).thread, + ).toMatchObject(thread); + expect(recoveryFenceActive).toBe(false); + expect(completionSnapshotIds).toHaveLength(snapshotFault ? 1 : 2); + expect(new Set(completionSnapshotIds).size).toBe( + completionSnapshotIds.length, + ); + expect(await readFile(callsPath, "utf8")).toBe( + providerCallsBeforeRetry, + ); + return; + } + const read = await within( + resumed.transport.request("thread/read", {}), + 10_000, + ); + if ((resumed.evidence().codexPid ?? 0) > 0) + providerPids.add(resumed.evidence().codexPid!); + expect(read.thread).toMatchObject(thread); + expect(cores).toHaveLength(2); + expect(cores[1]!.store.state.identity).toEqual(desired); + expect(cores[1]!.store.state.warmTransition).toBeUndefined(); + expect( + cores[1]!.store.state.completedWarmTransition?.receipt.transitionId, + ).toBe(runner.warmTransition.receipt.transitionId); + if (routed) { + expect(routeCalls.slice(routeCallCount)).toEqual([ + oldIdentity.runId, + desired.runId, + ]); + expect([...routes.keys()]).toEqual([ + `/api/runner/v1/connect/${desired.runId}`, + ]); + } + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(2); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/resume")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + if (ordinaryFollowup) { + recoveryClaimCurrent = false; + const resumedCore = cores[1]!; + resumedCore.disconnectActiveRunner(); + await vi.waitFor( + () => expect(resumedCore.activeRunnerConnectionCount()).toBe(1), + { timeout: 2_000 }, + ); + expect(recoveryFenceActive).toBe(false); + expect( + (await within(resumed.transport.request("thread/read", {}))).thread, + ).toMatchObject(thread); + const thirdRunId = randomUUID(); + await within( + resumed.transport.attachRun!({ + runId: thirdRunId, + turnId: `turn-${thirdRunId}`, + itemId: `item-${thirdRunId}`, + }), + ); + expect(resumedCore.store.state.identity.runId).toBe(thirdRunId); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual([...calls, "thread/goal/get"]); + } + } finally { + snapshotObserverSpy?.mockRestore(); + commitSpy?.mockRestore(); + commandObserverSpy?.mockRestore(); + try { + await within(resumed?.transport.close() ?? Promise.resolve()).catch( + () => undefined, + ); + await within(first.transport.detachControllerForRestart!()).catch( + () => undefined, + ); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within(handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + for (const pid of providerPids) { + await stopOwnedProvider(pid); + } + cleanupProven = true; + } finally { + for (const core of cores) await core.stop().catch(() => undefined); + if (routed) + await new Promise((resolveClose) => + routeServer.close(() => resolveClose()), + ); + coreSpy.mockRestore(); + launchSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + if (cleanupProven) + for (const directory of legacyProbeDirectories) + await rm(directory, { recursive: true, force: true }); + } + } + }, + 30_000, +); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts new file mode 100644 index 00000000000..dba8ec470b3 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts @@ -0,0 +1,769 @@ +import { mkdtemp, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { ControlPlanePort } from "../contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "../contracts/native-execution.js"; +import type { NativeSession, NativeSessionBackend } from "../contracts/native-session-backend.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; +import { executeNativeSession } from "../native-session-runtime.js"; +import { createCodexTaskEnvelope } from "../contracts/codex.js"; +import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver.js"; +import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, expectTurnStarted, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +it("rejects active work and buffered tools from a resumed stopped checkpoint", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-stopped-resume-active-"), + ); + const identity = { + runnerInstanceId: "runner-stopped-active", + environmentLeaseId: "lease-stopped-active", + runId: "run-stopped-first", + normalizedSessionId: "session-stopped-active", + turnId: "turn-stopped-first", + itemId: "item-stopped-first", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--linger-after-turn-start", + "--resume-unowned-turn-when-marked", + "--emit-tool-call-on-resume", + ), + stateDirectory, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + }; + const first = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: identity, + }); + let second: + ReturnType | undefined; + const semanticHandler = vi.fn(async () => ({ + success: true, + contentItems: [], + })); + first.transport.setServerRequestHandler(semanticHandler); + try { + const opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the current task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ], + }); + await first.transport.request("turn/start", { + input: [{ type: "text", text: "Wait for another instruction." }], + }); + await expectTurnStarted( + first.transport.notifications()[Symbol.asyncIterator](), + ); + await first.transport.close(); + const providerPath = join( + stateDirectory, + "runner", + "codex-provider-state.json", + ); + expect(JSON.parse(await readFile(providerPath, "utf8"))).toMatchObject({ + lifecycle: "prepared", + activeProviderTurnId: null, + }); + await writeFile(join(stateDirectory, "resume-unowned-turn"), "armed"); + const secondIdentity = { + ...identity, + runId: "run-stopped-second", + turnId: "turn-stopped-second", + itemId: "item-stopped-second", + }; + second = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: secondIdentity, + }); + second.transport.setServerRequestHandler(semanticHandler); + await expect(second.transport.request("thread/read", {})).rejects.toThrow( + "prepared provider checkpoint resumed unexpected active work", + ); + // Exercise the normal durable transfer before inspecting the rejection: + // provider pendingEvents is an acknowledged queue, not an event journal. + await vi.waitFor( + async () => { + const provider = JSON.parse(await readFile(providerPath, "utf8")); + expect(provider.pendingEvents).toEqual([]); + const control = JSON.parse( + await readFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + "utf8", + ), + ) as { + committedEvents: Array<{ + eventType: string; + envelope: { payload: { payload: { code?: string } } }; + }>; + }; + const rejections = control.committedEvents.filter( + (event) => + event.eventType === "harness.diagnostic" && + event.envelope.payload.payload.code === + "prepared_provider_checkpoint_has_active_work", + ); + expect(rejections).toEqual([ + expect.objectContaining({ + logicalEffectCount: 1, + envelope: expect.objectContaining({ + ...secondIdentity, + payload: expect.objectContaining({ + payload: expect.objectContaining({ + code: "prepared_provider_checkpoint_has_active_work", + paperclipAccepted: false, + providerReportedActive: true, + }), + }), + }), + }), + ]); + }, + { timeout: 5_000 }, + ); + const closed = JSON.parse(await readFile(providerPath, "utf8")); + expect(closed).toMatchObject({ + lifecycle: "closed", + threadId: (opened.thread as Record).id, + activeProviderTurnId: null, + }); + expect(semanticHandler).not.toHaveBeenCalled(); + expect(closed.pendingEvents).toEqual([]); + } finally { + await Promise.allSettled([ + first.transport.close(), + second?.transport.close(), + ]); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => { + const identity = { + runId: "run-recovery", + sessionId: "session-recovery", + companyId: "company-recovery", + issueId: "issue-recovery", + agentId: "agent-recovery", + }; + + const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Recovered native work completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [ + { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, + ], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "recovery", status: "passed" }], + attentionRequests: [], + artifacts: [], + }; + + const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }; + + const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: identity.companyId, + runId: identity.runId, + issueId: identity.issueId, + agentId: identity.agentId, + executionWorkspaceId: "workspace-recovery", + }, + task: { + identifier: "PAP-RECOVERY", + title: "Recover native work", + description: null, + prompt: "# PAP-RECOVERY: Recover native work", + workMode: "standard", + }, + workspace: { + cwd: "/workspace", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: identity.sessionId, + driverKind: "codex_app_server", + protocolVersion: 1, + }, + provider: { kind: "codex", model: null }, + completionContract: { + id: "contract-recovery", + sha256: "contract-recovery-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Recover native work", + criteria: [{ id: "objective", requirement: "Complete after recovery" }], + }, + }, + interactionResponses: [], + credentialBindings: [], + }; + + function runnerEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record = {}, + ): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-recovery", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; + } + const stateDirectory = await mkdtemp( + join(tmpdir(), "native-close-quarantine-"), + ); + const readRunnerState = vi.fn(async () => ({ + schema: "paperclip.runner.durable.state.v1", + runnerInstanceId: "runner-close-quarantine", + environmentLeaseId: "lease-close-quarantine", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-close-quarantine", + itemId: "item-close-quarantine", + lifecycle: "ready", + })); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", + ), + codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], + stateDirectory, + closeGraceMs: 400, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: await readRunnerState(), + readRunnerState, + // A checkpoint owner requires durable suspension proof. A local transport + // without a checkpoint can simply terminate its process on close. + controlPlaneRegistration: async (authority) => { + await authority.start(); + return { + connectUrl: authority.connectUrl, + checkpoint: async () => {}, + release: async () => {}, + }; + }, + }); + try { + await bundle.transport.request("thread/start", { + cwd: stateDirectory, + dynamicTools: [], + }); + const failedClose = bundle.transport.close(); + const failure = await failedClose.catch((error: unknown) => error); + expect(failure).toMatchObject({ + code: "native_session_close_unrecoverable", + }); + expect(bundle.transport.close()).toBe(failedClose); + + vi.useFakeTimers(); + const close = vi.fn(({ reason }: { reason: string }) => + bundle.transport.close(reason), + ); + const capabilities = { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return capabilities; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "real-memoized-close-quarantine", + version: "1", + capabilities, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + requireSessionCloseBeforeReturn: true, + }); + await expect(execute()).rejects.toBe(failure); + const readsAfterClose = readRunnerState.mock.calls.length; + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + recovery: "operator_required", + }); + await vi.advanceTimersByTimeAsync(600_000); + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + }); + expect(openSession).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose); + } finally { + vi.useRealTimers(); + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 10_000); + +it.each(["not_suspended", "wrong_identity"] as const)( + "does not report local runner close healthy with %s durable evidence", + async (mode) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-local-close-unproven-"), + ); + const identity = { + runnerInstanceId: "runner-close-unproven", + environmentLeaseId: "lease-close-unproven", + runId: "run-close-unproven", + normalizedSessionId: "session-close-unproven", + turnId: "turn-close-unproven", + itemId: "item-close-unproven", + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + closeGraceMs: 400, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + readRunnerState: async () => ({ + schema: "paperclip.runner.durable.state.v1", + ...identity, + ...(mode === "wrong_identity" ? { runId: "some-other-run" } : {}), + lifecycle: mode === "not_suspended" ? "ready" : "suspended", + }), + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + await expect(bundle.transport.close()).rejects.toThrow( + "runner did not durably suspend before checkpoint", + ); + const control = JSON.parse( + await readFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + "utf8", + ), + ); + expect(control.identity).toEqual(identity); + expect(await readdir(stateDirectory)).toContain("runner"); + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } + }, + 10_000, +); + +it.each([ + { + label: "interrupted", + args: ["--hold-turn"], + terminal: "turn.interrupted", + disposition: "needs_review", + }, + { + label: "failed", + args: ["--fail-turn-immediately"], + terminal: "turn.failed", + disposition: "needs_review", + }, + { + label: "completed", + args: [], + terminal: "turn.completed", + disposition: "done", + }, +])("retains a $label runner's result before closing its provider turn", async ({ + label, args, terminal, disposition, +}) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-stop-result-")); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, ...args), + stateDirectory, + }); + const driver = new CodexAppServerDriver({ + taskEnvelope: createCodexTaskEnvelope({ + objective: "Stop and retain unfinished work.", + }), + environment: { + PATH: process.env.PATH, + HOME: join(tmpdir(), "runnerd-stop-host-home"), + PAPERCLIP_WORKSPACE_CWD: stateDirectory, + }, + approvalPolicy: "never", + transportFactory: () => bundle.transport, + }); + const session = await driver.openSession({ + runId: "run-stop-result", + normalizedSessionId: "session-stop-result", + workingDirectory: stateDirectory, + }); + try { + const turn = await session.startTurn({ + message: { role: "user", text: "Keep working until interrupted." }, + }); + if (label === "interrupted") { + await session.interrupt({ + turnId: turn.turnId, + reason: "Stopped by the user", + }); + } + const events: PrpEvent[] = []; + for await (const event of session.events()) { + events.push(event); + if (event.eventType === "session.failed" || event.eventType === terminal) { + break; + } + } + // Let already committed durable suffix events reach the facade as well. + await bundle.transport.request("thread/read", {}); + const snapshot = await session.snapshot(); + expect(events.some((event) => event.eventType === "session.failed")).toBe(false); + expect(events.map((event) => event.eventType)).toContain("run.result.proposed"); + expect(events.at(-1)?.eventType).toBe(terminal); + expect(snapshot).toMatchObject({ + activeTurnId: null, + semanticResult: { + turnId: turn.turnId, + result: { reportedWorkDisposition: disposition }, + }, + }); + } finally { + await session.close().catch(() => undefined); + await rm(stateDirectory, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 25, + }); + } +}, 30_000); + +it.each([0, 193 * 1024 * 1024])( + "probes an exact-authority resume with %i extra journal bytes and confirms its live provider identity", + async (extraJournalBytes) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-exact-authority-resume-"), + ); + const identity = { + runnerInstanceId: "runner-exact-resume", + environmentLeaseId: "lease-exact-resume", + runId: "run-exact-resume", + normalizedSessionId: "session-exact-resume", + turnId: "turn-exact-resume", + itemId: "item-exact-resume", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--durable-turn-ids"), + stateDirectory, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + prpIdentity: identity, + }; + const first = createCapabilityRunnerdCodexTransport(options); + first.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let providerThread: { id: string; sessionId: string } | null = null; + try { + const opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + const thread = opened.thread as Record; + providerThread = { + id: String(thread.id), + sessionId: String(thread.sessionId), + }; + } finally { + await first.transport.close(); + } + if (providerThread === null) { + throw new Error( + "exact-authority fixture did not return a provider thread", + ); + } + + const statePath = join( + stateDirectory, + "control-plane", + "control-plane-state.json", + ); + if (extraJournalBytes > 0) { + const padding = Buffer.alloc(1024 * 1024, 0x20); + const stateHandle = await open(statePath, "a"); + try { + for (let remaining = extraJournalBytes; remaining > 0;) { + const bytesToWrite = Math.min(remaining, padding.length); + await stateHandle.write(padding, 0, bytesToWrite); + remaining -= bytesToWrite; + } + } finally { + await stateHandle.close(); + } + expect((await stat(statePath)).size).toBeGreaterThan( + 64 * 1024 * 1024, + ); + } + const beforeResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ type: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect( + beforeResume.commands.some((command) => command.type === "run.attach"), + ).toBe(false); + const priorResumeEvents = beforeResume.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ).length; + const priorSnapshots = beforeResume.commands.filter( + (command) => command.type === "session.snapshot", + ).length; + + const resumed = createCapabilityRunnerdCodexTransport({ + ...options, + resumeProviderSession: { + driverSessionId: providerThread.id, + providerSessionId: providerThread.sessionId, + }, + }); + resumed.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await resumed.transport.request("thread/read", {}); + expect(read.thread).toMatchObject(providerThread); + const afterResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ commandId: string; type: string; status: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + commandId: expect.stringMatching(/^command_resume_probe_/), + type: "runner.drain", + status: "completed", + }), + ); + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + type: "session.snapshot", + status: "completed", + }), + ); + expect( + afterResume.commands.filter( + (command) => command.type === "session.snapshot", + ), + ).toHaveLength(priorSnapshots + 2); + // The authenticated snapshot above proves the live provider identity. + // Control-first dispatch may deliver that command before the independent + // session event is ingested. Still require exactly one durable event; + // don't mistake an immediate file read for an event-delivery barrier. + await vi.waitFor( + async () => { + const delivered = JSON.parse(await readFile(statePath, "utf8")) as { + committedEvents: Array<{ eventType: string }>; + }; + expect( + delivered.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ), + ).toHaveLength(priorResumeEvents + 1); + }, + { timeout: 3_000, interval: 25 }, + ); + } finally { + await resumed.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } + }, + 30_000, +); + +it("still fails closed when a real close grace period cannot fit a durable suspension round trip", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-close-grace-too-small-"), + ); + const identity = { + runnerInstanceId: "runner-close-grace-too-small", + environmentLeaseId: "lease-close-grace-too-small", + runId: "run-close-grace-too-small", + normalizedSessionId: "session-close-grace-too-small", + turnId: "turn-close-grace-too-small", + itemId: "item-close-grace-too-small", + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + // No real durable command round trip can complete this fast. A wider + // budget for the provider-drain proof must not turn this barrier into + // one that always passes; it still needs the actual proof to arrive. + closeGraceMs: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + const closed = bundle.transport.close(); + await expect(closed).rejects.toMatchObject({ + code: "native_session_close_unrecoverable", + semanticToolsSettled: true, + }); + // Retirement preserves the original rejected promise and its proof. + expect(bundle.transport.close()).toBe(closed); + } finally { + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("persists an active provider as settled before bounded suspension", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-active-suspension-"), + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--linger-after-turn-start"), + stateDirectory, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("initialize", {}); + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + await bundle.transport.request("turn/start", { + input: [{ type: "text", text: "Wait for another instruction." }], + }); + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + await expectTurnStarted(notifications); + await bundle.transport.close(); + + const providerState = JSON.parse( + await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + "utf8", + ), + ) as Record; + expect(providerState).toMatchObject({ + lifecycle: "prepared", + activeProviderTurnId: null, + }); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts new file mode 100644 index 00000000000..7739d4e2851 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts @@ -0,0 +1,1649 @@ +import { mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { PrpEvent } from "../protocol/replay-contract.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { + defaultCapabilityRunnerdBinary, + fakeCodex, + fakeCodexArgs, + assignedRuntimeContext, +} from "../../test/support/runnerd-transport.js"; + +it.each([ + ["held-ack", "normal"], + ["lost-ack", "normal"], + ["rejected-attach", "normal"], + ["lost-ack", "after-activation"], +] as const)( + "preserves old warm-attach authority and event ownership across %s (%s observer)", + async (mode, observer) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-ack-")); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const effects = new Map(); + const handles: ReturnType[] = []; + let armed = false; + let heldEvent: PrpEvent | null = null; + let releaseCommit!: () => void; + let enteredCommit!: () => void; + const commitGate = new Promise((resolveCommit) => { + releaseCommit = resolveCommit; + }); + const commitEntered = new Promise((resolveEntered) => { + enteredCommit = resolveEntered; + }); + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore({ + ...options, + onCommittedEvent: async (event) => { + await options.onCommittedEvent?.(event); + const prior = effects.get(event.sourceEventId); + if (prior) { + expect(event).toEqual(prior.event); + prior.deliveries += 1; + } else { + effects.set(event.sourceEventId, { + event: structuredClone(event), + deliveries: 1, + }); + } + if ( + armed && + mode !== "rejected-attach" && + heldEvent === null && + event.eventType === "run.attached" + ) { + heldEvent = structuredClone(event); + enteredCommit(); + await commitGate; + if (mode === "lost-ack") { + // The external durable effect exists, but this connection + // disappears before its local cursor/ACK can be published. + throw new Error( + "fixture lost the old authority ACK after commit", + ); + } + } + }, + }); + cores.push(core); + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch(options); + handles.push(handle); + return handle; + }); + const within = async ( + label: string, + promise: Promise, + timeout = 5_000, + ) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolveWait, rejectWait) => { + timer = setTimeout( + () => rejectWait(new Error(`${label} timeout`)), + timeout, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + }; + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, + }); + const readRunner = async () => + JSON.parse( + await readFile( + join(stateDirectory, "runner/runner-state.json"), + "utf8", + ), + ) as { + runId: string; + ackedSourceSeq: number; + processedCommands: Record; + outbox: { envelope: { payload: PrpEvent } }[]; + }; + let providerPid: number | null = null; + let primaryError: unknown; + let cleanupProven = false; + try { + const opened = await within( + "initial thread", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ) as { thread: { id: string } }; + const core = cores[0]!; + expect(cores).toHaveLength(1); + const oldIdentity = structuredClone(core.store.state.identity); + const runnerPid = bundle.evidence().runnerPid; + providerPid = bundle.evidence().codexPid; + const rotations: (typeof core.store.state)[] = []; + const preparedStates: (typeof core.store.state)[] = []; + const commit = core.store.commit.bind(core.store); + vi.spyOn(core.store, "commit").mockImplementation((candidate) => { + commit(candidate); + // Capture the durable old epoch at publication. The authenticated + // successor may activate before attachRun observes the completed command. + if ( + candidate.identity.runId === oldIdentity.runId && + candidate.warmTransition?.phase === "prepared" + ) { + preparedStates.push(structuredClone(core.store.state)); + } + }); + if (observer === "after-activation") { + const getCommand = core.getCommand.bind(core); + vi.spyOn(core, "getCommand").mockImplementation((commandId) => { + const command = getCommand(commandId); + if ( + command?.type === "run.attach" && + command.status === "completed" && + core.store.state.completedWarmTransition === undefined + ) { + // Deterministically exercise an observer that misses the old epoch. + return { ...command, status: "pending", result: null }; + } + return command; + }); + } + const rotate = core.rotateRunIdentity.bind(core); + vi.spyOn(core, "rotateRunIdentity").mockImplementation( + (identity, template) => { + rotations.push(structuredClone(core.store.state)); + return rotate(identity, template); + }, + ); + if (mode === "rejected-attach") { + const queue = core.queueCommand.bind(core); + vi.spyOn(core, "queueCommand").mockImplementation( + (type, payload = {}, id, immediate) => + queue( + type, + type === "run.attach" + ? { + ...payload, + provider: { + ...(payload.provider as Record), + model: "foreign-profile", + }, + } + : payload, + id, + immediate, + ), + ); + } + armed = true; + const attachment = bundle.transport.attachRun!({ + runId: "run-warm-ack-next", + turnId: "turn-warm-ack-next", + itemId: "item-warm-ack-next", + }); + void attachment.catch(() => undefined); + if (mode === "rejected-attach") { + await expect(within("rejected attach", attachment)).rejects.toThrow( + "run.attach cannot change the durable Codex provider profile", + ); + expect(rotations).toHaveLength(0); + expect(core.store.state.identity).toEqual(oldIdentity); + expect((await readRunner()).runId).toBe(oldIdentity.runId); + const read = await within( + "read under unchanged authority", + bundle.transport.request("thread/read", {}), + ); + expect((read.thread as { id: string }).id).toBe(opened.thread.id); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("failed"); + } else { + await within("old authority commit barrier", commitEntered); + const retained = await readRunner(); + expect(retained.runId).toBe(oldIdentity.runId); + expect( + Object.values(retained.processedCommands).find((entry) => entry.commandType === "run.attach") + ?.status, + ).toBe("completed"); + expect( + retained.outbox.some( + (entry) => + entry.envelope.payload.sourceEventId === heldEvent!.sourceEventId, + ), + ).toBe(true); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("pending"); + expect(rotations).toHaveLength(0); + if (mode === "lost-ack") core.disconnectActiveRunner(); + releaseCommit(); + await within("warm attach after old ACK", attachment, 10_000); + expect(rotations).toHaveLength(1); + expect(preparedStates.length).toBeGreaterThan(0); + const retired = preparedStates[0]!; + if (observer === "after-activation") { + expect(rotations[0]!.identity.runId).toBe("run-warm-ack-next"); + expect( + rotations[0]!.committedEvents.some( + (entry) => entry.sourceEventId === heldEvent!.sourceEventId, + ), + ).toBe(false); + } + const attachedEvent = retired.committedEvents.find( + (entry) => entry.sourceEventId === heldEvent!.sourceEventId, + )!; + expect(attachedEvent.logicalEffectCount).toBe(1); + expect(retired.ackedSourceSeq).toBeGreaterThanOrEqual( + attachedEvent.sourceSeq, + ); + expect( + retired.committedEvents.slice(-4).map((entry) => entry.eventType), + ).toEqual([ + "session.resumed", + "session.capabilities.updated", + "session.goal.snapshot", + "run.attached", + ]); + expect( + retired.committedEvents.every( + (entry) => entry.envelope.runId === oldIdentity.runId, + ), + ).toBe(true); + if (mode === "lost-ack") { + expect(retired.connectionCount).toBeGreaterThanOrEqual(2); + expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(2); + } else { + expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(1); + } + await vi.waitFor(async () => + expect((await readRunner()).runId).toBe("run-warm-ack-next"), + ); + const read = await within( + "read under new authority", + bundle.transport.request("thread/read", {}), + ); + expect((read.thread as { id: string }).id).toBe(opened.thread.id); + } + expect(bundle.evidence()).toMatchObject({ + runnerPid, + codexPid: providerPid, + runnerExited: false, + }); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + } catch (error) { + primaryError = error; + throw error; + } finally { + releaseCommit(); + try { + await within( + "warm fixture close", + bundle.transport.close(), + 10_000, + ).catch(() => undefined); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within("exact warm fixture runner exit", handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + if (providerPid && !dead(-providerPid)) + process.kill(-providerPid, "SIGKILL"); + if (providerPid) + await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); + cleanupProven = true; + } catch (error) { + if (primaryError === undefined) throw error; + console.error( + "Warm ACK fixture cleanup unproven; retaining its private state directory.", + ); + } finally { + try { + for (const core of cores) await core.stop().catch(() => undefined); + } finally { + launchSpy.mockRestore(); + coreSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + } + } + } + }, + 30_000, +); + +it.each([false, true])( + "retains warm attach authority when its result is lost before controller persistence (held observer=%s)", + async (holdObserver) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-attach-result-loss-"), + ); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const handles: ReturnType[] = []; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore(options); + cores.push(core); + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch(options); + handles.push(handle); + return handle; + }); + const within = async ( + label: string, + promise: Promise, + timeout = 5_000, + ) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timeout`)), + timeout, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + }; + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: + process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 2_000, + }); + let providerPid: number | null = null; + let cleanupProven = false; + let saveSpy: ReturnType | undefined; + let observerSpy: { mockRestore(): void } | undefined; + let attachment: Promise | undefined; + try { + await within( + "initial thread", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ); + expect(cores).toHaveLength(1); + const core = cores[0]!; + const oldIdentity = structuredClone(core.store.state.identity); + if (holdObserver) { + const getCommand = core.getCommand.bind(core); + observerSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((commandId) => { + const command = getCommand(commandId); + if ( + command?.type === "run.attach" && + core.store.state.completedWarmTransition?.command.commandId !== + commandId + ) { + return { ...command, status: "pending", result: null }; + } + return command; + }); + } + providerPid = bundle.evidence().codexPid; + const store = core.store as typeof core.store & { + commit(candidate: typeof core.store.state): void; + }; + const commit = store.commit.bind(store); + let lostResult = false; + let observeLoss!: () => void; + const loss = new Promise((resolveLoss) => { + observeLoss = resolveLoss; + }); + saveSpy = vi.spyOn(store, "commit").mockImplementation((candidate) => { + const attach = candidate.commands.find( + (entry) => entry.type === "run.attach", + ); + if (!lostResult && attach?.status === "completed") { + lostResult = true; + // The authenticated result reached the receiver, but the durable write + // did not. The clone has not been exposed in memory or on disk. + const persisted = JSON.parse(readFileSync(core.store.path, "utf8")); + expect(persisted.identity).toEqual(oldIdentity); + expect( + persisted.commands.find( + (entry: { type: string }) => entry.type === "run.attach", + ).status, + ).toBe("pending"); + core.disconnectActiveRunner(); + observeLoss(); + throw new Error( + "fixture lost attach result before durable controller commit", + ); + } + commit(candidate); + }); + attachment = bundle.transport.attachRun!({ + runId: "run-result-loss-next", + turnId: "turn-result-loss-next", + itemId: "item-result-loss-next", + }); + void attachment.catch(() => undefined); + await within("lost attach result", loss); + expect(lostResult).toBe(true); + expect(core.store.state.identity).toEqual(oldIdentity); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("pending"); + const outcome = await within( + "exact result replay after reconnect", + attachment.then( + () => ({ status: "completed" as const }), + (error: unknown) => ({ + status: "failed" as const, + message: String(error), + }), + ), + 10_000, + ); + const runner = JSON.parse( + await readFile( + join(stateDirectory, "runner/runner-state.json"), + "utf8", + ), + ); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + expect( + { + outcome, + runnerRunId: runner.runId, + controllerRunId: core.store.state.identity.runId, + }, + "lost attach result must replay without leaving the two durable authorities split", + ).toEqual({ + outcome: { status: "completed" }, + runnerRunId: "run-result-loss-next", + controllerRunId: "run-result-loss-next", + }); + if (holdObserver) + expect( + core.store.state.completedWarmTransition?.receipt.newIdentity.runId, + ).toBe("run-result-loss-next"); + } finally { + saveSpy?.mockRestore(); + observerSpy?.mockRestore(); + try { + await within( + "result-loss fixture close", + bundle.transport.close(), + 5_000, + ).catch(() => undefined); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within("exact result-loss runner exit", handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + if (providerPid && !dead(-providerPid)) + process.kill(-providerPid, "SIGKILL"); + if (providerPid) + await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); + cleanupProven = true; + } finally { + for (const core of cores) await core.stop().catch(() => undefined); + launchSpy.mockRestore(); + coreSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + } + } + }, + 30_000, +); + +it("rotates PRP authority in place for a warm cross-run attachment", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-")); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + const within = async (label: string, promise: Promise) => + await Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${label} timeout`)), 5_000), + ), + ]); + try { + await within("initialize", bundle.transport.request("initialize", {})); + await within( + "thread start", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ], + completionContract: { + revision: "sha256:warm-three-turn-contract", + criterionIds: ["objective"], + }, + }), + ); + const runnerPid = bundle.evidence().runnerPid; + const providerPid = bundle.evidence().codexPid; + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + const waitForCompletion = async (label: string) => { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + const next = await Promise.race([ + notifications.next(), + new Promise((_, reject) => + setTimeout( + () => reject(new Error(`${label} notification timeout`)), + 1_000, + ), + ), + ]); + if (next.value?.method === "turn/completed") return; + } + throw new Error(`${label} completion timeout`); + }; + await within( + "first turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "first run" }], + }), + ); + await waitForCompletion("first run"); + + await within( + "warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-second", + turnId: "turn-warm-second", + itemId: "item-warm-second", + }), + ); + await within( + "second turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "second run" }], + }), + ); + await waitForCompletion("second run"); + + await within( + "second warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-third", + turnId: "turn-warm-third", + itemId: "item-warm-third", + }), + ); + await within( + "third turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "third run" }], + }), + ); + await waitForCompletion("third run"); + + expect(bundle.evidence()).toMatchObject({ + runnerPid, + codexPid: providerPid, + runnerExited: false, + }); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("recovers an active second warm business run after deleting its controller context assets", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-warm-second-active-")); + const firstRoot = join(root, "first-controller"), nextRoot = join(root, "next-controller"); + const assetRoot = join(root, "retired-controller-assets"); + const skillRoot = join(assetRoot, "skill"), instructionRoot = join(assetRoot, "instructions"); + await mkdir(skillRoot, { recursive: true }); await mkdir(instructionRoot, { recursive: true }); + await writeFile(join(skillRoot, "SKILL.md"), "# Pinned skill\n"); + await writeFile(join(instructionRoot, "AGENTS.md"), "Pinned instructions\n"); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null, generation = 0; + let checkpoint: durableControlPlane.DurablePrpCheckpoint | undefined; + const registered = new WeakSet(); + server.on("upgrade", (request, socket, head) => authority + ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); + await new Promise(done => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing owned Runner route"); + const registration = async (core: DurablePrpControlPlane) => { + if (!registered.has(core)) { + core.setCheckpointPersistence(async value => { checkpoint = value; }); + registered.add(core); + } + authority = core; const current = ++generation; + return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: async () => { if (generation === current) authority = null; } }; + }; + const initialIdentity = { runnerInstanceId: "runner-warm-second-active", environmentLeaseId: "lease-warm-second-active", + runId: "run-warm-first-healthy", normalizedSessionId: "session-warm-second-active", turnId: "turn-first", itemId: "item-first" }; + const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(root, "--emit-tool-call", "--durable-tool-ids"), stateDirectory: firstRoot, + runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), + prpIdentity: initialIdentity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration }; + const first = createCapabilityRunnerdCodexTransport(options); + const within = async (label: string, promise: Promise) => { + let timer: ReturnType | undefined; + try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`warm active recovery stage timed out: ${label}`)), 5_000); })]); } + finally { clearTimeout(timer); } + }; + const waitComplete = async (transport: typeof first.transport) => { + for await (const notification of transport.notifications()) if (notification.method === "turn/completed") return; + }; + let calls = 0, runnerPid: number | null = null, providerPid: number | null = null; + let recovered: ReturnType | undefined; + const result = { ok: true, task: { id: "task-1" } }; + first.transport.setServerRequestHandler(async () => ++calls === 1 + ? { success: true, contentItems: [{ type: "inputText", text: JSON.stringify(result) }] } + : new Promise>(() => undefined)); + try { + const opened = await within("first thread/start", first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [{ name: "get_task_context", description: "Read current task", + inputSchema: { type: "object", properties: {}, additionalProperties: false } }] })); + runnerPid = first.evidence().runnerPid; + providerPid = first.evidence().codexPid; + expect(providerPid).toEqual(expect.any(Number)); + const providerStart = execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim(); + const startedAt = first.transport.processInfo().startedAt; + const firstCompleted = waitComplete(first.transport); + await within("first turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "first healthy business run" }] })); + await within("healthy first turn completion", firstCompleted); + const attachment = { runId: "run-warm-second-active", turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; + await within("warm second business attachment", first.transport.attachRun!(attachment)); + const active = await within("second active turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "second business run held at tool" }] })); + await vi.waitFor(() => expect(calls).toBe(2), { timeout: 5_000 }); + const activeCore = authority!; + const savedIdentity = { ...activeCore.store.state.identity }; + expect(savedIdentity).toEqual({ ...initialIdentity, ...attachment }); + await activeCore.flushCheckpoint(); + await within("controller detach", first.detachControllerForRestart()); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + await rm(assetRoot, { recursive: true, force: true }); + expect(existsSync(assetRoot)).toBe(false); + checkpoint!.restoreSessionDirectory(nextRoot); + const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); + const openedThread = opened.thread as Record; + const activeTurn = active.turn as Record; + expect(openedThread.id).toEqual(expect.any(String)); + expect(activeTurn.id).toEqual(expect.any(String)); + const shared = { ...options, stateDirectory: nextRoot, + resumeProviderSession: { driverSessionId: String(openedThread.id), providerSessionId: String(openedThread.sessionId) }, + resumeActiveTurnId: String(activeTurn.id), runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), + runnerStateDirectory: join(firstRoot, "runner"), readRunnerState: async () => JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8")), + prepareExternalRunnerState: async () => {}, archiveExternalRunnerState: async () => { throw new Error("Active authority must not rotate"); }, + runnerProcessLauncher: noSpawn, adoptExistingRunner: { pid: runnerPid!, processGroupId: runnerPid, startedAt, + isAlive: () => { try { process.kill(runnerPid!, 0); return true; } catch { return false; } }, + signal: (signal: NodeJS.Signals) => { try { process.kill(runnerPid!, signal); return true; } catch { return false; } } } }; + // The old server's synthetic pair must continue failing the strict Runner guard. + const wrong = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: { ...savedIdentity, + turnId: `turn-${attachment.runId}`, itemId: `item-${attachment.runId}` } }); + await expect(wrong.transport.request("thread/read", {})).rejects.toThrow("PRP recovery identity does not match the durable session binding"); + await wrong.detachControllerForRestart(); + // Leave the pending tool outcome unknown. Reattachment must authenticate + // and preserve it without fabricating a receipt or redispatching its handler. + recovered = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: savedIdentity }); + await within("replacement thread/read", recovered.transport.request("thread/read", {})); + expect(recovered.evidence()).toMatchObject({ runnerPid, runnerExited: false }); + expect(authority!.activeRunnerConnectionCount()).toBe(1); + expect(() => process.kill(providerPid!, 0)).not.toThrow(); + expect(execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim()).toBe(providerStart); + expect(calls).toBe(2); + expect(authority!.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0); + expect(recovered.transport.processInfo().startedAt).toBe(startedAt); + expect(authority!.store.state.identity).toEqual(savedIdentity); + expect(authority!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(noSpawn).not.toHaveBeenCalled(); + } catch (error) { + const saved = JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8").catch(() => "{}")); + const stderr = await readFile(join(firstRoot, "diagnostics/runnerd.stderr.log"), "utf8").catch(() => ""); + throw new Error(`${String(error)}; owned protocol evidence ${JSON.stringify({ + transport: recovered?.evidence(), connections: authority?.activeRunnerConnectionCount(), + identity: authority?.store.state.identity, commands: authority?.store.state.commands.map(c => ({type:c.type,status:c.status})), + events: authority?.store.state.committedEvents.map(e => e.eventType).slice(-8), + runner: { lifecycle:saved.lifecycle, runId:saved.runId, turnId:saved.turnId, itemId:saved.itemId, + outbox:saved.outbox?.length }, stderr:stderr.slice(-1500), + })}`); + } finally { + await (recovered ? within("recovered cleanup", recovered.transport.close()).catch(() => undefined) : undefined); + await within("first cleanup", first.transport.close()).catch(() => undefined); + if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } + await (authority as DurablePrpControlPlane | null)?.stop(); + if (runnerPid) { try { process.kill(-runnerPid, "SIGKILL"); } catch {} } + server.closeAllConnections(); + if (server.listening) await new Promise(done => server.close(() => done())); + for (const controller of [firstRoot, nextRoot]) await releaseMaterializedNativeRuntimeSkills(join(controller, "codex-home", "skills")); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); + +it("waits for a warm runner to re-authenticate before probing attachment readiness", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-warm-reattach-before-probe-"), + ); + const server = createServer(); + const authorities = new Map(); + let blockFirstAuthorityReconnect = false; + let resolveRejectedReconnect!: () => void; + const rejectedReconnect = new Promise((resolvePromise) => { + resolveRejectedReconnect = resolvePromise; + }); + server.on("upgrade", (request, socket, head) => { + const route = request.url ?? ""; + if (route === "/runner-1" && blockFirstAuthorityReconnect) { + resolveRejectedReconnect(); + socket.destroy(); + return; + } + const authority = authorities.get(route); + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, route, head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected warm reconnect test listener"); + } + let registrationCount = 0; + const diagnostics: string[] = []; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, + onDiagnostic: (message) => diagnostics.push(message), + controlPlaneRegistration: async (authority) => { + registrationCount += 1; + const route = `/runner-${registrationCount}`; + authorities.set(route, authority); + return { + connectUrl: `ws://127.0.0.1:${address.port}${route}`, + release: () => { + if (authorities.get(route) === authority) authorities.delete(route); + }, + }; + }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: codexSemanticToolSpecs(), + }); + runnerPid = bundle.evidence().runnerPid; + const firstAuthority = authorities.get("/runner-1"); + if (!firstAuthority) throw new Error("Missing first warm authority"); + const priorSnapshotCount = firstAuthority.store.state.commands.filter( + (command) => command.type === "session.snapshot", + ).length; + + blockFirstAuthorityReconnect = true; + firstAuthority.disconnectActiveRunner(); + const attachment = bundle.transport.attachRun!({ + runId: "run-warm-after-reconnect", + turnId: "turn-warm-after-reconnect", + itemId: "item-warm-after-reconnect", + }); + await Promise.race([ + rejectedReconnect, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("runner did not attempt to reconnect")), + 5_000, + ), + ), + ]); + + // No command may be queued while its sole authenticated consumer is + // absent. The generic 30-second command timeout used to turn this state + // into same-run recovery and replace the healthy warm runner process. + expect( + firstAuthority.store.state.commands.filter( + (command) => command.type === "session.snapshot", + ), + ).toHaveLength(priorSnapshotCount); + + blockFirstAuthorityReconnect = false; + await Promise.race([ + attachment, + new Promise((_, reject) => + setTimeout(() => reject(new Error("warm attachment timeout")), 10_000), + ), + ]); + expect(bundle.evidence()).toMatchObject({ + runnerPid, + runnerExited: false, + }); + expect(diagnostics).toContain( + "warm runner connection interrupted; waiting for re-authentication before authority rotation", + ); + expect(diagnostics).toContain( + "warm runner re-authenticated before authority rotation", + ); + } finally { + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // A successful durable close already stopped the runner process group. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("releases both PRP authorities when warm rotation activation fails", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-warm-attach-activation-failure-"), + ); + const server = createServer(); + const authorities = new Map(); + const released: string[] = []; + server.on("upgrade", (request, socket, head) => { + const route = request.url ?? ""; + const authority = authorities.get(route); + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, route, head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected warm activation failure test listener"); + } + let registrationCount = 0; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + controlPlaneRegistration: async (authority) => { + registrationCount += 1; + const route = `/runner-${registrationCount}`; + authorities.set(route, authority); + return { + connectUrl: `ws://127.0.0.1:${address.port}${route}`, + ...(registrationCount === 1 + ? {} + : { + activate: () => { + throw new Error("rotation activation failed"); + }, + }), + release: () => { + released.push(route); + if (authorities.get(route) === authority) authorities.delete(route); + }, + }; + }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: codexSemanticToolSpecs(), + }); + runnerPid = bundle.evidence().runnerPid; + + await expect( + bundle.transport.attachRun!({ + runId: "run-warm-activation-failure", + turnId: "turn-warm-activation-failure", + itemId: "item-warm-activation-failure", + }), + ).rejects.toThrow("rotation activation failed"); + expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"])); + expect(authorities.size).toBe(0); + await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( + "rotation activation failed", + ); + } finally { + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // A successful durable close already stopped the runner process group. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it.each([ + { + binding: "runner instance", + priorRunnerInstanceId: "runner-other", + priorEnvironmentLeaseId: "lease-current", + }, + { + binding: "environment lease", + priorRunnerInstanceId: "runner-current", + priorEnvironmentLeaseId: "lease-other", + }, +])( + "quarantines a mismatched $binding instead of reusing its provider session", + async ({ priorRunnerInstanceId, priorEnvironmentLeaseId }) => { + const container = await mkdtemp( + join(tmpdir(), "runnerd-mismatched-authority-"), + ); + const stateDirectory = join(container, "state"); + await mkdir(join(stateDirectory, "control-plane"), { recursive: true }); + await writeFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + JSON.stringify({ + identity: { + runnerInstanceId: priorRunnerInstanceId, + environmentLeaseId: priorEnvironmentLeaseId, + runId: "run-old", + normalizedSessionId: "session-current", + turnId: "turn-old", + itemId: "item-old", + }, + }), + { mode: 0o600 }, + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + stateDirectory, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: { + runnerInstanceId: "runner-current", + environmentLeaseId: "lease-current", + runId: "run-new", + normalizedSessionId: "session-current", + turnId: "turn-new", + itemId: "item-new", + }, + }); + try { + await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( + "native_runner_state_quarantined", + ); + expect(await readdir(stateDirectory)).toEqual([]); + expect( + (await readdir(container)).some((entry) => + entry.startsWith("state.quarantine-"), + ), + ).toBe(true); + } finally { + await bundle.transport.close(); + await rm(container, { recursive: true, force: true }); + } + }, +); + +it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected", "credentials-cleaned"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); + const tracePath = join(stateDirectory, "provider-trace.ndjson"); + const skillRoot = join(stateDirectory, "runtime-skill"); + const instructionRoot = join(stateDirectory, "runtime-instructions"); + await Promise.all([mkdir(skillRoot), mkdir(instructionRoot)]); + await writeFile(join(skillRoot, "SKILL.md"), "# Assigned runtime skill\n"); + await writeFile(join(instructionRoot, "AGENTS.md"), "Runtime instructions\n"); + const credentialsCleaned = pendingTail === "credentials-cleaned"; + const sourceCodexHome = join(stateDirectory, "selected-account-first"); + if (credentialsCleaned) { + await mkdir(sourceCodexHome); + await writeFile(join(sourceCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } + const baseIdentity = { + runnerInstanceId: "runner-cold-attach", + environmentLeaseId: "lease-cold-attach", + runId: "run-cold-first", + normalizedSessionId: "session-cold-attach", + turnId: "turn-cold-first", + itemId: "item-cold-first", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--include-skill-instructions", + "--durable-turn-ids", + ...(credentialsCleaned ? ["--require-codex-home-auth"] : []), + "-c", + 'shell_environment_policy.set={PATH="/run/A"}', + ), + stateDirectory, + ...(credentialsCleaned ? { sourceCodexHome } : {}), + environment: { + PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-A-capability", + PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, + PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_NAME: "assigned", + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/first", + PAPERCLIP_NATIVE_MCP_TOKEN: "first-assigned-gateway-credential", + } : {}), + }, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), + }; + const dynamicTools = [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ]; + const first = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: baseIdentity, + }); + first.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let firstProviderThread: { id: string; sessionId: string } | null = null; + try { + const started = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [...dynamicTools, ...codexSemanticToolSpecs()], + completionContract: { + revision: "contract-first", + criterionIds: ["criterion-first"], + }, + }); + const startedThread = started.thread as Record; + firstProviderThread = { + id: String(startedThread.id), + sessionId: String(startedThread.sessionId), + }; + expect( + (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) + .mode & 0o222, + ).toBe(0); + expect( + ( + await stat( + join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), + ) + ).mode & 0o222, + ).toBe(0); + await first.transport.request("turn/start", { + input: [{ type: "text", text: "$assigned first process" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], + }); + for await (const event of first.transport.notifications()) { + if (event.method === "turn/completed") break; + } + } finally { + await first.transport.close(); + } + if (!firstProviderThread) { + throw new Error("cold attach fixture did not return a provider thread"); + } + const successorCodexHome = join(stateDirectory, "selected-account-next"); + if (credentialsCleaned) { + await rm(join(stateDirectory, "codex-home", "auth.json")); + await rm(sourceCodexHome, { recursive: true }); + await mkdir(successorCodexHome); + await writeFile(join(successorCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } + + // Model a retained provider-owned suffix: rotation must deliver and durably + // accept it before attempting run.attach, rather than dropping or ACKing it. + const retainedDelta = "cold-recovery-retained-tail"; + if (pendingTail) { + const providerFile = join(stateDirectory, "runner", "codex-provider-state.json"); + const provider = JSON.parse(await readFile(providerFile, "utf8")); + expect(provider.pendingEvents).toEqual([]); + provider.pendingEvents.push({ + executorEventId: `codex_provider_${String(provider.nextProviderEventSeq++).padStart(16, "0")}`, + eventType: pendingTail === "unknown-semantic" ? "semantic_tool.input" : "usage.reported", + priority: "p0", + payload: pendingTail === "unknown-semantic" ? { semantic_tool: { + schema: "paperclip.prp.semantic_tool.v1", schemaVersion: 1, phase: "input", + callId: "unknown-source-call", operationId: "get_task_context", input: {}, idempotencyKey: null, + correlation: { runId: baseIdentity.runId, normalizedSessionId: baseIdentity.normalizedSessionId, + turnId: baseIdentity.turnId, itemId: baseIdentity.itemId }, + content: { digest: `sha256:${createHash("sha256").update("{}").digest("hex")}`, redactionDisposition: "digest_only", references: [] }, + } } : { provider: "codex", kind: "usage", delta: retainedDelta, + providerTurnId: provider.completed_provider_turn_id, runDelta: { inputTokens: 20, outputTokens: 2 }, + runDeltaAvailable: true, runDeltaBasis: "run" }, + }); + await writeFile(providerFile, JSON.stringify(provider)); + } + const retainedEvents: PrpEvent[] = []; + let releaseRetained!: () => void, retainedEntered!: () => void; + const retainedAccepted = new Promise(done => { releaseRetained = done; }); + const retainedAdmission = new Promise(done => { retainedEntered = done; }); + const secondIdentity = { + ...baseIdentity, + runId: "run-cold-second", + turnId: "turn-cold-second", + itemId: "item-cold-second", + }; + let stagedCheckpoint: durableControlPlane.DurablePrpCheckpoint | undefined; + let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; + const rotatedOptions = { + ...options, + ...(credentialsCleaned ? { sourceCodexHome: successorCodexHome } : {}), + onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { + authority.setCheckpointPersistence(async checkpoint => { + stagedCheckpoint = checkpoint; + if (rejectStagedCommit) throw new Error("fixture lost controller after durable intent"); + }); + } : undefined, + onRecoveryAccountingEvent: pendingTail ? async (event: PrpEvent) => { + if (event.payload.delta === retainedDelta) { + retainedEvents.push(structuredClone(event)); + retainedEntered(); + if (pendingTail === "accounting-rejected") throw new Error("source accounting receipt unavailable"); + await retainedAccepted; + } + } : undefined, + environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability", + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/current", + PAPERCLIP_NATIVE_MCP_TOKEN: "current-assigned-gateway-credential", + } : {}), + }, + codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), + resumeDynamicTools: dynamicTools, + resumeCompletionContract: { + revision: "contract-second", + criterionIds: ["criterion-second"], + }, + prpIdentity: secondIdentity, + }; + let rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + if (pendingTail === "restart-before-bootstrap") { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("fixture lost controller after durable intent"); + expect(rotated.evidence().runnerPid).toBeNull(); + expect(stagedCheckpoint?.runId).toBe(secondIdentity.runId); + expect([...stagedCheckpoint!.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value).type)).not.toContain("run.attach"); + await rotated.detachControllerForRestart().catch(() => undefined); + rejectStagedCommit = false; + await rm(join(stateDirectory, "control-plane"), { recursive: true, force: true }); + stagedCheckpoint!.restoreSessionDirectory(stateDirectory); + rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + } + const currentHandler = vi.fn(async () => ({ success: true, contentItems: [] })); + rotated.transport.setServerRequestHandler(currentHandler); + if (pendingTail === "accounting-rejected") { + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + try { + await Promise.race([retainedAdmission, reading]); + const source = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(source.identity).toEqual(baseIdentity); + expect(source.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(source.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + expect(currentHandler).not.toHaveBeenCalled(); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await expect(reading).rejects.toThrow(); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } + if (pendingTail === "unknown-semantic") { + try { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("native_runner_source_semantic_reconciliation_required"); + expect(currentHandler).not.toHaveBeenCalled(); + const retained = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retained.identity).toEqual(baseIdentity); + expect(retained.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(retained.commands.some((command: { type: string }) => ["run.attach", "semantic_tool.result"].includes(command.type))).toBe(false); + expect(retained.committedEvents.some((event: PrpEvent) => event.eventType === "semantic_tool.input")).toBe(true); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } + let recoveryError: unknown; + try { + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + if (pendingTail) { + await Promise.race([retainedAdmission, reading]); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + const retainedCore = stagedCheckpoint ? { + identity: JSON.parse(stagedCheckpoint.records.get("header")!).state.identity, + commands: [...stagedCheckpoint.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value)), + } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retainedCore.identity).toEqual(baseIdentity); + expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + if (credentialsCleaned) { + const config = await readFile(join(stateDirectory, "codex-home", "config.toml"), "utf8"); + expect(config).toContain("https://paperclip.example/mcp/current"); + expect(config).toContain("current-assigned-gateway-credential"); + expect(config).not.toContain("first-assigned-gateway-credential"); + expect(currentHandler).not.toHaveBeenCalled(); + } + } + releaseRetained(); + const read = await reading; + expect(retainedEvents).toHaveLength(pendingTail ? 1 : 0); + if (pendingTail) expect(retainedEvents[0]).toMatchObject({ + runId: baseIdentity.runId, turnId: baseIdentity.turnId, itemId: baseIdentity.itemId, + }); + const persistedProvider = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); + expect(persistedProvider.pendingEvents).toEqual([]); + expect(persistedProvider.config.args.join("\n")).toContain('/run/B'); + expect(JSON.stringify(persistedProvider)).not.toContain("test-run-B-capability"); + expect(read.thread).toMatchObject({ + id: firstProviderThread.id, + sessionId: firstProviderThread.sessionId, + cwd: tmpdir(), + }); + await rotated.transport.request("turn/start", { + input: [{ type: "text", text: "$assigned second authority epoch" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], + }); + for await (const event of rotated.transport.notifications()) { + if (event.method === "paperclip/runResult") break; + } + expect(rotated.evidence().diagnostics).toContain( + "runnerd attached the durable provider session to a fresh PRP run authority", + ); + if (pendingTail) { + const target = stagedCheckpoint ? JSON.parse(stagedCheckpoint.records.get("header")!).state + : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(target.stagedRunAttachment).toBeUndefined(); + expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); + } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); + } catch (error) { + recoveryError = error; + throw error; + } finally { + releaseRetained(); + await rotated.transport.close().catch(error => { + if (recoveryError) throw new AggregateError([recoveryError, error], "Provider recovery and cleanup failed"); + throw error; + }); + } + const requestFrames = (await readFile(tracePath, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record) + .filter( + (entry) => + entry.kind === "frame" && entry.direction === "client_to_provider", + ) + .map( + (entry) => + JSON.parse( + Buffer.from(String(entry.rawBase64), "base64").toString("utf8"), + ) as Record, + ) + ; + const resumeFrames = requestFrames.filter((frame) => frame.method === "thread/resume"); + for (const frame of requestFrames.filter((f) => ["thread/start", "thread/resume"].includes(String(f.method)))) { + expect(frame.params).toMatchObject({ config: { "skills.include_instructions": true } }); + } + const turnFrames = requestFrames.filter((f) => f.method === "turn/start"); + expect(turnFrames).toHaveLength(2); + for (const frame of turnFrames) { + expect(frame.params).toMatchObject({ input: expect.arrayContaining([{ type: "skill", name: "assigned", path: join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md") }]) }); + } + expect(resumeFrames.length).toBeGreaterThanOrEqual(1); + for (const frame of resumeFrames) { + expect(frame).toEqual( + expect.objectContaining({ + method: "thread/resume", + params: expect.objectContaining({ threadId: firstProviderThread.id }), + }), + ); + } + + if (pendingTail) { + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + return; + } + + // A remote process owner keeps runner-state outside the controller's local + // session root. Resume must defer to its explicit state reader instead of + // rejecting recovery before the remote checkpoint can be made available. + const externallyOwnedRunnerStateDirectory = join( + stateDirectory, + "externally-owned-runner", + ); + await rename( + join(stateDirectory, "runner"), + externallyOwnedRunnerStateDirectory, + ); + const readRunnerState = async () => + JSON.parse( + await readFile( + join(externallyOwnedRunnerStateDirectory, "runner-state.json"), + "utf8", + ), + ) as Record; + const mismatched = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + resumeDynamicTools: dynamicTools, + prpIdentity: { + ...secondIdentity, + runId: "run-cold-other", + turnId: "turn-cold-other", + itemId: "item-cold-other", + }, + }); + await expect(mismatched.transport.request("thread/read", {})).rejects.toThrow( + "native_runner_prp_run_rotation_unavailable", + ); + await mismatched.transport.close(); + + const externalIdentity = { + ...secondIdentity, + runId: "run-cold-external", + turnId: "turn-cold-external", + itemId: "item-cold-external", + }; + const rejectedExternalRotationSteps: string[] = []; + let externalArchiveDirectory: string | null = null; + const rejectedExternalRotation = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + rejectedExternalRotationSteps.push("prepared"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + await expect( + stat(join(stateDirectory, "control-plane")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect( + await stat( + join( + stateDirectory, + "authority-epochs", + `epoch-${archiveKey}`, + "control-plane", + ), + ), + ).toBeDefined(); + externalArchiveDirectory = join( + stateDirectory, + "external-authority-epochs", + archiveKey, + ); + await mkdir(externalArchiveDirectory, { recursive: true }); + await rename( + join(externallyOwnedRunnerStateDirectory, "runner-state.json"), + join(externalArchiveDirectory, "runner-state.json"), + ); + rejectedExternalRotationSteps.push("remote-archived"); + throw new Error("controller crashed after external archive"); + }, + resumeDynamicTools: dynamicTools, + prpIdentity: externalIdentity, + }); + await expect( + rejectedExternalRotation.transport.request("thread/read", {}), + ).rejects.toThrow("controller crashed after external archive"); + await rejectedExternalRotation.transport.close(); + expect(rejectedExternalRotationSteps).toEqual([ + "prepared", + "remote-archived", + ]); + await expect(stat(join(stateDirectory, "control-plane"))).rejects.toThrow(); + await expect( + stat(join(externallyOwnedRunnerStateDirectory, "runner-state.json")), + ).rejects.toThrow(); + + const externalRotationSteps: string[] = []; + const externallyRotated = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + throw new Error("retry must not prepare a new external runner"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + expect(externalArchiveDirectory).toBe( + join(stateDirectory, "external-authority-epochs", archiveKey), + ); + externalRotationSteps.push("archived"); + return JSON.parse( + await readFile( + join(externalArchiveDirectory!, "runner-state.json"), + "utf8", + ), + ) as Record; + }, + resumeDynamicTools: dynamicTools, + resumeCompletionContract: { + revision: "contract-external", + criterionIds: ["criterion-external"], + }, + prpIdentity: externalIdentity, + }); + externallyRotated.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await externallyRotated.transport.request("thread/read", {}); + expect(read.thread).toMatchObject({ + id: firstProviderThread.id, + sessionId: firstProviderThread.sessionId, + cwd: tmpdir(), + }); + expect(externalRotationSteps).toEqual(["archived"]); + } finally { + await externallyRotated.transport.close(); + } + + const restored = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + resumeDynamicTools: dynamicTools, + prpIdentity: externalIdentity, + }); + restored.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await restored.transport.request("thread/read", {}); + expect(read.thread).toMatchObject({ + id: "codex-thread-1", + cwd: tmpdir(), + }); + expect( + (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) + .mode & 0o222, + ).toBe(0); + expect( + ( + await stat( + join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), + ) + ).mode & 0o222, + ).toBe(0); + const providerState = JSON.parse( + await readFile( + join(externallyOwnedRunnerStateDirectory, "codex-provider-state.json"), + "utf8", + ), + ) as { toolBridge?: { authorized?: Record } }; + expect(Object.keys(providerState.toolBridge?.authorized ?? {})).toEqual([ + "get_task_context", + "paperclip_block", + "paperclip_finish", + ]); + await restored.transport.request("turn/start", { + input: [{ type: "text", text: "restored process" }], + }); + for await (const event of restored.transport.notifications()) { + if (event.method === "turn/completed") break; + } + expect(restored.evidence()).toMatchObject({ + runnerExited: false, + codexPid: expect.any(Number), + }); + } finally { + await restored.transport.close(); + await releaseMaterializedNativeRuntimeSkills( + join(stateDirectory, "codex-home", "skills"), + ); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index da352598f0c..9a3f342ef43 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1,80 +1,41 @@ -import { bindRunnerLaunchResources, createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type PreparedRunnerProviderLaunch, type RunnerProviderLaunchRequirements } from "./runner-provider-launch.js"; +import { + bindRunnerLaunchResources, + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; import { coldAdmissionTimeoutMs, waitForRunnerCommand } from "./runnerd-codex-transport.js"; import { configuredEnvironmentProjection } from "../configured-environment.js"; -import { - chmod, - cp, - mkdir, - lstat, - mkdtemp, - open, - readFile, - readdir, - readlink, - realpath, - rename, - rm, - stat, - truncate, - writeFile, -} from "node:fs/promises"; -import { createHash, randomUUID } from "node:crypto"; +import { chmod, cp, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, stat, truncate, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { execFileSync, spawn } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { bindAcpxAgentFiles } from "../drivers/acpx/agent-files-binding.js"; import { resolveRunnerProviderAssetsRoot } from "../drivers/acpx/provider-assets-root.js"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; - import { describe, expect, it, vi } from "vitest"; -import type { ControlPlanePort } from "../contracts/control-plane-port.js"; -import type { NativeExecutionInputV1 } from "../contracts/native-execution.js"; -import type { - NativeSession, - NativeSessionBackend, -} from "../contracts/native-session-backend.js"; -import type { - PrpEvent, - PrpStructuredRunResult, - PrpTerminalState, -} from "../protocol/replay-contract.js"; +import type { PrpEvent } from "../protocol/replay-contract.js"; import { PRP_PROTOCOL_VERSION } from "../protocol/replay-contract.js"; -import { executeNativeSession } from "../native-session-runtime.js"; import { redactCapabilityEvidenceData } from "./evidence-redaction.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import { parsePaperclipQuestionSet } from "../contracts/question-set.js"; -import { ACPX_CREDENTIAL_BINDING_ENV, createAcpxCredentialBinding, createAcpxSidecarHostEnvironment } from "../drivers/acpx/environment.js"; +import { + ACPX_CREDENTIAL_BINDING_ENV, + createAcpxCredentialBinding, + createAcpxSidecarHostEnvironment, +} from "../drivers/acpx/environment.js"; import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; import * as codexCommandRuntime from "../drivers/codex/codex-command.js"; - -import { - NATIVE_RUNTIME_ASSET_SCHEMA, - PAPERCLIP_EXECUTION_PROMPT, - PAPERCLIP_EXECUTION_PROMPT_REVISION, - canonicalNativeRuntimeContextDigest, - composeNativeSystemInstructions, - nativeRuntimePromptDigest, - type NativeRuntimeContextSnapshot, -} from "../contracts/runtime-context.js"; -import { - CODEX_SKILLLESS_BASE_INSTRUCTIONS, - createCodexTaskEnvelope, -} from "../contracts/codex.js"; -import { - PRP_BLOCK_TOOL_DESCRIPTION, - PRP_COMPLETION_TOOL_DESCRIPTION, -} from "../contracts/completion-result.js"; -import { - CodexAppServerDriver, - codexSemanticToolSpecs, -} from "../drivers/codex/codex-app-server-driver.js"; -import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; +import { composeNativeSystemInstructions, type NativeRuntimeContextSnapshot } from "../contracts/runtime-context.js"; +import { CODEX_SKILLLESS_BASE_INSTRUCTIONS, createCodexTaskEnvelope } from "../contracts/codex.js"; +import { PRP_BLOCK_TOOL_DESCRIPTION, PRP_COMPLETION_TOOL_DESCRIPTION } from "../contracts/completion-result.js"; +import { CodexAppServerDriver, codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; import * as runtimeMaterializer from "../drivers/runtime-context-materializer.js"; import { RUNNERD_CANONICAL_ITEM } from "../drivers/codex/codex-driver-values.js"; - import { probeNativeRunnerEnvironment, NativeEnvironmentProbeCleanupError, @@ -83,7 +44,6 @@ import { createCapabilityRunnerdCodexTransport, createCapabilityRunnerdProviderEnvironment, createRunnerdCodexAppServerArgs, - defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary, readRunnerdArtifactBinding, expandRunnerdCanonicalNotifications, latestRunnerdSessionReadiness, @@ -109,11 +69,13 @@ import { resolveRunnerdCodexSkillInputs, withCodexCollaborationRuntimeInstructions, } from "./runnerd-codex-transport.js"; - -// Explicit private-artifact test lane; production/default dist is never changed. -const defaultCapabilityRunnerdBinary = () => - process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - qualifiedCapabilityRunnerdBinary(); +import { + defaultCapabilityRunnerdBinary, + expectTurnStarted, + fakeCodex, + fakeCodexArgs, + assignedRuntimeContext, +} from "../../test/support/runnerd-transport.js"; it("attributes opened candidate sessions to the harness biller instead of the model vendor", () => { const provider = runnerdLaunchProfileInternals.openedThreadModelProvider; @@ -130,21 +92,6 @@ it("attributes opened candidate sessions to the harness biller instead of the mo expect(provider("aws_agentcore", "codex", "global.anthropic.claude-sonnet-4-6")).toBe("aws"); }); -async function expectTurnStarted( - notifications: AsyncIterator<{ method: string }>, -) { - for (let index = 0; index < 32; index += 1) { - const next = await notifications.next(); - expect(next.done).not.toBe(true); - if (next.value.method === "turn/started") return; - // Startup ownership and capability facts can precede the active turn. - expect(next.value.method).toBe("paperclip/canonicalProviderEvent"); - } - throw new Error( - "provider turn did not start within the bounded notification prefix", - ); -} - it("replaces an owned v1 runner with fresh v2 authorization before warm attachment", async () => { const directory = await mkdtemp(join(tmpdir(), "runnerd-v1-v2-replacement-")); const handles: durableControlPlane.RunnerProcessHandle[] = []; @@ -458,12 +405,6 @@ it("preserves the cause when a replacement runner's health cannot be verified", } }, 15_000); -// The retained-settlement maintenance family ("settles only retained control -// authority without starting another provider turn") lives in -// runnerd-codex-transport-settlement.test.ts so vitest can run its ~160s of -// sequential cases on a separate worker instead of serializing them behind -// this file's tests. - it.each(["alive", "pending_liveness", "pending_registration"] as const)( "bounds adopted runner authentication at its exact deadline with %s evidence", async (mode) => { @@ -2055,6 +1996,7 @@ it.each([false, true])("binds a vendored source server to its pnpm dependency ro await rm(workspace, { recursive: true, force: true }); } }); + it.each((["copilot", "pi", "antigravity"] as const).flatMap(agent => ["source", "scoped-npm", "deployment"].map(layout => ({ agent, layout }))))( "admits the controller-selected $agent package authority in a $layout layout", @@ -3498,11 +3440,6 @@ it("recovers provider readiness from an already-committed journal without replay ).toEqual(persistedReady); }); -const fakeCodex = resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", -); - it("keeps the Copilot authentication launch requirements free of Codex search policy", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-copilot-probe-policy-")); let captured: RunnerProviderLaunchRequirements | undefined; @@ -3641,14 +3578,6 @@ it.each(["stuck", "late-release", "late-release-error", "stuck-release", "cooper }, ); -function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { - return [ - "--state-file", - join(stateDirectory, "fake-codex-state.json"), - ...args, - ]; -} - it("joins owned credential preparation before close returns and never launches after close", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-owned-startup-")); const artifact = join(root, "unexecuted-runner-artifact"); @@ -3894,51 +3823,6 @@ it.each(["admission-only", "admission-only-mismatched-model", "complete", "api-k } finally { await rm(root, { recursive: true, force: true }); } }, 20_000); -function assignedRuntimeContext( - skillRoot: string, - instructionRoot: string, -): NativeRuntimeContextSnapshot { - const digest = "0".repeat(64); - const value = { - prompt: { - revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, - text: PAPERCLIP_EXECUTION_PROMPT, - digest: nativeRuntimePromptDigest(), - }, - instructions: { - entryPath: "AGENTS.md", - bundle: { - schema: NATIVE_RUNTIME_ASSET_SCHEMA, - digest, - manifestDigest: digest, - rootPath: instructionRoot, - fileCount: 1, - totalBytes: 1, - }, - }, - skills: [ - { - key: "company/assigned", - runtimeName: "assigned", - versionId: "version-1", - bundle: { - schema: NATIVE_RUNTIME_ASSET_SCHEMA, - digest, - manifestDigest: digest, - rootPath: skillRoot, - fileCount: 1, - totalBytes: 1, - }, - }, - ], - mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" }, - } satisfies Omit; - return { - ...value, - aggregateDigest: canonicalNativeRuntimeContextDigest(value), - }; -} - it("unwraps a coalesced provider notification without losing its turn identity", () => { expect( unwrapRunnerdProviderNotification({ @@ -4817,528 +4701,6 @@ it.each([ 60_000, ); -it("rejects active work and buffered tools from a resumed stopped checkpoint", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-stopped-resume-active-"), - ); - const identity = { - runnerInstanceId: "runner-stopped-active", - environmentLeaseId: "lease-stopped-active", - runId: "run-stopped-first", - normalizedSessionId: "session-stopped-active", - turnId: "turn-stopped-first", - itemId: "item-stopped-first", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--linger-after-turn-start", - "--resume-unowned-turn-when-marked", - "--emit-tool-call-on-resume", - ), - stateDirectory, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - }; - const first = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: identity, - }); - let second: - ReturnType | undefined; - const semanticHandler = vi.fn(async () => ({ - success: true, - contentItems: [], - })); - first.transport.setServerRequestHandler(semanticHandler); - try { - const opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [ - { - name: "get_task_context", - description: "Read the current task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ], - }); - await first.transport.request("turn/start", { - input: [{ type: "text", text: "Wait for another instruction." }], - }); - await expectTurnStarted( - first.transport.notifications()[Symbol.asyncIterator](), - ); - await first.transport.close(); - const providerPath = join( - stateDirectory, - "runner", - "codex-provider-state.json", - ); - expect(JSON.parse(await readFile(providerPath, "utf8"))).toMatchObject({ - lifecycle: "prepared", - activeProviderTurnId: null, - }); - await writeFile(join(stateDirectory, "resume-unowned-turn"), "armed"); - const secondIdentity = { - ...identity, - runId: "run-stopped-second", - turnId: "turn-stopped-second", - itemId: "item-stopped-second", - }; - second = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: secondIdentity, - }); - second.transport.setServerRequestHandler(semanticHandler); - await expect(second.transport.request("thread/read", {})).rejects.toThrow( - "prepared provider checkpoint resumed unexpected active work", - ); - // Exercise the normal durable transfer before inspecting the rejection: - // provider pendingEvents is an acknowledged queue, not an event journal. - await vi.waitFor( - async () => { - const provider = JSON.parse(await readFile(providerPath, "utf8")); - expect(provider.pendingEvents).toEqual([]); - const control = JSON.parse( - await readFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - "utf8", - ), - ) as { - committedEvents: Array<{ - eventType: string; - envelope: { payload: { payload: { code?: string } } }; - }>; - }; - const rejections = control.committedEvents.filter( - (event) => - event.eventType === "harness.diagnostic" && - event.envelope.payload.payload.code === - "prepared_provider_checkpoint_has_active_work", - ); - expect(rejections).toEqual([ - expect.objectContaining({ - logicalEffectCount: 1, - envelope: expect.objectContaining({ - ...secondIdentity, - payload: expect.objectContaining({ - payload: expect.objectContaining({ - code: "prepared_provider_checkpoint_has_active_work", - paperclipAccepted: false, - providerReportedActive: true, - }), - }), - }), - }), - ]); - }, - { timeout: 5_000 }, - ); - const closed = JSON.parse(await readFile(providerPath, "utf8")); - expect(closed).toMatchObject({ - lifecycle: "closed", - threadId: (opened.thread as Record).id, - activeProviderTurnId: null, - }); - expect(semanticHandler).not.toHaveBeenCalled(); - expect(closed.pendingEvents).toEqual([]); - } finally { - await Promise.allSettled([ - first.transport.close(), - second?.transport.close(), - ]); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); -it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => { - const identity = { - runId: "run-recovery", - sessionId: "session-recovery", - companyId: "company-recovery", - issueId: "issue-recovery", - agentId: "agent-recovery", - }; - - const result: PrpStructuredRunResult = { - schema: "paperclip.run_result.v1", - reportedWorkDisposition: "done", - summary: "Recovered native work completed.", - completionClaim: { - contractRevision: "1", - objectiveSatisfied: true, - criteria: [ - { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, - ], - remainingWork: [], - }, - evidence: [], - verification: [{ commandOrCheck: "recovery", status: "passed" }], - attentionRequests: [], - artifacts: [], - }; - - const terminal: PrpTerminalState = { - schema: "paperclip.prp.terminal.v1", - turnTerminalState: "completed", - runTerminalState: "succeeded", - reportedWorkDisposition: "done", - }; - - const input: NativeExecutionInputV1 = { - schema: "paperclip.native-execution-input.v1", - binding: { - companyId: identity.companyId, - runId: identity.runId, - issueId: identity.issueId, - agentId: identity.agentId, - executionWorkspaceId: "workspace-recovery", - }, - task: { - identifier: "PAP-RECOVERY", - title: "Recover native work", - description: null, - prompt: "# PAP-RECOVERY: Recover native work", - workMode: "standard", - }, - workspace: { - cwd: "/workspace", - repoUrl: null, - repoRef: null, - branchName: null, - }, - session: { - normalizedSessionId: identity.sessionId, - driverKind: "codex_app_server", - protocolVersion: 1, - }, - provider: { kind: "codex", model: null }, - completionContract: { - id: "contract-recovery", - sha256: "contract-recovery-sha", - schemaVersion: "paperclip.completion-contract.v1", - contract: { - revision: "1", - objective: "Recover native work", - criteria: [{ id: "objective", requirement: "Complete after recovery" }], - }, - }, - interactionResponses: [], - credentialBindings: [], - }; - - function runnerEvent( - sourceSeq: number, - eventType: PrpEvent["eventType"], - payload: Record = {}, - ): PrpEvent { - return { - schema: "paperclip.prp.event.v1", - sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, - sourceSeq, - sourceInstanceId: "runner-recovery", - sourceKind: "runner", - runId: identity.runId, - normalizedSessionId: identity.sessionId, - turnId: "turn-recovery", - eventType, - schemaVersion: 1, - priority: 0, - emittedAt: "2026-08-09T00:00:00.000Z", - payload, - }; - } - const stateDirectory = await mkdtemp( - join(tmpdir(), "native-close-quarantine-"), - ); - const readRunnerState = vi.fn(async () => ({ - schema: "paperclip.runner.durable.state.v1", - runnerInstanceId: "runner-close-quarantine", - environmentLeaseId: "lease-close-quarantine", - runId: identity.runId, - normalizedSessionId: identity.sessionId, - turnId: "turn-close-quarantine", - itemId: "item-close-quarantine", - lifecycle: "ready", - })); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", - ), - codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], - stateDirectory, - closeGraceMs: 400, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: await readRunnerState(), - readRunnerState, - // A checkpoint owner requires durable suspension proof. A local transport - // without a checkpoint can simply terminate its process on close. - controlPlaneRegistration: async (authority) => { - await authority.start(); - return { - connectUrl: authority.connectUrl, - checkpoint: async () => {}, - release: async () => {}, - }; - }, - }); - try { - await bundle.transport.request("thread/start", { - cwd: stateDirectory, - dynamicTools: [], - }); - const failedClose = bundle.transport.close(); - const failure = await failedClose.catch((error: unknown) => error); - expect(failure).toMatchObject({ - code: "native_session_close_unrecoverable", - }); - expect(bundle.transport.close()).toBe(failedClose); - - vi.useFakeTimers(); - const close = vi.fn(({ reason }: { reason: string }) => - bundle.transport.close(reason), - ); - const capabilities = { - resume: true, - typedEvents: true, - steering: false, - interruption: true, - structuredResult: true, - }; - const session: NativeSession = { - identity: () => identity, - async capabilities() { - return capabilities; - }, - async *events() { - yield runnerEvent(1, "turn.completed"); - }, - async startTurn() { - return { turnId: "turn-recovery" }; - }, - async result() { - return { result, terminal, turnId: "turn-recovery" }; - }, - async snapshot() { - return { - backendKind: "mock", - sessionId: identity.sessionId, - identity, - providerSessionId: "provider-recovery", - cursor: null, - activeTurnId: null, - pendingRuntimeRequests: [], - lineage: [], - }; - }, - close, - }; - const openSession = vi.fn(async () => session); - const backend: NativeSessionBackend = { - async descriptor() { - return { - kind: "mock", - name: "real-memoized-close-quarantine", - version: "1", - capabilities, - }; - }, - openSession, - }; - const port: ControlPlanePort = { - async openRun() {}, - async checkpointSession() {}, - async appendEvent() { - return { - cursor: 1, - highestContiguousSourceSeq: 1, - disposition: "committed", - }; - }, - async replayEvents() { - return { events: [], highestContiguousSourceSeq: 0 }; - }, - async completeRun() {}, - }; - const execute = () => - executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - requireSessionCloseBeforeReturn: true, - }); - await expect(execute()).rejects.toBe(failure); - const readsAfterClose = readRunnerState.mock.calls.length; - await expect(execute()).rejects.toMatchObject({ - code: "native_session_cleanup_quarantined", - recovery: "operator_required", - }); - await vi.advanceTimersByTimeAsync(600_000); - await expect(execute()).rejects.toMatchObject({ - code: "native_session_cleanup_quarantined", - }); - expect(openSession).toHaveBeenCalledOnce(); - expect(close).toHaveBeenCalledOnce(); - expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose); - } finally { - vi.useRealTimers(); - await bundle.transport.close().catch(() => undefined); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 10_000); - -it.each(["not_suspended", "wrong_identity"] as const)( - "does not report local runner close healthy with %s durable evidence", - async (mode) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-local-close-unproven-"), - ); - const identity = { - runnerInstanceId: "runner-close-unproven", - environmentLeaseId: "lease-close-unproven", - runId: "run-close-unproven", - normalizedSessionId: "session-close-unproven", - turnId: "turn-close-unproven", - itemId: "item-close-unproven", - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - closeGraceMs: 400, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: identity, - readRunnerState: async () => ({ - schema: "paperclip.runner.durable.state.v1", - ...identity, - ...(mode === "wrong_identity" ? { runId: "some-other-run" } : {}), - lifecycle: mode === "not_suspended" ? "ready" : "suspended", - }), - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - await expect(bundle.transport.close()).rejects.toThrow( - "runner did not durably suspend before checkpoint", - ); - const control = JSON.parse( - await readFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - "utf8", - ), - ); - expect(control.identity).toEqual(identity); - expect(await readdir(stateDirectory)).toContain("runner"); - } finally { - await bundle.transport.close().catch(() => undefined); - await rm(stateDirectory, { recursive: true, force: true }); - } - }, - 10_000, -); - -it.each([ - { - label: "interrupted", - args: ["--hold-turn"], - terminal: "turn.interrupted", - disposition: "needs_review", - }, - { - label: "failed", - args: ["--fail-turn-immediately"], - terminal: "turn.failed", - disposition: "needs_review", - }, - { - label: "completed", - args: [], - terminal: "turn.completed", - disposition: "done", - }, -])("retains a $label runner's result before closing its provider turn", async ({ - label, args, terminal, disposition, -}) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-stop-result-")); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, ...args), - stateDirectory, - }); - const driver = new CodexAppServerDriver({ - taskEnvelope: createCodexTaskEnvelope({ - objective: "Stop and retain unfinished work.", - }), - environment: { - PATH: process.env.PATH, - HOME: join(tmpdir(), "runnerd-stop-host-home"), - PAPERCLIP_WORKSPACE_CWD: stateDirectory, - }, - approvalPolicy: "never", - transportFactory: () => bundle.transport, - }); - const session = await driver.openSession({ - runId: "run-stop-result", - normalizedSessionId: "session-stop-result", - workingDirectory: stateDirectory, - }); - try { - const turn = await session.startTurn({ - message: { role: "user", text: "Keep working until interrupted." }, - }); - if (label === "interrupted") { - await session.interrupt({ - turnId: turn.turnId, - reason: "Stopped by the user", - }); - } - const events: PrpEvent[] = []; - for await (const event of session.events()) { - events.push(event); - if (event.eventType === "session.failed" || event.eventType === terminal) { - break; - } - } - // Let already committed durable suffix events reach the facade as well. - await bundle.transport.request("thread/read", {}); - const snapshot = await session.snapshot(); - expect(events.some((event) => event.eventType === "session.failed")).toBe(false); - expect(events.map((event) => event.eventType)).toContain("run.result.proposed"); - expect(events.at(-1)?.eventType).toBe(terminal); - expect(snapshot).toMatchObject({ - activeTurnId: null, - semanticResult: { - turnId: turn.turnId, - result: { reportedWorkDisposition: disposition }, - }, - }); - } finally { - await session.close().catch(() => undefined); - await rm(stateDirectory, { - recursive: true, - force: true, - maxRetries: 5, - retryDelay: 25, - }); - } -}, 30_000); - it("binds an immediately failed durable turn before exposing its terminal", async () => { const stateDirectory = await mkdtemp( join(tmpdir(), "runnerd-fast-terminal-"), @@ -5908,3406 +5270,6 @@ it("steers the active provider turn through the durable PRP command path", async } }, 30_000); -it.each([ - ["held-ack", "normal"], - ["lost-ack", "normal"], - ["rejected-attach", "normal"], - ["lost-ack", "after-activation"], -] as const)( - "preserves old warm-attach authority and event ownership across %s (%s observer)", - async (mode, observer) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-ack-")); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const effects = new Map(); - const handles: ReturnType[] = []; - let armed = false; - let heldEvent: PrpEvent | null = null; - let releaseCommit!: () => void; - let enteredCommit!: () => void; - const commitGate = new Promise((resolveCommit) => { - releaseCommit = resolveCommit; - }); - const commitEntered = new Promise((resolveEntered) => { - enteredCommit = resolveEntered; - }); - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore({ - ...options, - onCommittedEvent: async (event) => { - await options.onCommittedEvent?.(event); - const prior = effects.get(event.sourceEventId); - if (prior) { - expect(event).toEqual(prior.event); - prior.deliveries += 1; - } else { - effects.set(event.sourceEventId, { - event: structuredClone(event), - deliveries: 1, - }); - } - if ( - armed && - mode !== "rejected-attach" && - heldEvent === null && - event.eventType === "run.attached" - ) { - heldEvent = structuredClone(event); - enteredCommit(); - await commitGate; - if (mode === "lost-ack") { - // The external durable effect exists, but this connection - // disappears before its local cursor/ACK can be published. - throw new Error( - "fixture lost the old authority ACK after commit", - ); - } - } - }, - }); - cores.push(core); - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch(options); - handles.push(handle); - return handle; - }); - const within = async ( - label: string, - promise: Promise, - timeout = 5_000, - ) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolveWait, rejectWait) => { - timer = setTimeout( - () => rejectWait(new Error(`${label} timeout`)), - timeout, - ); - }), - ]); - } finally { - if (timer !== undefined) clearTimeout(timer); - } - }; - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 5_000, - }); - const readRunner = async () => - JSON.parse( - await readFile( - join(stateDirectory, "runner/runner-state.json"), - "utf8", - ), - ) as { - runId: string; - ackedSourceSeq: number; - processedCommands: Record; - outbox: { envelope: { payload: PrpEvent } }[]; - }; - let providerPid: number | null = null; - let primaryError: unknown; - let cleanupProven = false; - try { - const opened = await within( - "initial thread", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ) as { thread: { id: string } }; - const core = cores[0]!; - expect(cores).toHaveLength(1); - const oldIdentity = structuredClone(core.store.state.identity); - const runnerPid = bundle.evidence().runnerPid; - providerPid = bundle.evidence().codexPid; - const rotations: (typeof core.store.state)[] = []; - const preparedStates: (typeof core.store.state)[] = []; - const commit = core.store.commit.bind(core.store); - vi.spyOn(core.store, "commit").mockImplementation((candidate) => { - commit(candidate); - // Capture the durable old epoch at publication. The authenticated - // successor may activate before attachRun observes the completed command. - if ( - candidate.identity.runId === oldIdentity.runId && - candidate.warmTransition?.phase === "prepared" - ) { - preparedStates.push(structuredClone(core.store.state)); - } - }); - if (observer === "after-activation") { - const getCommand = core.getCommand.bind(core); - vi.spyOn(core, "getCommand").mockImplementation((commandId) => { - const command = getCommand(commandId); - if ( - command?.type === "run.attach" && - command.status === "completed" && - core.store.state.completedWarmTransition === undefined - ) { - // Deterministically exercise an observer that misses the old epoch. - return { ...command, status: "pending", result: null }; - } - return command; - }); - } - const rotate = core.rotateRunIdentity.bind(core); - vi.spyOn(core, "rotateRunIdentity").mockImplementation( - (identity, template) => { - rotations.push(structuredClone(core.store.state)); - return rotate(identity, template); - }, - ); - if (mode === "rejected-attach") { - const queue = core.queueCommand.bind(core); - vi.spyOn(core, "queueCommand").mockImplementation( - (type, payload = {}, id, immediate) => - queue( - type, - type === "run.attach" - ? { - ...payload, - provider: { - ...(payload.provider as Record), - model: "foreign-profile", - }, - } - : payload, - id, - immediate, - ), - ); - } - armed = true; - const attachment = bundle.transport.attachRun!({ - runId: "run-warm-ack-next", - turnId: "turn-warm-ack-next", - itemId: "item-warm-ack-next", - }); - void attachment.catch(() => undefined); - if (mode === "rejected-attach") { - await expect(within("rejected attach", attachment)).rejects.toThrow( - "run.attach cannot change the durable Codex provider profile", - ); - expect(rotations).toHaveLength(0); - expect(core.store.state.identity).toEqual(oldIdentity); - expect((await readRunner()).runId).toBe(oldIdentity.runId); - const read = await within( - "read under unchanged authority", - bundle.transport.request("thread/read", {}), - ); - expect((read.thread as { id: string }).id).toBe(opened.thread.id); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("failed"); - } else { - await within("old authority commit barrier", commitEntered); - const retained = await readRunner(); - expect(retained.runId).toBe(oldIdentity.runId); - expect( - Object.values(retained.processedCommands).find((entry) => entry.commandType === "run.attach") - ?.status, - ).toBe("completed"); - expect( - retained.outbox.some( - (entry) => - entry.envelope.payload.sourceEventId === heldEvent!.sourceEventId, - ), - ).toBe(true); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("pending"); - expect(rotations).toHaveLength(0); - if (mode === "lost-ack") core.disconnectActiveRunner(); - releaseCommit(); - await within("warm attach after old ACK", attachment, 10_000); - expect(rotations).toHaveLength(1); - expect(preparedStates.length).toBeGreaterThan(0); - const retired = preparedStates[0]!; - if (observer === "after-activation") { - expect(rotations[0]!.identity.runId).toBe("run-warm-ack-next"); - expect( - rotations[0]!.committedEvents.some( - (entry) => entry.sourceEventId === heldEvent!.sourceEventId, - ), - ).toBe(false); - } - const attachedEvent = retired.committedEvents.find( - (entry) => entry.sourceEventId === heldEvent!.sourceEventId, - )!; - expect(attachedEvent.logicalEffectCount).toBe(1); - expect(retired.ackedSourceSeq).toBeGreaterThanOrEqual( - attachedEvent.sourceSeq, - ); - expect( - retired.committedEvents.slice(-4).map((entry) => entry.eventType), - ).toEqual([ - "session.resumed", - "session.capabilities.updated", - "session.goal.snapshot", - "run.attached", - ]); - expect( - retired.committedEvents.every( - (entry) => entry.envelope.runId === oldIdentity.runId, - ), - ).toBe(true); - if (mode === "lost-ack") { - expect(retired.connectionCount).toBeGreaterThanOrEqual(2); - expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(2); - } else { - expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(1); - } - await vi.waitFor(async () => - expect((await readRunner()).runId).toBe("run-warm-ack-next"), - ); - const read = await within( - "read under new authority", - bundle.transport.request("thread/read", {}), - ); - expect((read.thread as { id: string }).id).toBe(opened.thread.id); - } - expect(bundle.evidence()).toMatchObject({ - runnerPid, - codexPid: providerPid, - runnerExited: false, - }); - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - } catch (error) { - primaryError = error; - throw error; - } finally { - releaseCommit(); - try { - await within( - "warm fixture close", - bundle.transport.close(), - 10_000, - ).catch(() => undefined); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within("exact warm fixture runner exit", handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - if (providerPid && !dead(-providerPid)) - process.kill(-providerPid, "SIGKILL"); - if (providerPid) - await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); - cleanupProven = true; - } catch (error) { - if (primaryError === undefined) throw error; - console.error( - "Warm ACK fixture cleanup unproven; retaining its private state directory.", - ); - } finally { - try { - for (const core of cores) await core.stop().catch(() => undefined); - } finally { - launchSpy.mockRestore(); - coreSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - } - } - } - }, - 30_000, -); - -it.each([false, true])( - "retains warm attach authority when its result is lost before controller persistence (held observer=%s)", - async (holdObserver) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-attach-result-loss-"), - ); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const handles: ReturnType[] = []; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore(options); - cores.push(core); - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch(options); - handles.push(handle); - return handle; - }); - const within = async ( - label: string, - promise: Promise, - timeout = 5_000, - ) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new Error(`${label} timeout`)), - timeout, - ); - }), - ]); - } finally { - if (timer !== undefined) clearTimeout(timer); - } - }; - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: - process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 2_000, - }); - let providerPid: number | null = null; - let cleanupProven = false; - let saveSpy: ReturnType | undefined; - let observerSpy: { mockRestore(): void } | undefined; - let attachment: Promise | undefined; - try { - await within( - "initial thread", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ); - expect(cores).toHaveLength(1); - const core = cores[0]!; - const oldIdentity = structuredClone(core.store.state.identity); - if (holdObserver) { - const getCommand = core.getCommand.bind(core); - observerSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((commandId) => { - const command = getCommand(commandId); - if ( - command?.type === "run.attach" && - core.store.state.completedWarmTransition?.command.commandId !== - commandId - ) { - return { ...command, status: "pending", result: null }; - } - return command; - }); - } - providerPid = bundle.evidence().codexPid; - const store = core.store as typeof core.store & { - commit(candidate: typeof core.store.state): void; - }; - const commit = store.commit.bind(store); - let lostResult = false; - let observeLoss!: () => void; - const loss = new Promise((resolveLoss) => { - observeLoss = resolveLoss; - }); - saveSpy = vi.spyOn(store, "commit").mockImplementation((candidate) => { - const attach = candidate.commands.find( - (entry) => entry.type === "run.attach", - ); - if (!lostResult && attach?.status === "completed") { - lostResult = true; - // The authenticated result reached the receiver, but the durable write - // did not. The clone has not been exposed in memory or on disk. - const persisted = JSON.parse(readFileSync(core.store.path, "utf8")); - expect(persisted.identity).toEqual(oldIdentity); - expect( - persisted.commands.find( - (entry: { type: string }) => entry.type === "run.attach", - ).status, - ).toBe("pending"); - core.disconnectActiveRunner(); - observeLoss(); - throw new Error( - "fixture lost attach result before durable controller commit", - ); - } - commit(candidate); - }); - attachment = bundle.transport.attachRun!({ - runId: "run-result-loss-next", - turnId: "turn-result-loss-next", - itemId: "item-result-loss-next", - }); - void attachment.catch(() => undefined); - await within("lost attach result", loss); - expect(lostResult).toBe(true); - expect(core.store.state.identity).toEqual(oldIdentity); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("pending"); - const outcome = await within( - "exact result replay after reconnect", - attachment.then( - () => ({ status: "completed" as const }), - (error: unknown) => ({ - status: "failed" as const, - message: String(error), - }), - ), - 10_000, - ); - const runner = JSON.parse( - await readFile( - join(stateDirectory, "runner/runner-state.json"), - "utf8", - ), - ); - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - expect( - { - outcome, - runnerRunId: runner.runId, - controllerRunId: core.store.state.identity.runId, - }, - "lost attach result must replay without leaving the two durable authorities split", - ).toEqual({ - outcome: { status: "completed" }, - runnerRunId: "run-result-loss-next", - controllerRunId: "run-result-loss-next", - }); - if (holdObserver) - expect( - core.store.state.completedWarmTransition?.receipt.newIdentity.runId, - ).toBe("run-result-loss-next"); - } finally { - saveSpy?.mockRestore(); - observerSpy?.mockRestore(); - try { - await within( - "result-loss fixture close", - bundle.transport.close(), - 5_000, - ).catch(() => undefined); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within("exact result-loss runner exit", handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - if (providerPid && !dead(-providerPid)) - process.kill(-providerPid, "SIGKILL"); - if (providerPid) - await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); - cleanupProven = true; - } finally { - for (const core of cores) await core.stop().catch(() => undefined); - launchSpy.mockRestore(); - coreSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - } - } - }, - 30_000, -); - -it.each([ - ...[ - "before-result", - "after-result", - "after-activation", - "before-confirmation", - "after-confirmation", - ].flatMap((lossPoint) => - [false, true].map((routed) => ({ - lossPoint, - routed, - recoveryFault: "none", - })), - ), - ...["endpoint", "missing-capability", "listen", "malformed-core"].map( - (recoveryFault) => ({ - lossPoint: "after-result", - routed: true, - recoveryFault, - }), - ), - ...["before_bootstrap", "before_spawn", "before_authentication"].map( - (stage) => ({ - lossPoint: "after-result", - routed: true, - recoveryFault: `authorize_${stage}`, - }), - ), - ...[ - "attach-wait", - ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER - ? ["attach-capability"] - : []), - ].map((recoveryFault) => ({ - lossPoint: "before-result", - routed: true, - recoveryFault, - })), - ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER - ? [ - { - lossPoint: "after-result", - routed: false, - recoveryFault: "legacy-parser", - }, - ] - : []), - ...["before-result", "after-result", "after-activation", "after-confirmation"].map(lossPoint => ({ - lossPoint, routed: true, recoveryFault: "none", executionPrepared: true, - })), - { - lossPoint: "after-result", - routed: true, - recoveryFault: "none", - externalArtifact: true, - }, - { - lossPoint: "after-confirmation", - routed: true, - recoveryFault: "none", - ordinaryFollowup: true, - }, - ...[ - "missing_snapshot", - "rejected_snapshot", - "wrong_thread", - "callback_failure", - "callback_async_failure", - ].map((ordinaryFollowup) => ({ - lossPoint: "after-confirmation", - routed: true, - recoveryFault: "none", - ordinaryFollowup, - })), -])( - "recovers a warm attachment with a fresh controller and runner ($lossPoint, routed=$routed, fault=$recoveryFault, followup=$ordinaryFollowup, external=$externalArtifact, prepared=$executionPrepared)", - async (testCase) => { - const { lossPoint, routed, recoveryFault } = testCase; - const externalArtifact = "externalArtifact" in testCase && testCase.externalArtifact; - const executionPrepared = "executionPrepared" in testCase && testCase.executionPrepared; - const configurations = new Map(); - const ordinaryFollowup = - "ordinaryFollowup" in testCase && testCase.ordinaryFollowup; - const snapshotFault = - typeof ordinaryFollowup === "string" && - !ordinaryFollowup.startsWith("callback_"); - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-attach-restart-"), - ); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const handles: ReturnType[] = []; - const routes = new Map< - string, - { core: DurablePrpControlPlane; generation: symbol } - >(); - const routeCalls: string[] = []; - let recovering = false; - let recoveryClaimCurrent = true; - let recoveryFenceActive = true; - const completionSnapshotIds: string[] = []; - let rejectHeldCompletion: ((error: Error) => void) | undefined; - let markHeldCompletionEntered!: () => void; - const heldCompletionEntered = new Promise((resolve) => { - markHeldCompletionEntered = resolve; - }); - let snapshotObserverSpy: ReturnType | undefined; - const routeServer = createServer((_request, response) => - response.writeHead(404).end(), - ); - routeServer.on("upgrade", (request, socket, head) => { - const entry = routes.get(request.url ?? ""); - if (!entry) { - socket.destroy(); - return; - } - entry.core.handleUpgrade(request, socket, request.url!, head); - }); - if (routed) - await new Promise((resolveListen) => - routeServer.listen(0, "127.0.0.1", resolveListen), - ); - const routeAddress = routeServer.address(); - const routePort = - routeAddress && typeof routeAddress === "object" ? routeAddress.port : 0; - const registration = async ( - core: DurablePrpControlPlane, - identity = core.store.state.identity, - ) => { - if ( - recovering && - ordinaryFollowup && - recoveryFenceActive && - !recoveryClaimCurrent - ) { - throw new Error("fixture old recovery claim is no longer current"); - } - const path = `/api/runner/v1/connect/${identity.runId}`; - const generation = Symbol(); - routes.set(path, { core, generation }); - routeCalls.push(identity.runId); - return { - connection: - recovering && recoveryFault === "listen" - ? { - mode: "listen" as const, - listenAddress: "0.0.0.0" as const, - listenPort: routePort, - listenPath: path, - } - : { - mode: "connect" as const, - connectUrl: `ws://127.0.0.1:${routePort}${path}${recovering && recoveryFault === "endpoint" ? "/changed" : ""}`, - }, - release: () => { - if (routes.get(path)?.generation === generation) routes.delete(path); - }, - }; - }; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore(options); - cores.push(core); - if (recovering && snapshotFault) { - const getCommand = core.getCommand.bind(core); - snapshotObserverSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((id) => { - const command = getCommand(id); - if ( - command?.type !== "session.snapshot" || - command.status !== "completed" - ) - return command; - const observed = structuredClone(command); - if (ordinaryFollowup === "rejected_snapshot") { - observed.status = "failed"; - observed.result = { - result: { message: "fixture snapshot observation rejected" }, - }; - } else if (ordinaryFollowup === "wrong_thread") { - observed.result = { - ...observed.result, - result: { - ...(observed.result?.result as Record), - driverSessionId: "foreign-thread", - providerSessionId: "foreign-thread", - }, - }; - } else observed.result = { result: { status: "session_open" } }; - return observed; - }); - } - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch({ ...options, - ...(executionPrepared ? { processLauncher: undefined } : {}), - ...(externalArtifact && recovering ? { runnerBinaryPath: defaultCapabilityRunnerdBinary() } : {}), - }); - handles.push(handle); - return handle; - }); - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const stopOwnedProvider = async (pid: number) => { - // Exact runner completion can race the OS reaping its already-signalled - // provider. Absence, not the outcome of a redundant signal, is required. - try { - await vi.waitFor(() => expect(dead(-pid)).toBe(true), { - timeout: 500, - interval: 10, - }); - } catch { - try { - process.kill(-pid, "SIGKILL"); - } catch (error) { - if ( - !["ESRCH", "EPERM"].includes( - String((error as NodeJS.ErrnoException).code), - ) - ) - throw error; - } - await vi.waitFor(() => expect(dead(-pid)).toBe(true), { - timeout: 2_000, - interval: 10, - }); - } - expect(dead(pid)).toBe(true); - }; - const fingerprintTree = async (root: string): Promise => { - const rows: unknown[] = []; - const visit = async (path: string, relative: string) => { - const metadata = await lstat(path); - rows.push({ - path: relative, - inode: metadata.ino, - mode: metadata.mode, - mtimeMs: metadata.mtimeMs, - digest: metadata.isFile() - ? createHash("sha256") - .update(await readFile(path)) - .digest("hex") - : metadata.isSymbolicLink() - ? createHash("sha256") - .update(await readlink(path)) - .digest("hex") - : null, - }); - if (metadata.isDirectory()) - for (const child of (await readdir(path)).sort()) - await visit(join(path, child), `${relative}/${child}`); - }; - await visit(root, "."); - return rows; - }; - const within = async (promise: Promise, timeout = 5_000) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new Error("warm restart fixture timed out")), - timeout, - ); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } - }; - const options = { - runnerBinary: - recoveryFault === "attach-capability" - ? process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER! - : (process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - defaultCapabilityRunnerdBinary()), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 2_000, - prpIdentity: (() => { - const runId = randomUUID(); - return { - runnerInstanceId: randomUUID(), - environmentLeaseId: - process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_SCOPE === - "transient" - ? runId - : randomUUID(), - runId, - normalizedSessionId: randomUUID(), - turnId: `turn-${runId}`, - itemId: `item-${runId}`, - }; - })(), - ...(routed - ? { - controlPlaneRegistration: registration, - warmTransitionRegistrationMode: "routed_connect" as const, - } - : {}), - ...(executionPrepared ? { - runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), - readRunnerState: async () => JSON.parse(await readFile(join(stateDirectory, "runner/runner-state.json"), "utf8")), - // The fixture's spawn spy supplies the real local process owner. - runnerProcessLauncher: () => { throw new Error("fixture process owner was bypassed"); }, - prepareLaunch: async (requirements: RunnerProviderLaunchRequirements, recovery?: PreparedRunnerProviderLaunch) => { - const existing = configurations.get(requirements.identity.runId); - if (existing) { - expect(requirements).toEqual(existing.requirements); - if (recovering) expect(recovery).toEqual(existing.prepared); - return existing.prepared; - } - const prepared = { configurationId: `prepared-${requirements.identity.runId}`, - requirementsDigest: runnerLaunchRequirementsDigest(requirements), - provider: createRunnerProviderLaunch(requirements, { - options: { codexArgs: fakeCodexArgs(stateDirectory, "--call-log", callsPath, "--record-process-start") }, - root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), - codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "", - }), - }; - configurations.set(requirements.identity.runId, { requirements: structuredClone(requirements), prepared }); - return prepared; - }, - } : {}), - }; - const first = createCapabilityRunnerdCodexTransport(options); - let resumed: - ReturnType | undefined; - const providerPids = new Set(); - let commitSpy: ReturnType | undefined; - let commandObserverSpy: ReturnType | undefined; - let detached: Promise | undefined; - let cleanupProven = false; - const legacyProbeDirectories: string[] = []; - try { - const opened = await within( - first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ); - const openedThread = opened.thread as { id: string; sessionId: string }; - const thread = { id: openedThread.id, sessionId: openedThread.sessionId }; - const firstThreadSnapshot = await within( - first.transport.request("thread/read", {}), - ); - const firstEvidence = first.evidence(); - const cleanRunnerStateBytes = await readFile( - join(stateDirectory, "runner/runner-state.json"), - ); - const runnerProcessStartedAt = - process.platform === "darwin" - ? new Date( - execFileSync( - "ps", - ["-o", "lstart=", "-p", String(handles[0]!.child.pid)], - { encoding: "utf8", timeout: 1_500 }, - ).trim(), - ).toISOString() - : null; - if ((first.evidence().codexPid ?? 0) > 0) - providerPids.add(first.evidence().codexPid!); - const core = cores[0]!; - const oldIdentity = structuredClone(core.store.state.identity); - const nextRunId = randomUUID(); - const desired = { - ...oldIdentity, - runId: nextRunId, - turnId: `turn-${nextRunId}`, - itemId: `item-${nextRunId}`, - }; - if ( - recoveryFault === "attach-capability" || - recoveryFault === "attach-wait" - ) { - if (recoveryFault === "attach-wait") { - const getCommand = core.getCommand.bind(core); - commandObserverSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((id) => { - if ( - core.store.state.commands.find( - (entry) => entry.commandId === id, - )?.type === "run.attach" - ) { - throw new Error("fixture result observer unavailable"); - } - return getCommand(id); - }); - } - const bootstrapCount = core.store.state.freshBootstraps; - await expect( - first.transport.attachRun!({ - runId: desired.runId, - turnId: desired.turnId, - itemId: desired.itemId, - }), - ).rejects.toThrow( - recoveryFault === "attach-capability" - ? "capability is required" - : "result observer unavailable", - ); - expect(routeCalls).toEqual([oldIdentity.runId, desired.runId]); - expect([...routes.keys()]).toEqual([ - `/api/runner/v1/connect/${oldIdentity.runId}`, - ]); - expect(core.store.state.freshBootstraps).toBe(bootstrapCount); - expect(handles).toHaveLength(1); - if (recoveryFault === "attach-capability") - expect( - core.store.state.commands.some( - (entry) => entry.type === "run.attach", - ), - ).toBe(false); - return; - } - const commit = core.store.commit.bind(core.store); - let lossObserved = false; - let signalLoss!: () => void; - const loss = new Promise((resolveLoss) => { - signalLoss = resolveLoss; - }); - commitSpy = vi - .spyOn(core.store, "commit") - .mockImplementation((candidate) => { - // The selected process-crash boundary stays unavailable until the - // exact owned runner is joined. A later replay must not silently - // settle this fixture before its retained pair is exported. - if (lossObserved) - throw new Error( - "fixture controller persistence unavailable after loss", - ); - const phase = candidate.warmTransition?.phase; - const target = - lossPoint === "after-activation" ? "activated" : "prepared"; - const atBoundary = lossPoint.includes("confirmation") - ? candidate.completedWarmTransition !== undefined && - candidate.warmTransition === undefined - : phase === target; - if (!lossObserved && atBoundary) { - lossObserved = true; - if ( - lossPoint !== "before-result" && - lossPoint !== "before-confirmation" - ) - commit(candidate); - // A dead controller cannot accept a reconnect through the second - // route installed for the in-flight authority rotation. Detaching - // only the current route leaves that overlapping fixture listener - // alive; completed-receipt replay needs no further store commit. - for (const [path, entry] of routes) { - if (entry.core === core) routes.delete(path); - } - core.disconnectActiveRunner(); - detached = first.transport.detachControllerForRestart!(); - signalLoss(); - // A committed confirmation is still before its ACK. Returning from - // this hook lets the current frame handler send that ACK even after - // close begins, collapsing the intended crash window on fast peers. - throw new Error("fixture interrupted exact result commit"); - } - commit(candidate); - }); - const attachment = first.transport.attachRun!({ - runId: desired.runId, - turnId: desired.turnId, - itemId: desired.itemId, - }); - void attachment.catch(() => undefined); - await within(loss); - await within(detached!); - expect(lossObserved).toBe(true); - const runnerPath = join(stateDirectory, "runner/runner-state.json"); - const runner = JSON.parse(await readFile(runnerPath, "utf8")); - expect(runner.warmTransition.phase).toBe( - lossPoint === "after-activation" || lossPoint.includes("confirmation") - ? "activating" - : "prepared", - ); - const persisted = JSON.parse(await readFile(core.store.path, "utf8")); - expect( - persisted.commands.find( - (entry: { type: string }) => entry.type === "run.attach", - )?.status, - ).toBe( - lossPoint === "before-result" - ? "pending" - : lossPoint === "after-result" - ? "completed" - : undefined, - ); - // Stop only this fixture's exact owned process handles. No stored receipt - // or PID absence is used as authority to terminate an unknown owner. - await durableControlPlane - .waitForProcess(handles[0]!, 100) - .catch(() => undefined); - await within(handles[0]!.completion); - for (const pid of providerPids) { - await stopOwnedProvider(pid); - } - await within(attachment.catch(() => undefined)); - const joinedRunner = JSON.parse(await readFile(runnerPath, "utf8")); - const joinedCore = JSON.parse(await readFile(core.store.path, "utf8")); - expect(joinedRunner.warmTransition).toEqual(runner.warmTransition); - expect(joinedCore.schema).toBe(persisted.schema); - expect(joinedCore.warmTransition).toEqual(persisted.warmTransition); - expect(joinedCore.completedWarmTransition).toEqual( - persisted.completedWarmTransition, - ); - commitSpy.mockRestore(); - const fixtureOutput = - process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_DIRECTORY; - if (fixtureOutput && recoveryFault === "none") { - const retainedArtifact = join(fixtureOutput, "paperclip-runnerd"); - await cp(options.runnerBinary, retainedArtifact, { force: false }); - expect( - createHash("sha256") - .update(await readFile(retainedArtifact)) - .digest("hex"), - ).toBe( - createHash("sha256") - .update(await readFile(options.runnerBinary)) - .digest("hex"), - ); - const retained = join( - fixtureOutput, - `${lossPoint}-${routed ? "routed" : "local"}`, - ); - await cp(stateDirectory, retained, { - recursive: true, - errorOnExist: true, - force: false, - }); - await writeFile( - join(retained, "transition-fixture-metadata.json"), - JSON.stringify( - { - schema: "paperclip.test.warm-transition-fixture.v1", - oldIdentity, - newIdentity: desired, - lossPoint, - routed, - runner: { - pid: handles[0]!.child.pid, - processGroupId: handles[0]!.processGroupId, - startedAt: runnerProcessStartedAt, - spawnObservedAt: handles[0]!.startedAt, - completion: await handles[0]!.completion, - processAbsent: dead(handles[0]!.child.pid!), - groupAbsent: dead(-handles[0]!.processGroupId!), - }, - providers: [...providerPids].map((pid) => ({ - pid, - processGroupId: pid, - startedAt: firstEvidence.providerProcessStartedAt, - processAbsent: dead(pid), - groupAbsent: dead(-pid), - })), - artifact: { - path: retainedArtifact, - version: runner.warmTransition.receipt.runnerVersion, - digest: runner.warmTransition.receipt.runnerDigest, - }, - thread, - firstThreadSnapshot, - firstEvidence, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - } - routes.clear(); - const routeCallCount = routeCalls.length; - const beforeCalls = (await readFile(callsPath, "utf8")) - .trim() - .split(/\r?\n/); - expect( - beforeCalls.filter((call) => call === "process-start"), - ).toHaveLength(1); - if (recoveryFault === "legacy-parser") { - const legacyBinary = - process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER!; - const legacyDigest = `sha256:${createHash("sha256") - .update(await readFile(legacyBinary)) - .digest("hex")}`; - for (const [mode, bytes] of [ - ["pending", await readFile(runnerPath)], - ["clean", cleanRunnerStateBytes], - ] as const) { - const probe = await mkdtemp( - join(tmpdir(), "runnerd-legacy-schema-probe-"), - ); - legacyProbeDirectories.push(probe); - await writeFile(join(probe, "runner-state.json"), bytes, { - mode: 0o600, - }); - const handle = durableControlPlane.spawnRunner({ - connection: runner.warmTransition.receipt.connection, - stateDirectory: probe, - identity: oldIdentity, - runnerBinaryPath: legacyBinary, - runnerVersion: "0.3.0", - runnerDigest: legacyDigest, - ticket: "bootstrap_legacy_parser_probe", - maxOutboxBytes: runner.maxOutboxBytes, - p0ReserveBytes: runner.p0ReserveBytes, - maxRuntimeMs: 200, - reconnectGraceMs: 200, - }); - const result = await within(handle.completion, 5_000); - expect(result.code).not.toBe(0); - if (mode === "pending") { - expect(result.stderr).toContain( - "durable state binding does not match", - ); - expect(await readFile(join(probe, "runner-state.json"))).toEqual( - bytes, - ); - } else { - expect(result.stderr).not.toContain( - "durable state binding does not match", - ); - expect( - JSON.parse( - await readFile(join(probe, "runner-state.json"), "utf8"), - ).nextSourceSeq, - ).toBeGreaterThan(JSON.parse(bytes.toString("utf8")).nextSourceSeq); - } - } - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - return; - } - recovering = true; - if (recoveryFault === "malformed-core") - await writeFile(core.store.path, "{", { mode: 0o600 }); - const beforeRecoveryBytes = await Promise.all( - [core.store.path, runnerPath].map((path) => readFile(path)), - ); - const beforeRecoveryTree = - recoveryFault === "none" ? null : await fingerprintTree(stateDirectory); - const authorizationStages: string[] = []; - let releaseHeldAuthorization: (() => void) | undefined; - let markHeldAuthorizationEntered!: () => void; - const heldAuthorizationEntered = new Promise((resolve) => { - markHeldAuthorizationEntered = resolve; - }); - const authorizationFault = recoveryFault.startsWith("authorize_"); - const failedAuthorizationStage = recoveryFault.slice("authorize_".length); - const beforeProviderBytes = await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - ); - resumed = createCapabilityRunnerdCodexTransport({ - ...options, - ...(externalArtifact ? { - runnerBinary: "/controller-missing-runner", - runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), - } : {}), - ...(ordinaryFollowup - ? { - authorizeWarmTransitionRecovery: async () => { - if (!recoveryClaimCurrent) - throw new Error( - "fixture old recovery claim is no longer current", - ); - }, - onWarmTransitionRecoveryCompleted: (completion: { - transitionId: string; - }) => { - expect(completion.transitionId).toBe( - runner.warmTransition.receipt.transitionId, - ); - const completedSnapshot = cores[1]!.store.state.commands - .filter((command) => command.type === "session.snapshot") - .at(-1)!; - expect(completedSnapshot.status).toBe("completed"); - expect(cores[1]!.store.state.identity).toEqual(desired); - expect(cores[1]!.store.state.warmTransition).toBeUndefined(); - completionSnapshotIds.push(completedSnapshot.commandId); - if ( - ordinaryFollowup === "callback_failure" && - completionSnapshotIds.length === 1 - ) { - throw new Error( - "fixture recovery completion callback failed", - ); - } - if ( - ordinaryFollowup === "callback_async_failure" && - completionSnapshotIds.length === 1 - ) { - return new Promise((_resolve, reject) => { - rejectHeldCompletion = reject; - markHeldCompletionEntered(); - }); - } - recoveryFenceActive = false; - }, - } - : {}), - ...(authorizationFault - ? { - authorizeWarmTransitionRecovery: async (stage: string) => { - authorizationStages.push(stage); - if ( - stage === "before_bootstrap" && - failedAuthorizationStage === "before_bootstrap" - ) { - await new Promise((resolveGate) => { - releaseHeldAuthorization = resolveGate; - markHeldAuthorizationEntered(); - }); - } - if (stage === failedAuthorizationStage) - throw new Error("fixture recovery authority revoked"); - }, - } - : {}), - ...(recoveryFault === "missing-capability" - ? { warmTransitionRegistrationMode: undefined } - : {}), - ...(recoveryFault !== "none" - ? { - environment: { - CODEX_API_KEY: "synthetic-refused-route-credential", - }, - } - : {}), - prpIdentity: desired, - resumeProviderSession: { - driverSessionId: thread.id, - providerSessionId: thread.sessionId, - }, - }); - if (recoveryFault !== "none") { - const recoveryRequest = within( - resumed.transport.request("thread/read", {}), - ); - void recoveryRequest.catch(() => undefined); - if ( - authorizationFault && - failedAuthorizationStage === "before_bootstrap" - ) { - await within(Promise.race([ - heldAuthorizationEntered, - recoveryRequest.then(() => { - throw new Error("warm restart authorization callback was not entered"); - }), - ])); - expect(releaseHeldAuthorization).toBeTypeOf("function"); - const queuedBefore = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - try { - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must not queue before bootstrap" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.attachRun!({ - runId: "must-not-attach", - turnId: "must-not-attach", - itemId: "must-not-attach", - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.resolveRuntimeRequest!({ - requestId: "must-not-resolve", - turnId: desired.turnId, - resolution: { action: "cancel" }, - }), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map( - (command) => command.commandId, - ), - ).toEqual(queuedBefore); - expect(handles).toHaveLength(1); - } finally { - releaseHeldAuthorization!(); - } - } - const failedRecovery = expect(recoveryRequest).rejects; - if (authorizationFault) { - await failedRecovery.toThrow( - failedAuthorizationStage === "before_authentication" - ? "native_runner_warm_transition_recovery_pending" - : "fixture recovery authority revoked", - ); - const expectedStages = [ - "before_bootstrap", - "before_spawn", - "before_authentication", - ]; - expect([...new Set(authorizationStages)]).toEqual( - expectedStages.slice( - 0, - expectedStages.indexOf(failedAuthorizationStage) + 1, - ), - ); - expect(handles).toHaveLength( - failedAuthorizationStage === "before_authentication" ? 2 : 1, - ); - expect( - await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - ), - ).toEqual(beforeProviderBytes); - const refusedRunner = JSON.parse(await readFile(runnerPath, "utf8")); - expect(refusedRunner.warmTransition).toEqual(runner.warmTransition); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - if (failedAuthorizationStage !== "before_authentication") - expect(await readFile(runnerPath)).toEqual(beforeRecoveryBytes[1]); - if (failedAuthorizationStage === "before_bootstrap") - expect(await readFile(core.store.path)).toEqual( - beforeRecoveryBytes[0], - ); - await within(resumed.transport.close()).catch(() => undefined); - expect([...routes.keys()]).toEqual([]); - return; - } - if (recoveryFault === "malformed-core") await failedRecovery.toThrow(); - else - await failedRecovery.toThrow( - recoveryFault === "missing-capability" - ? "requires_exact_owned_endpoint" - : "registered_endpoint_mismatch", - ); - expect(handles).toHaveLength(1); - expect( - await Promise.all( - [core.store.path, runnerPath].map((path) => readFile(path)), - ), - ).toEqual(beforeRecoveryBytes); - expect(await fingerprintTree(stateDirectory)).toEqual( - beforeRecoveryTree, - ); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - expect([...routes.keys()]).toEqual([]); - return; - } - if ( - snapshotFault || - ordinaryFollowup === "callback_failure" || - ordinaryFollowup === "callback_async_failure" - ) { - const firstRead = within(resumed.transport.request("thread/read", {})); - void firstRead.catch(() => undefined); - if (ordinaryFollowup === "callback_async_failure") { - await within(Promise.race([ - heldCompletionEntered, - firstRead.then(() => { - throw new Error("warm restart completion callback was not entered"); - }), - ])); - expect(rejectHeldCompletion).toBeTypeOf("function"); - try { - expect(recoveryFenceActive).toBe(true); - const queuedBefore = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must not pass held completion" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.request("thread/resume", {}), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map( - (command) => command.commandId, - ), - ).toEqual(queuedBefore); - } finally { - rejectHeldCompletion!( - new Error("fixture recovery completion callback failed"), - ); - } - } - await expect(firstRead).rejects.toThrow( - snapshotFault - ? ordinaryFollowup === "rejected_snapshot" - ? "snapshot observation rejected" - : "completion_unproven" - : "recovery completion callback failed", - ); - expect(recoveryFenceActive).toBe(true); - expect(completionSnapshotIds).toHaveLength(snapshotFault ? 0 : 1); - const beforeDeniedWork = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - const beforeDeniedProviderCalls = await readFile(callsPath, "utf8"); - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must remain fenced" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.request("thread/resume", {}), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.attachRun!({ - runId: "must-not-attach", - turnId: "must-not-attach", - itemId: "must-not-attach", - }), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map((command) => command.commandId), - ).toEqual(beforeDeniedWork); - expect(await readFile(callsPath, "utf8")).toBe( - beforeDeniedProviderCalls, - ); - snapshotObserverSpy?.mockRestore(); - const providerCallsBeforeRetry = await readFile(callsPath, "utf8"); - expect( - (await within(resumed.transport.request("thread/read", {}))).thread, - ).toMatchObject(thread); - expect(recoveryFenceActive).toBe(false); - expect(completionSnapshotIds).toHaveLength(snapshotFault ? 1 : 2); - expect(new Set(completionSnapshotIds).size).toBe( - completionSnapshotIds.length, - ); - expect(await readFile(callsPath, "utf8")).toBe( - providerCallsBeforeRetry, - ); - return; - } - const read = await within( - resumed.transport.request("thread/read", {}), - 10_000, - ); - if ((resumed.evidence().codexPid ?? 0) > 0) - providerPids.add(resumed.evidence().codexPid!); - expect(read.thread).toMatchObject(thread); - expect(cores).toHaveLength(2); - expect(cores[1]!.store.state.identity).toEqual(desired); - expect(cores[1]!.store.state.warmTransition).toBeUndefined(); - expect( - cores[1]!.store.state.completedWarmTransition?.receipt.transitionId, - ).toBe(runner.warmTransition.receipt.transitionId); - if (routed) { - expect(routeCalls.slice(routeCallCount)).toEqual([ - oldIdentity.runId, - desired.runId, - ]); - expect([...routes.keys()]).toEqual([ - `/api/runner/v1/connect/${desired.runId}`, - ]); - } - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(2); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/resume")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - if (ordinaryFollowup) { - recoveryClaimCurrent = false; - const resumedCore = cores[1]!; - resumedCore.disconnectActiveRunner(); - await vi.waitFor( - () => expect(resumedCore.activeRunnerConnectionCount()).toBe(1), - { timeout: 2_000 }, - ); - expect(recoveryFenceActive).toBe(false); - expect( - (await within(resumed.transport.request("thread/read", {}))).thread, - ).toMatchObject(thread); - const thirdRunId = randomUUID(); - await within( - resumed.transport.attachRun!({ - runId: thirdRunId, - turnId: `turn-${thirdRunId}`, - itemId: `item-${thirdRunId}`, - }), - ); - expect(resumedCore.store.state.identity.runId).toBe(thirdRunId); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual([...calls, "thread/goal/get"]); - } - } finally { - snapshotObserverSpy?.mockRestore(); - commitSpy?.mockRestore(); - commandObserverSpy?.mockRestore(); - try { - await within(resumed?.transport.close() ?? Promise.resolve()).catch( - () => undefined, - ); - await within(first.transport.detachControllerForRestart!()).catch( - () => undefined, - ); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within(handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - for (const pid of providerPids) { - await stopOwnedProvider(pid); - } - cleanupProven = true; - } finally { - for (const core of cores) await core.stop().catch(() => undefined); - if (routed) - await new Promise((resolveClose) => - routeServer.close(() => resolveClose()), - ); - coreSpy.mockRestore(); - launchSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - if (cleanupProven) - for (const directory of legacyProbeDirectories) - await rm(directory, { recursive: true, force: true }); - } - } - }, - 30_000, -); - -it("rotates PRP authority in place for a warm cross-run attachment", async () => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-")); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - const within = async (label: string, promise: Promise) => - await Promise.race([ - promise, - new Promise((_, reject) => - setTimeout(() => reject(new Error(`${label} timeout`)), 5_000), - ), - ]); - try { - await within("initialize", bundle.transport.request("initialize", {})); - await within( - "thread start", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [ - { - name: "get_task_context", - description: "Read the active task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ], - completionContract: { - revision: "sha256:warm-three-turn-contract", - criterionIds: ["objective"], - }, - }), - ); - const runnerPid = bundle.evidence().runnerPid; - const providerPid = bundle.evidence().codexPid; - const notifications = bundle.transport - .notifications() - [Symbol.asyncIterator](); - const waitForCompletion = async (label: string) => { - const deadline = Date.now() + 5_000; - while (Date.now() < deadline) { - const next = await Promise.race([ - notifications.next(), - new Promise((_, reject) => - setTimeout( - () => reject(new Error(`${label} notification timeout`)), - 1_000, - ), - ), - ]); - if (next.value?.method === "turn/completed") return; - } - throw new Error(`${label} completion timeout`); - }; - await within( - "first turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "first run" }], - }), - ); - await waitForCompletion("first run"); - - await within( - "warm attach", - bundle.transport.attachRun!({ - runId: "run-warm-second", - turnId: "turn-warm-second", - itemId: "item-warm-second", - }), - ); - await within( - "second turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "second run" }], - }), - ); - await waitForCompletion("second run"); - - await within( - "second warm attach", - bundle.transport.attachRun!({ - runId: "run-warm-third", - turnId: "turn-warm-third", - itemId: "item-warm-third", - }), - ); - await within( - "third turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "third run" }], - }), - ); - await waitForCompletion("third run"); - - expect(bundle.evidence()).toMatchObject({ - runnerPid, - codexPid: providerPid, - runnerExited: false, - }); - } finally { - await bundle.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it("recovers an active second warm business run after deleting its controller context assets", async () => { - const root = await mkdtemp(join(tmpdir(), "runnerd-warm-second-active-")); - const firstRoot = join(root, "first-controller"), nextRoot = join(root, "next-controller"); - const assetRoot = join(root, "retired-controller-assets"); - const skillRoot = join(assetRoot, "skill"), instructionRoot = join(assetRoot, "instructions"); - await mkdir(skillRoot, { recursive: true }); await mkdir(instructionRoot, { recursive: true }); - await writeFile(join(skillRoot, "SKILL.md"), "# Pinned skill\n"); - await writeFile(join(instructionRoot, "AGENTS.md"), "Pinned instructions\n"); - const server = createServer(); - let authority: DurablePrpControlPlane | null = null, generation = 0; - let checkpoint: durableControlPlane.DurablePrpCheckpoint | undefined; - const registered = new WeakSet(); - server.on("upgrade", (request, socket, head) => authority - ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); - await new Promise(done => server.listen(0, "127.0.0.1", done)); - const address = server.address(); - if (!address || typeof address === "string") throw new Error("Missing owned Runner route"); - const registration = async (core: DurablePrpControlPlane) => { - if (!registered.has(core)) { - core.setCheckpointPersistence(async value => { checkpoint = value; }); - registered.add(core); - } - authority = core; const current = ++generation; - return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: async () => { if (generation === current) authority = null; } }; - }; - const initialIdentity = { runnerInstanceId: "runner-warm-second-active", environmentLeaseId: "lease-warm-second-active", - runId: "run-warm-first-healthy", normalizedSessionId: "session-warm-second-active", turnId: "turn-first", itemId: "item-first" }; - const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(root, "--emit-tool-call", "--durable-tool-ids"), stateDirectory: firstRoot, - runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), - prpIdentity: initialIdentity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration }; - const first = createCapabilityRunnerdCodexTransport(options); - const within = async (label: string, promise: Promise) => { - let timer: ReturnType | undefined; - try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`warm active recovery stage timed out: ${label}`)), 5_000); })]); } - finally { clearTimeout(timer); } - }; - const waitComplete = async (transport: typeof first.transport) => { - for await (const notification of transport.notifications()) if (notification.method === "turn/completed") return; - }; - let calls = 0, runnerPid: number | null = null, providerPid: number | null = null; - let recovered: ReturnType | undefined; - const result = { ok: true, task: { id: "task-1" } }; - first.transport.setServerRequestHandler(async () => ++calls === 1 - ? { success: true, contentItems: [{ type: "inputText", text: JSON.stringify(result) }] } - : new Promise>(() => undefined)); - try { - const opened = await within("first thread/start", first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [{ name: "get_task_context", description: "Read current task", - inputSchema: { type: "object", properties: {}, additionalProperties: false } }] })); - runnerPid = first.evidence().runnerPid; - providerPid = first.evidence().codexPid; - expect(providerPid).toEqual(expect.any(Number)); - const providerStart = execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim(); - const startedAt = first.transport.processInfo().startedAt; - const firstCompleted = waitComplete(first.transport); - await within("first turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "first healthy business run" }] })); - await within("healthy first turn completion", firstCompleted); - const attachment = { runId: "run-warm-second-active", turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; - await within("warm second business attachment", first.transport.attachRun!(attachment)); - const active = await within("second active turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "second business run held at tool" }] })); - await vi.waitFor(() => expect(calls).toBe(2), { timeout: 5_000 }); - const activeCore = authority!; - const savedIdentity = { ...activeCore.store.state.identity }; - expect(savedIdentity).toEqual({ ...initialIdentity, ...attachment }); - await activeCore.flushCheckpoint(); - await within("controller detach", first.detachControllerForRestart()); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - await rm(assetRoot, { recursive: true, force: true }); - expect(existsSync(assetRoot)).toBe(false); - checkpoint!.restoreSessionDirectory(nextRoot); - const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); - const openedThread = opened.thread as Record; - const activeTurn = active.turn as Record; - expect(openedThread.id).toEqual(expect.any(String)); - expect(activeTurn.id).toEqual(expect.any(String)); - const shared = { ...options, stateDirectory: nextRoot, - resumeProviderSession: { driverSessionId: String(openedThread.id), providerSessionId: String(openedThread.sessionId) }, - resumeActiveTurnId: String(activeTurn.id), runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), - runnerStateDirectory: join(firstRoot, "runner"), readRunnerState: async () => JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8")), - prepareExternalRunnerState: async () => {}, archiveExternalRunnerState: async () => { throw new Error("Active authority must not rotate"); }, - runnerProcessLauncher: noSpawn, adoptExistingRunner: { pid: runnerPid!, processGroupId: runnerPid, startedAt, - isAlive: () => { try { process.kill(runnerPid!, 0); return true; } catch { return false; } }, - signal: (signal: NodeJS.Signals) => { try { process.kill(runnerPid!, signal); return true; } catch { return false; } } } }; - // The old server's synthetic pair must continue failing the strict Runner guard. - const wrong = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: { ...savedIdentity, - turnId: `turn-${attachment.runId}`, itemId: `item-${attachment.runId}` } }); - await expect(wrong.transport.request("thread/read", {})).rejects.toThrow("PRP recovery identity does not match the durable session binding"); - await wrong.detachControllerForRestart(); - // Leave the pending tool outcome unknown. Reattachment must authenticate - // and preserve it without fabricating a receipt or redispatching its handler. - recovered = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: savedIdentity }); - await within("replacement thread/read", recovered.transport.request("thread/read", {})); - expect(recovered.evidence()).toMatchObject({ runnerPid, runnerExited: false }); - expect(authority!.activeRunnerConnectionCount()).toBe(1); - expect(() => process.kill(providerPid!, 0)).not.toThrow(); - expect(execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim()).toBe(providerStart); - expect(calls).toBe(2); - expect(authority!.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0); - expect(recovered.transport.processInfo().startedAt).toBe(startedAt); - expect(authority!.store.state.identity).toEqual(savedIdentity); - expect(authority!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(noSpawn).not.toHaveBeenCalled(); - } catch (error) { - const saved = JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8").catch(() => "{}")); - const stderr = await readFile(join(firstRoot, "diagnostics/runnerd.stderr.log"), "utf8").catch(() => ""); - throw new Error(`${String(error)}; owned protocol evidence ${JSON.stringify({ - transport: recovered?.evidence(), connections: authority?.activeRunnerConnectionCount(), - identity: authority?.store.state.identity, commands: authority?.store.state.commands.map(c => ({type:c.type,status:c.status})), - events: authority?.store.state.committedEvents.map(e => e.eventType).slice(-8), - runner: { lifecycle:saved.lifecycle, runId:saved.runId, turnId:saved.turnId, itemId:saved.itemId, - outbox:saved.outbox?.length }, stderr:stderr.slice(-1500), - })}`); - } finally { - await (recovered ? within("recovered cleanup", recovered.transport.close()).catch(() => undefined) : undefined); - await within("first cleanup", first.transport.close()).catch(() => undefined); - if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } - await (authority as DurablePrpControlPlane | null)?.stop(); - if (runnerPid) { try { process.kill(-runnerPid, "SIGKILL"); } catch {} } - server.closeAllConnections(); - if (server.listening) await new Promise(done => server.close(() => done())); - for (const controller of [firstRoot, nextRoot]) await releaseMaterializedNativeRuntimeSkills(join(controller, "codex-home", "skills")); - await rm(root, { recursive: true, force: true }); - } -}, 30_000); - -it("waits for a warm runner to re-authenticate before probing attachment readiness", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-warm-reattach-before-probe-"), - ); - const server = createServer(); - const authorities = new Map(); - let blockFirstAuthorityReconnect = false; - let resolveRejectedReconnect!: () => void; - const rejectedReconnect = new Promise((resolvePromise) => { - resolveRejectedReconnect = resolvePromise; - }); - server.on("upgrade", (request, socket, head) => { - const route = request.url ?? ""; - if (route === "/runner-1" && blockFirstAuthorityReconnect) { - resolveRejectedReconnect(); - socket.destroy(); - return; - } - const authority = authorities.get(route); - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, route, head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Expected warm reconnect test listener"); - } - let registrationCount = 0; - const diagnostics: string[] = []; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 5_000, - onDiagnostic: (message) => diagnostics.push(message), - controlPlaneRegistration: async (authority) => { - registrationCount += 1; - const route = `/runner-${registrationCount}`; - authorities.set(route, authority); - return { - connectUrl: `ws://127.0.0.1:${address.port}${route}`, - release: () => { - if (authorities.get(route) === authority) authorities.delete(route); - }, - }; - }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let runnerPid: number | null = null; - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: codexSemanticToolSpecs(), - }); - runnerPid = bundle.evidence().runnerPid; - const firstAuthority = authorities.get("/runner-1"); - if (!firstAuthority) throw new Error("Missing first warm authority"); - const priorSnapshotCount = firstAuthority.store.state.commands.filter( - (command) => command.type === "session.snapshot", - ).length; - - blockFirstAuthorityReconnect = true; - firstAuthority.disconnectActiveRunner(); - const attachment = bundle.transport.attachRun!({ - runId: "run-warm-after-reconnect", - turnId: "turn-warm-after-reconnect", - itemId: "item-warm-after-reconnect", - }); - await Promise.race([ - rejectedReconnect, - new Promise((_, reject) => - setTimeout( - () => reject(new Error("runner did not attempt to reconnect")), - 5_000, - ), - ), - ]); - - // No command may be queued while its sole authenticated consumer is - // absent. The generic 30-second command timeout used to turn this state - // into same-run recovery and replace the healthy warm runner process. - expect( - firstAuthority.store.state.commands.filter( - (command) => command.type === "session.snapshot", - ), - ).toHaveLength(priorSnapshotCount); - - blockFirstAuthorityReconnect = false; - await Promise.race([ - attachment, - new Promise((_, reject) => - setTimeout(() => reject(new Error("warm attachment timeout")), 10_000), - ), - ]); - expect(bundle.evidence()).toMatchObject({ - runnerPid, - runnerExited: false, - }); - expect(diagnostics).toContain( - "warm runner connection interrupted; waiting for re-authentication before authority rotation", - ); - expect(diagnostics).toContain( - "warm runner re-authenticated before authority rotation", - ); - } finally { - await bundle.transport.close().catch(() => undefined); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // A successful durable close already stopped the runner process group. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it("releases both PRP authorities when warm rotation activation fails", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-warm-attach-activation-failure-"), - ); - const server = createServer(); - const authorities = new Map(); - const released: string[] = []; - server.on("upgrade", (request, socket, head) => { - const route = request.url ?? ""; - const authority = authorities.get(route); - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, route, head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Expected warm activation failure test listener"); - } - let registrationCount = 0; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - controlPlaneRegistration: async (authority) => { - registrationCount += 1; - const route = `/runner-${registrationCount}`; - authorities.set(route, authority); - return { - connectUrl: `ws://127.0.0.1:${address.port}${route}`, - ...(registrationCount === 1 - ? {} - : { - activate: () => { - throw new Error("rotation activation failed"); - }, - }), - release: () => { - released.push(route); - if (authorities.get(route) === authority) authorities.delete(route); - }, - }; - }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let runnerPid: number | null = null; - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: codexSemanticToolSpecs(), - }); - runnerPid = bundle.evidence().runnerPid; - - await expect( - bundle.transport.attachRun!({ - runId: "run-warm-activation-failure", - turnId: "turn-warm-activation-failure", - itemId: "item-warm-activation-failure", - }), - ).rejects.toThrow("rotation activation failed"); - expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"])); - expect(authorities.size).toBe(0); - await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( - "rotation activation failed", - ); - } finally { - await bundle.transport.close().catch(() => undefined); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // A successful durable close already stopped the runner process group. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it.each([ - { - binding: "runner instance", - priorRunnerInstanceId: "runner-other", - priorEnvironmentLeaseId: "lease-current", - }, - { - binding: "environment lease", - priorRunnerInstanceId: "runner-current", - priorEnvironmentLeaseId: "lease-other", - }, -])( - "quarantines a mismatched $binding instead of reusing its provider session", - async ({ priorRunnerInstanceId, priorEnvironmentLeaseId }) => { - const container = await mkdtemp( - join(tmpdir(), "runnerd-mismatched-authority-"), - ); - const stateDirectory = join(container, "state"); - await mkdir(join(stateDirectory, "control-plane"), { recursive: true }); - await writeFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - JSON.stringify({ - identity: { - runnerInstanceId: priorRunnerInstanceId, - environmentLeaseId: priorEnvironmentLeaseId, - runId: "run-old", - normalizedSessionId: "session-current", - turnId: "turn-old", - itemId: "item-old", - }, - }), - { mode: 0o600 }, - ); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - stateDirectory, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: { - runnerInstanceId: "runner-current", - environmentLeaseId: "lease-current", - runId: "run-new", - normalizedSessionId: "session-current", - turnId: "turn-new", - itemId: "item-new", - }, - }); - try { - await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( - "native_runner_state_quarantined", - ); - expect(await readdir(stateDirectory)).toEqual([]); - expect( - (await readdir(container)).some((entry) => - entry.startsWith("state.quarantine-"), - ), - ).toBe(true); - } finally { - await bundle.transport.close(); - await rm(container, { recursive: true, force: true }); - } - }, -); - -it.each([0, 193 * 1024 * 1024])( - "probes an exact-authority resume with %i extra journal bytes and confirms its live provider identity", - async (extraJournalBytes) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-exact-authority-resume-"), - ); - const identity = { - runnerInstanceId: "runner-exact-resume", - environmentLeaseId: "lease-exact-resume", - runId: "run-exact-resume", - normalizedSessionId: "session-exact-resume", - turnId: "turn-exact-resume", - itemId: "item-exact-resume", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, "--durable-turn-ids"), - stateDirectory, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - prpIdentity: identity, - }; - const first = createCapabilityRunnerdCodexTransport(options); - first.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let providerThread: { id: string; sessionId: string } | null = null; - try { - const opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - const thread = opened.thread as Record; - providerThread = { - id: String(thread.id), - sessionId: String(thread.sessionId), - }; - } finally { - await first.transport.close(); - } - if (providerThread === null) { - throw new Error( - "exact-authority fixture did not return a provider thread", - ); - } - - const statePath = join( - stateDirectory, - "control-plane", - "control-plane-state.json", - ); - if (extraJournalBytes > 0) { - const padding = Buffer.alloc(1024 * 1024, 0x20); - const stateHandle = await open(statePath, "a"); - try { - for (let remaining = extraJournalBytes; remaining > 0;) { - const bytesToWrite = Math.min(remaining, padding.length); - await stateHandle.write(padding, 0, bytesToWrite); - remaining -= bytesToWrite; - } - } finally { - await stateHandle.close(); - } - expect((await stat(statePath)).size).toBeGreaterThan( - 64 * 1024 * 1024, - ); - } - const beforeResume = JSON.parse(await readFile(statePath, "utf8")) as { - commands: Array<{ type: string }>; - committedEvents: Array<{ eventType: string }>; - }; - expect( - beforeResume.commands.some((command) => command.type === "run.attach"), - ).toBe(false); - const priorResumeEvents = beforeResume.committedEvents.filter( - (event) => event.eventType === "session.resumed", - ).length; - const priorSnapshots = beforeResume.commands.filter( - (command) => command.type === "session.snapshot", - ).length; - - const resumed = createCapabilityRunnerdCodexTransport({ - ...options, - resumeProviderSession: { - driverSessionId: providerThread.id, - providerSessionId: providerThread.sessionId, - }, - }); - resumed.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await resumed.transport.request("thread/read", {}); - expect(read.thread).toMatchObject(providerThread); - const afterResume = JSON.parse(await readFile(statePath, "utf8")) as { - commands: Array<{ commandId: string; type: string; status: string }>; - committedEvents: Array<{ eventType: string }>; - }; - expect(afterResume.commands).toContainEqual( - expect.objectContaining({ - commandId: expect.stringMatching(/^command_resume_probe_/), - type: "runner.drain", - status: "completed", - }), - ); - expect(afterResume.commands).toContainEqual( - expect.objectContaining({ - type: "session.snapshot", - status: "completed", - }), - ); - expect( - afterResume.commands.filter( - (command) => command.type === "session.snapshot", - ), - ).toHaveLength(priorSnapshots + 2); - // The authenticated snapshot above proves the live provider identity. - // Control-first dispatch may deliver that command before the independent - // session event is ingested. Still require exactly one durable event; - // don't mistake an immediate file read for an event-delivery barrier. - await vi.waitFor( - async () => { - const delivered = JSON.parse(await readFile(statePath, "utf8")) as { - committedEvents: Array<{ eventType: string }>; - }; - expect( - delivered.committedEvents.filter( - (event) => event.eventType === "session.resumed", - ), - ).toHaveLength(priorResumeEvents + 1); - }, - { timeout: 3_000, interval: 25 }, - ); - } finally { - await resumed.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } - }, - 30_000, -); - -it("still fails closed when a real close grace period cannot fit a durable suspension round trip", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-close-grace-too-small-"), - ); - const identity = { - runnerInstanceId: "runner-close-grace-too-small", - environmentLeaseId: "lease-close-grace-too-small", - runId: "run-close-grace-too-small", - normalizedSessionId: "session-close-grace-too-small", - turnId: "turn-close-grace-too-small", - itemId: "item-close-grace-too-small", - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - // No real durable command round trip can complete this fast. A wider - // budget for the provider-drain proof must not turn this barrier into - // one that always passes; it still needs the actual proof to arrive. - closeGraceMs: 1, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: identity, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - const closed = bundle.transport.close(); - await expect(closed).rejects.toMatchObject({ - code: "native_session_close_unrecoverable", - semanticToolsSettled: true, - }); - // Retirement preserves the original rejected promise and its proof. - expect(bundle.transport.close()).toBe(closed); - } finally { - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected", "credentials-cleaned"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); - const tracePath = join(stateDirectory, "provider-trace.ndjson"); - const skillRoot = join(stateDirectory, "runtime-skill"); - const instructionRoot = join(stateDirectory, "runtime-instructions"); - await Promise.all([mkdir(skillRoot), mkdir(instructionRoot)]); - await writeFile(join(skillRoot, "SKILL.md"), "# Assigned runtime skill\n"); - await writeFile(join(instructionRoot, "AGENTS.md"), "Runtime instructions\n"); - const credentialsCleaned = pendingTail === "credentials-cleaned"; - const sourceCodexHome = join(stateDirectory, "selected-account-first"); - if (credentialsCleaned) { - await mkdir(sourceCodexHome); - await writeFile(join(sourceCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); - } - const baseIdentity = { - runnerInstanceId: "runner-cold-attach", - environmentLeaseId: "lease-cold-attach", - runId: "run-cold-first", - normalizedSessionId: "session-cold-attach", - turnId: "turn-cold-first", - itemId: "item-cold-first", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--include-skill-instructions", - "--durable-turn-ids", - ...(credentialsCleaned ? ["--require-codex-home-auth"] : []), - "-c", - 'shell_environment_policy.set={PATH="/run/A"}', - ), - stateDirectory, - ...(credentialsCleaned ? { sourceCodexHome } : {}), - environment: { - PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-A-capability", - PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, - PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), - ...(credentialsCleaned ? { - PAPERCLIP_NATIVE_MCP_NAME: "assigned", - PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/first", - PAPERCLIP_NATIVE_MCP_TOKEN: "first-assigned-gateway-credential", - } : {}), - }, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), - }; - const dynamicTools = [ - { - name: "get_task_context", - description: "Read the active task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ]; - const first = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: baseIdentity, - }); - first.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let firstProviderThread: { id: string; sessionId: string } | null = null; - try { - const started = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [...dynamicTools, ...codexSemanticToolSpecs()], - completionContract: { - revision: "contract-first", - criterionIds: ["criterion-first"], - }, - }); - const startedThread = started.thread as Record; - firstProviderThread = { - id: String(startedThread.id), - sessionId: String(startedThread.sessionId), - }; - expect( - (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) - .mode & 0o222, - ).toBe(0); - expect( - ( - await stat( - join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), - ) - ).mode & 0o222, - ).toBe(0); - await first.transport.request("turn/start", { - input: [{ type: "text", text: "$assigned first process" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], - }); - for await (const event of first.transport.notifications()) { - if (event.method === "turn/completed") break; - } - } finally { - await first.transport.close(); - } - if (!firstProviderThread) { - throw new Error("cold attach fixture did not return a provider thread"); - } - const successorCodexHome = join(stateDirectory, "selected-account-next"); - if (credentialsCleaned) { - await rm(join(stateDirectory, "codex-home", "auth.json")); - await rm(sourceCodexHome, { recursive: true }); - await mkdir(successorCodexHome); - await writeFile(join(successorCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); - } - - // Model a retained provider-owned suffix: rotation must deliver and durably - // accept it before attempting run.attach, rather than dropping or ACKing it. - const retainedDelta = "cold-recovery-retained-tail"; - if (pendingTail) { - const providerFile = join(stateDirectory, "runner", "codex-provider-state.json"); - const provider = JSON.parse(await readFile(providerFile, "utf8")); - expect(provider.pendingEvents).toEqual([]); - provider.pendingEvents.push({ - executorEventId: `codex_provider_${String(provider.nextProviderEventSeq++).padStart(16, "0")}`, - eventType: pendingTail === "unknown-semantic" ? "semantic_tool.input" : "usage.reported", - priority: "p0", - payload: pendingTail === "unknown-semantic" ? { semantic_tool: { - schema: "paperclip.prp.semantic_tool.v1", schemaVersion: 1, phase: "input", - callId: "unknown-source-call", operationId: "get_task_context", input: {}, idempotencyKey: null, - correlation: { runId: baseIdentity.runId, normalizedSessionId: baseIdentity.normalizedSessionId, - turnId: baseIdentity.turnId, itemId: baseIdentity.itemId }, - content: { digest: `sha256:${createHash("sha256").update("{}").digest("hex")}`, redactionDisposition: "digest_only", references: [] }, - } } : { provider: "codex", kind: "usage", delta: retainedDelta, - providerTurnId: provider.completed_provider_turn_id, runDelta: { inputTokens: 20, outputTokens: 2 }, - runDeltaAvailable: true, runDeltaBasis: "run" }, - }); - await writeFile(providerFile, JSON.stringify(provider)); - } - const retainedEvents: PrpEvent[] = []; - let releaseRetained!: () => void, retainedEntered!: () => void; - const retainedAccepted = new Promise(done => { releaseRetained = done; }); - const retainedAdmission = new Promise(done => { retainedEntered = done; }); - const secondIdentity = { - ...baseIdentity, - runId: "run-cold-second", - turnId: "turn-cold-second", - itemId: "item-cold-second", - }; - let stagedCheckpoint: durableControlPlane.DurablePrpCheckpoint | undefined; - let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; - const rotatedOptions = { - ...options, - ...(credentialsCleaned ? { sourceCodexHome: successorCodexHome } : {}), - onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { - authority.setCheckpointPersistence(async checkpoint => { - stagedCheckpoint = checkpoint; - if (rejectStagedCommit) throw new Error("fixture lost controller after durable intent"); - }); - } : undefined, - onRecoveryAccountingEvent: pendingTail ? async (event: PrpEvent) => { - if (event.payload.delta === retainedDelta) { - retainedEvents.push(structuredClone(event)); - retainedEntered(); - if (pendingTail === "accounting-rejected") throw new Error("source accounting receipt unavailable"); - await retainedAccepted; - } - } : undefined, - environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability", - ...(credentialsCleaned ? { - PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/current", - PAPERCLIP_NATIVE_MCP_TOKEN: "current-assigned-gateway-credential", - } : {}), - }, - codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), - resumeDynamicTools: dynamicTools, - resumeCompletionContract: { - revision: "contract-second", - criterionIds: ["criterion-second"], - }, - prpIdentity: secondIdentity, - }; - let rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); - if (pendingTail === "restart-before-bootstrap") { - await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("fixture lost controller after durable intent"); - expect(rotated.evidence().runnerPid).toBeNull(); - expect(stagedCheckpoint?.runId).toBe(secondIdentity.runId); - expect([...stagedCheckpoint!.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value).type)).not.toContain("run.attach"); - await rotated.detachControllerForRestart().catch(() => undefined); - rejectStagedCommit = false; - await rm(join(stateDirectory, "control-plane"), { recursive: true, force: true }); - stagedCheckpoint!.restoreSessionDirectory(stateDirectory); - rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); - } - const currentHandler = vi.fn(async () => ({ success: true, contentItems: [] })); - rotated.transport.setServerRequestHandler(currentHandler); - if (pendingTail === "accounting-rejected") { - const reading = rotated.transport.request("thread/read", {}); - void reading.catch(() => undefined); - try { - await Promise.race([retainedAdmission, reading]); - const source = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(source.identity).toEqual(baseIdentity); - expect(source.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); - expect(source.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); - const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => - entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); - expect(currentHandler).not.toHaveBeenCalled(); - } finally { - await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); - await expect(reading).rejects.toThrow(); - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - } - return; - } - if (pendingTail === "unknown-semantic") { - try { - await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("native_runner_source_semantic_reconciliation_required"); - expect(currentHandler).not.toHaveBeenCalled(); - const retained = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(retained.identity).toEqual(baseIdentity); - expect(retained.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); - expect(retained.commands.some((command: { type: string }) => ["run.attach", "semantic_tool.result"].includes(command.type))).toBe(false); - expect(retained.committedEvents.some((event: PrpEvent) => event.eventType === "semantic_tool.input")).toBe(true); - } finally { - await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - } - return; - } - let recoveryError: unknown; - try { - const reading = rotated.transport.request("thread/read", {}); - void reading.catch(() => undefined); - if (pendingTail) { - await Promise.race([retainedAdmission, reading]); - const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => - entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); - const retainedCore = stagedCheckpoint ? { - identity: JSON.parse(stagedCheckpoint.records.get("header")!).state.identity, - commands: [...stagedCheckpoint.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value)), - } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(retainedCore.identity).toEqual(baseIdentity); - expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); - if (credentialsCleaned) { - const config = await readFile(join(stateDirectory, "codex-home", "config.toml"), "utf8"); - expect(config).toContain("https://paperclip.example/mcp/current"); - expect(config).toContain("current-assigned-gateway-credential"); - expect(config).not.toContain("first-assigned-gateway-credential"); - expect(currentHandler).not.toHaveBeenCalled(); - } - } - releaseRetained(); - const read = await reading; - expect(retainedEvents).toHaveLength(pendingTail ? 1 : 0); - if (pendingTail) expect(retainedEvents[0]).toMatchObject({ - runId: baseIdentity.runId, turnId: baseIdentity.turnId, itemId: baseIdentity.itemId, - }); - const persistedProvider = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); - expect(persistedProvider.pendingEvents).toEqual([]); - expect(persistedProvider.config.args.join("\n")).toContain('/run/B'); - expect(JSON.stringify(persistedProvider)).not.toContain("test-run-B-capability"); - expect(read.thread).toMatchObject({ - id: firstProviderThread.id, - sessionId: firstProviderThread.sessionId, - cwd: tmpdir(), - }); - await rotated.transport.request("turn/start", { - input: [{ type: "text", text: "$assigned second authority epoch" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], - }); - for await (const event of rotated.transport.notifications()) { - if (event.method === "paperclip/runResult") break; - } - expect(rotated.evidence().diagnostics).toContain( - "runnerd attached the durable provider session to a fresh PRP run authority", - ); - if (pendingTail) { - const target = stagedCheckpoint ? JSON.parse(stagedCheckpoint.records.get("header")!).state - : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(target.stagedRunAttachment).toBeUndefined(); - expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); - } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); - } catch (error) { - recoveryError = error; - throw error; - } finally { - releaseRetained(); - await rotated.transport.close().catch(error => { - if (recoveryError) throw new AggregateError([recoveryError, error], "Provider recovery and cleanup failed"); - throw error; - }); - } - const requestFrames = (await readFile(tracePath, "utf8")) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record) - .filter( - (entry) => - entry.kind === "frame" && entry.direction === "client_to_provider", - ) - .map( - (entry) => - JSON.parse( - Buffer.from(String(entry.rawBase64), "base64").toString("utf8"), - ) as Record, - ) - ; - const resumeFrames = requestFrames.filter((frame) => frame.method === "thread/resume"); - for (const frame of requestFrames.filter((f) => ["thread/start", "thread/resume"].includes(String(f.method)))) { - expect(frame.params).toMatchObject({ config: { "skills.include_instructions": true } }); - } - const turnFrames = requestFrames.filter((f) => f.method === "turn/start"); - expect(turnFrames).toHaveLength(2); - for (const frame of turnFrames) { - expect(frame.params).toMatchObject({ input: expect.arrayContaining([{ type: "skill", name: "assigned", path: join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md") }]) }); - } - expect(resumeFrames.length).toBeGreaterThanOrEqual(1); - for (const frame of resumeFrames) { - expect(frame).toEqual( - expect.objectContaining({ - method: "thread/resume", - params: expect.objectContaining({ threadId: firstProviderThread.id }), - }), - ); - } - - if (pendingTail) { - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - return; - } - - // A remote process owner keeps runner-state outside the controller's local - // session root. Resume must defer to its explicit state reader instead of - // rejecting recovery before the remote checkpoint can be made available. - const externallyOwnedRunnerStateDirectory = join( - stateDirectory, - "externally-owned-runner", - ); - await rename( - join(stateDirectory, "runner"), - externallyOwnedRunnerStateDirectory, - ); - const readRunnerState = async () => - JSON.parse( - await readFile( - join(externallyOwnedRunnerStateDirectory, "runner-state.json"), - "utf8", - ), - ) as Record; - const mismatched = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - resumeDynamicTools: dynamicTools, - prpIdentity: { - ...secondIdentity, - runId: "run-cold-other", - turnId: "turn-cold-other", - itemId: "item-cold-other", - }, - }); - await expect(mismatched.transport.request("thread/read", {})).rejects.toThrow( - "native_runner_prp_run_rotation_unavailable", - ); - await mismatched.transport.close(); - - const externalIdentity = { - ...secondIdentity, - runId: "run-cold-external", - turnId: "turn-cold-external", - itemId: "item-cold-external", - }; - const rejectedExternalRotationSteps: string[] = []; - let externalArchiveDirectory: string | null = null; - const rejectedExternalRotation = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - prepareExternalRunnerState: async () => { - rejectedExternalRotationSteps.push("prepared"); - }, - archiveExternalRunnerState: async ({ archiveKey }) => { - await expect( - stat(join(stateDirectory, "control-plane")), - ).rejects.toMatchObject({ code: "ENOENT" }); - expect( - await stat( - join( - stateDirectory, - "authority-epochs", - `epoch-${archiveKey}`, - "control-plane", - ), - ), - ).toBeDefined(); - externalArchiveDirectory = join( - stateDirectory, - "external-authority-epochs", - archiveKey, - ); - await mkdir(externalArchiveDirectory, { recursive: true }); - await rename( - join(externallyOwnedRunnerStateDirectory, "runner-state.json"), - join(externalArchiveDirectory, "runner-state.json"), - ); - rejectedExternalRotationSteps.push("remote-archived"); - throw new Error("controller crashed after external archive"); - }, - resumeDynamicTools: dynamicTools, - prpIdentity: externalIdentity, - }); - await expect( - rejectedExternalRotation.transport.request("thread/read", {}), - ).rejects.toThrow("controller crashed after external archive"); - await rejectedExternalRotation.transport.close(); - expect(rejectedExternalRotationSteps).toEqual([ - "prepared", - "remote-archived", - ]); - await expect(stat(join(stateDirectory, "control-plane"))).rejects.toThrow(); - await expect( - stat(join(externallyOwnedRunnerStateDirectory, "runner-state.json")), - ).rejects.toThrow(); - - const externalRotationSteps: string[] = []; - const externallyRotated = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - prepareExternalRunnerState: async () => { - throw new Error("retry must not prepare a new external runner"); - }, - archiveExternalRunnerState: async ({ archiveKey }) => { - expect(externalArchiveDirectory).toBe( - join(stateDirectory, "external-authority-epochs", archiveKey), - ); - externalRotationSteps.push("archived"); - return JSON.parse( - await readFile( - join(externalArchiveDirectory!, "runner-state.json"), - "utf8", - ), - ) as Record; - }, - resumeDynamicTools: dynamicTools, - resumeCompletionContract: { - revision: "contract-external", - criterionIds: ["criterion-external"], - }, - prpIdentity: externalIdentity, - }); - externallyRotated.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await externallyRotated.transport.request("thread/read", {}); - expect(read.thread).toMatchObject({ - id: firstProviderThread.id, - sessionId: firstProviderThread.sessionId, - cwd: tmpdir(), - }); - expect(externalRotationSteps).toEqual(["archived"]); - } finally { - await externallyRotated.transport.close(); - } - - const restored = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - resumeDynamicTools: dynamicTools, - prpIdentity: externalIdentity, - }); - restored.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await restored.transport.request("thread/read", {}); - expect(read.thread).toMatchObject({ - id: "codex-thread-1", - cwd: tmpdir(), - }); - expect( - (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) - .mode & 0o222, - ).toBe(0); - expect( - ( - await stat( - join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), - ) - ).mode & 0o222, - ).toBe(0); - const providerState = JSON.parse( - await readFile( - join(externallyOwnedRunnerStateDirectory, "codex-provider-state.json"), - "utf8", - ), - ) as { toolBridge?: { authorized?: Record } }; - expect(Object.keys(providerState.toolBridge?.authorized ?? {})).toEqual([ - "get_task_context", - "paperclip_block", - "paperclip_finish", - ]); - await restored.transport.request("turn/start", { - input: [{ type: "text", text: "restored process" }], - }); - for await (const event of restored.transport.notifications()) { - if (event.method === "turn/completed") break; - } - expect(restored.evidence()).toMatchObject({ - runnerExited: false, - codexPid: expect.any(Number), - }); - } finally { - await restored.transport.close(); - await releaseMaterializedNativeRuntimeSkills( - join(stateDirectory, "codex-home", "skills"), - ); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -async function verifyLiveRunnerAdoption( - mismatchedCheckpoint: boolean, - mismatchedArtifact = false, - goalMidTurn = false, - detachBeforeCleanup = false, - startWithoutCheckpoint = false, - preparedTask = false, - adoptionAuthorization: "allow" | "deny" | "legacy" | "legacy-missing" | "missing-binding" = "allow", - toolHandlerOrder?: "before" | "after" | "completion-before" | "completion-after", - uncertainAtClose = false, - semanticRecoveryMode?: "denied-before-cancel" | "cleanup-only", - stopExitProofFault?: "missing" | "false", -) { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); - const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; - const completionPath = join(stateDirectory, "pending-completion.json"); - if (pendingCompletion) await writeFile(completionPath, JSON.stringify({ - schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed the synthetic work", - completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, - evidence: [], verification: [], attentionRequests: [], artifacts: [], - })); - const server = createServer(); - let authority: DurablePrpControlPlane | null = null; - const checkpoint = vi.fn(async () => undefined); - server.on("upgrade", (request, socket, head) => { - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, "/runner", head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") - throw new Error("Expected adoption test listener"); - const registration = vi.fn(async (next: DurablePrpControlPlane) => { - authority = next; - return { - connectUrl: `ws://127.0.0.1:${address.port}/runner`, - ...(mismatchedArtifact || uncertainAtClose || semanticRecoveryMode === "cleanup-only" ? { checkpoint } : {}), - release: async () => { - if (authority === next) authority = null; - }, - }; - }); - const identity = { - runnerInstanceId: "runner-live-adopt", - environmentLeaseId: "lease-live-adopt", - runId: "run-live-adopt", - normalizedSessionId: "session-live-adopt", - turnId: "turn-live-adopt", - itemId: "item-live-adopt", - }; - const sharedOptions = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, ...(pendingCompletion ? ["--terminal-result-file", completionPath] : toolHandlerOrder ? ["--emit-tool-call"] : []), ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), - stateDirectory, - prpIdentity: identity, - lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, - controlPlaneRegistration: registration, - }; - const prepareLaunch = vi.fn(async (requirements: RunnerProviderLaunchRequirements) => ({ - configurationId: "adopted-preparation", requirementsDigest: runnerLaunchRequirementsDigest(requirements), - provider: createRunnerProviderLaunch(requirements, { options: { codexArgs: sharedOptions.codexArgs }, - root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), - codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "" }), - })); - const options = { ...sharedOptions, ...(preparedTask ? { - prepareLaunch, runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), - runnerProcessLauncher: (spec: durableControlPlane.RunnerProcessLaunchSpec) => { - const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: { ...process.env, ...spec.environment }, - detached: process.platform !== "win32", stdio: "ignore" }); - return { child, processGroupId: child.pid ?? null, completion: new Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }>((resolveExit, reject) => { - child.once("error", reject); - child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); - }) }; - }, - } : {}) }; - const first = createCapabilityRunnerdCodexTransport(options); - const pendingTool = vi.fn(() => new Promise>(() => undefined)); - if (toolHandlerOrder) first.transport.setServerRequestHandler(pendingTool); - let releaseRecovery!: () => void; - const recoveryGate = new Promise(resolve => { releaseRecovery = resolve; }); - const recoveryDenied = new Error("execution authority revoked after Stop"); - const recoveredTool = vi.fn(async () => { - if (semanticRecoveryMode) { - // Model server authority denying replay after Stop cleared the task lock. - // The gate holds that denial until reattachment has returned its session, - // before the caller invokes the eventual onSession cancellation callback. - await recoveryGate; - throw recoveryDenied; - } - return { result: { receipt: "known-outcome" }, isError: false }; - }); - const redispatchedTool = vi.fn(async () => ({ success: true, contentItems: [] })); - let runnerPid: number | null = null; - let runnerStartedAt: string | null = null; - let adopted: ReturnType | null = - null; - let restoreIntervalSpies: () => void = () => undefined; - try { - let opened: Record; - try { - opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: toolHandlerOrder && !pendingCompletion ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : codexSemanticToolSpecs(), - }); - } catch (error) { - const stderr = await readFile( - join(stateDirectory, "diagnostics", "runnerd.stderr.log"), - "utf8", - ).catch(() => ""); - throw new Error( - `${String(error)}\n${JSON.stringify(first.evidence())}${stderr ? `\n${stderr}` : ""}`, - ); - } - runnerPid = first.evidence().runnerPid; - runnerStartedAt = first.transport.processInfo().startedAt; - expect(runnerPid).toEqual(expect.any(Number)); - - if (goalMidTurn) { - await first.transport.request("thread/goal/set", { objective: "Recover a live goal", status: "active" }); - for await (const event of first.transport.notifications()) { - if (event.method === "turn/started") break; - } - } - - if (toolHandlerOrder) { - await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); - await vi.waitFor(() => expect(pendingTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); - } - await first.detachControllerForRestart(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - if (goalMidTurn) { - await writeFile(join(stateDirectory, "emit-goal-item"), "emit"); - // runnerd need not poll the provider into its PRP outbox while disconnected. - // Wait for flushed provider output, not a platform-dependent final poll - // racing the disconnect. Adoption must still bind that buffered item. - await vi.waitFor(async () => { - expect(await readFile(join(stateDirectory, "emit-goal-item.sent"), "utf8")).toBe("sent"); - }, { timeout: 5_000 }); - } - - const controlPlaneStatePath = join( - stateDirectory, - "control-plane", - "control-plane-state.json", - ); - const compactProviderIdentityEvents = async () => { - const controlPlaneState = JSON.parse( - await readFile(controlPlaneStatePath, "utf8"), - ) as { committedEvents: Array<{ eventType: string }> }; - controlPlaneState.committedEvents = - controlPlaneState.committedEvents.filter( - (event) => - event.eventType !== "harness.ready" && - event.eventType !== "session.started" && - event.eventType !== "session.resumed", - ); - await writeFile( - controlPlaneStatePath, - `${JSON.stringify(controlPlaneState, null, 2)}\n`, - { mode: 0o600 }, - ); - }; - if (!startWithoutCheckpoint) await compactProviderIdentityEvents(); - - if (preparedTask) { - if (adoptionAuthorization !== "legacy") await rm(join(stateDirectory, "launch-preparation.json")); - prepareLaunch.mockClear(); - registration.mockClear(); - if (adoptionAuthorization === "missing-binding") { - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - delete retained.preparedLaunchConfigurations; - await writeFile(controlPlaneStatePath, JSON.stringify(retained), { mode: 0o600 }); - } - } - const authorizeRunnerAdoption = vi.fn(async () => { - expect(registration).not.toHaveBeenCalled(); - if (adoptionAuthorization === "deny") throw new Error("project grant revoked"); - }); - const duplicateLauncher = vi.fn(() => { - throw new Error("duplicate runner spawn attempted"); - }); - const openedThread = opened.thread as Record; - const signal = vi.fn(() => true); - const probeError = new Error("fixture remote process identity is unavailable"); - let rejectLiveness = false; - const intervals = uncertainAtClose ? vi.spyOn(globalThis, "setInterval") : null; - const clearedIntervals = uncertainAtClose ? vi.spyOn(globalThis, "clearInterval") : null; - restoreIntervalSpies = () => { intervals?.mockRestore(); clearedIntervals?.mockRestore(); }; - adopted = createCapabilityRunnerdCodexTransport({ - ...options, - ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), - ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), - ...(semanticRecoveryMode === "cleanup-only" ? { recoveryPurpose: "stop_cleanup" as const } : {}), - // Hash different stable bytes without replacing the real runner artifact - // used by concurrent tests. Adoption must never execute this path. - ...(mismatchedArtifact - ? { - runnerBinary: resolve(import.meta.dirname, "../../package.json"), - runnerReconnectGraceMs: 150, - } - : {}), - ...(toolHandlerOrder && !pendingCompletion ? { recoverSemanticToolInput: recoveredTool } : {}), - resumeDynamicTools: pendingCompletion ? codexSemanticToolSpecs() : toolHandlerOrder ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : [], - resumeProviderSession: startWithoutCheckpoint ? undefined : { - driverSessionId: String(openedThread.id), - providerSessionId: mismatchedCheckpoint - ? "wrong-provider-session" - : String(openedThread.sessionId), - }, - runnerProcessLauncher: duplicateLauncher, - adoptExistingRunner: { - pid: runnerPid!, - processGroupId: runnerPid, - startedAt: runnerStartedAt, - signal, - isAlive: () => { - if (rejectLiveness) return Promise.reject(probeError); - try { - process.kill(runnerPid!, 0); - return true; - } catch { - return false; - } - }, - }, - }); - if (preparedTask && ["deny", "legacy-missing", "missing-binding"].includes(adoptionAuthorization)) { - await expect(adopted.transport.request("thread/read", {})).rejects.toThrow( - adoptionAuthorization === "deny" ? "project grant revoked" : adoptionAuthorization === "legacy-missing" - ? "runner_launch_recovery_requirements_unavailable" : "runner_launch_recovery_configuration_unavailable"); - expect(registration).not.toHaveBeenCalled(); - expect(prepareLaunch).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(signal).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - expect(authorizeRunnerAdoption).toHaveBeenCalledTimes(adoptionAuthorization === "deny" ? 1 : 0); - return; - } - if (mismatchedArtifact) { - await expect( - adopted.transport.request("thread/read", {}), - ).rejects.toThrow("native_adopted_runner_authentication_timeout"); - expect(authority?.activeRunnerConnectionCount()).toBe(0); - await expect( - adopted.transport.request("turn/start", { - input: [{ type: "text", text: "must not be dispatched" }], - }), - ).rejects.toThrow("native_adopted_runner_authentication_timeout"); - await adopted.transport.close(); - expect(signal).not.toHaveBeenCalled(); - expect(checkpoint).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - const retained = JSON.parse( - await readFile(controlPlaneStatePath, "utf8"), - ) as { - identity: unknown; - commands: Array<{ type: string; status: string }>; - }; - expect(retained.identity).toEqual(identity); - expect( - retained.commands.some( - (command) => - command.type === "runner.drain" && command.status === "pending", - ), - ).toBe(true); - expect(retained.commands.map((command) => command.type)).not.toEqual( - expect.arrayContaining(["runner.suspend"]), - ); - expect(retained.commands.map((command) => command.type)).not.toEqual( - expect.arrayContaining(["turn.stop"]), - ); - return; - } - if (mismatchedCheckpoint) { - await expect( - adopted.transport.request("thread/read", {}), - ).rejects.toThrow("native_adopted_provider_identity_mismatch"); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - return; - } - if (toolHandlerOrder?.endsWith("before")) adopted.transport.setServerRequestHandler(redispatchedTool); - await expect(adopted.transport.request(startWithoutCheckpoint ? "thread/start" : "thread/read", {})).resolves.toEqual( - expect.objectContaining({ - thread: expect.objectContaining({ id: "codex-thread-1" }), - }), - ); - if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); - if (semanticRecoveryMode) { - const retained = authority as DurablePrpControlPlane | null; - expect(retained).not.toBeNull(); - expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); - expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); - if (semanticRecoveryMode === "denied-before-cancel") { - await vi.waitFor(() => expect(recoveredTool).toHaveBeenCalledOnce()); - // This is a real Core replay callback, not an artificial transport error. - // Deliver its denial before giving the caller its cancellation turn. - releaseRecovery(); - await vi.waitFor(async () => { - await expect(adopted!.transport.request("thread/read", {})).rejects.toBe(recoveryDenied); - }); - await expect(adopted.transport.request("turn/interrupt", {})).rejects.toBe(recoveryDenied); - expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); - } else { - // Let all startup/handler microtasks and a real reconciliation read run. - // Cleanup must not enter replay even while its authority would deny it. - await adopted.transport.request("thread/read", {}); - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).not.toHaveBeenCalled(); - releaseRecovery(); - await expect(adopted.transport.request("turn/interrupt", {})).resolves.toEqual({}); - expect(retained!.store.state.commands.filter(command => command.type === "turn.interrupt")).toMatchObject([{ status: "completed" }]); - } - expect(redispatchedTool).not.toHaveBeenCalled(); - expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); - expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(duplicateLauncher).not.toHaveBeenCalled(); - if (semanticRecoveryMode === "cleanup-only") { - const faultedStopProof = vi.fn(); - if (stopExitProofFault) { - const queue = retained!.queueCommand.bind(retained!); - vi.spyOn(retained!, "queueCommand").mockImplementation((type, ...args) => { - const command = queue(type, ...args); - if (type === "turn.stop") { - let result = command.result; - Object.defineProperty(command, "result", { - enumerable: true, - configurable: true, - get: () => result, - set: value => { - const receipt = value as { result?: Record } | null; - if (receipt?.result && typeof receipt.result === "object") { - faultedStopProof(receipt.result.providerExitConfirmed); - const stopped = { ...receipt.result }; - if (stopExitProofFault === "missing") delete stopped.providerExitConfirmed; - else stopped.providerExitConfirmed = false; - result = { ...receipt, result: stopped }; - } else result = value; - }, - }); - } - return command; - }); - await expect(adopted.transport.close()).rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); - expect(faultedStopProof).toHaveBeenCalledWith(true); - expect(checkpoint).toHaveBeenCalledExactlyOnceWith("unsettled"); - } else { - await expect(adopted.transport.close()).resolves.toBeUndefined(); - expect(checkpoint).toHaveBeenCalledExactlyOnceWith("settled"); - expect(retained!.store.state.commands.filter(command => command.type === "turn.stop")).toMatchObject([{ - status: "completed", result: { result: { providerExitConfirmed: true } }, - }]); - } - expect(retained!.semanticToolResultsSettled()).toBe(false); - expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); - expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); - if (!stopExitProofFault) { - expect(retained!.store.state.commands.filter(command => command.type === "runner.suspend")).toMatchObject([{ status: "completed" }]); - const runnerState = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(runnerState).toMatchObject({ ...identity, lifecycle: "suspended" }); - } - const providerState = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); - expect(providerState).toMatchObject({ activeProviderTurnId: null, pendingEvents: [], queuedEvents: [] }); - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).not.toHaveBeenCalled(); - } - return; - } - if (toolHandlerOrder) { - await vi.waitFor(() => { - expect(pendingCompletion ? redispatchedTool : recoveredTool).toHaveBeenCalledOnce(); - const result = authority!.store.state.commands.find(command => command.type === "semantic_tool.result"); - expect(result?.status).toBe("completed"); - }, { timeout: 5_000 }); - if (pendingCompletion) { - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).toHaveBeenCalledWith(expect.objectContaining({ params: expect.objectContaining({ tool: "paperclip_finish" }) })); - } else expect(redispatchedTool).not.toHaveBeenCalled(); - } - expect(adopted.evidence().runnerPid).toBe(runnerPid); - expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); - if (startWithoutCheckpoint) { - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - for (const type of ["run.prepare", "session.open"]) { - expect(retained.commands.filter((command: { type: string }) => command.type === type)).toHaveLength(1); - } - } - if (goalMidTurn) { - const observed = await Promise.race([ - (async () => { - for await (const notification of adopted!.transport.notifications()) { - if (notification.method === "item/started") return notification; - } - throw new Error("recovered goal item was lost"); - })(), - new Promise((_, reject) => setTimeout(() => reject(new Error("recovered item timed out")), 5_000)), - ]); - expect(observed.params).toMatchObject({ threadId: "codex-thread-1", turnId: "provider-goal-turn-1" }); - } - expect(duplicateLauncher).not.toHaveBeenCalled(); - if (preparedTask) { - if (adoptionAuthorization === "legacy") { - expect(prepareLaunch).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ identity }), expect.objectContaining({ configurationId: "adopted-preparation" })); - expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); - } else { - expect(prepareLaunch).not.toHaveBeenCalled(); - expect(authorizeRunnerAdoption).toHaveBeenCalledExactlyOnceWith("adopted-preparation", identity); - } - expect((await adopted.transport.request("thread/read", {})).thread).toMatchObject({ cwd: tmpdir() }); - } - if (uncertainAtClose) { - const before = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - const retainedAuthority = authority as DurablePrpControlPlane | null; - rejectLiveness = true; - await expect(adopted.transport.close()).rejects.toBe(probeError); - expect(retainedAuthority?.activeRunnerConnectionCount()).toBe(0); - expect(intervals!.mock.results.length).toBeGreaterThan(0); - for (const interval of intervals!.mock.results) if (interval.type === "return") expect(clearedIntervals).toHaveBeenCalledWith(interval.value); - const notificationsClosed = (async () => { for await (const _event of adopted!.transport.notifications()) {} return true; })(); - let queueDeadline: ReturnType | undefined; - try { - expect(await Promise.race([notificationsClosed, new Promise(resolve => { queueDeadline = setTimeout(() => resolve(false), 250); })])).toBe(true); - } finally { clearTimeout(queueDeadline); } - await expect(adopted.transport.close()).rejects.toBe(probeError); - expect(adopted.transport.processInfo()).toMatchObject({ exited: false }); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - expect(signal).not.toHaveBeenCalled(); - expect(checkpoint).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - expect(retained.identity).toEqual(before.identity); - expect(retained.commands).toEqual(before.commands); - return; - } - expect(adopted.evidence().diagnostics).toContain( - `adopted runner ${runnerPid} authenticated to its durable PRP authority`, - ); - if (!startWithoutCheckpoint) { - expect(adopted.evidence().diagnostics).toContain( - "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", - ); - expect(adopted.evidence().diagnostics).toContain( - "confirmed adopted provider identity against authenticated recovery session.snapshot", - ); - } - if (detachBeforeCleanup) { - await adopted.detachControllerForRestart(); - await adopted.transport.close("old controller finalizer"); - expect(signal).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - const commandTypes = retained.commands.map((command: { type: string }) => command.type); - expect(commandTypes).not.toContain("turn.stop"); - expect(commandTypes).not.toContain("runner.suspend"); - } - } finally { - releaseRecovery(); - await adopted?.transport.close().catch(() => undefined); - if (uncertainAtClose) { - const providerPid = adopted?.evidence().codexPid; - if (providerPid) { - try { process.kill(providerPid, "SIGKILL"); } - catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } - } - await (authority as DurablePrpControlPlane | null)?.stop(); - } - restoreIntervalSpies(); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // The adopted runner normally exits after its durable suspend command. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -} - -it( - "adopts a live runner on the same durable authority without spawning a duplicate", - () => verifyLiveRunnerAdoption(false), - 30_000, -); - -it("adopts a prepared runner after losing its controller launch cache without preparing resources again", - () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); - -it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", - mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); -it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); - -it( - "blocks adopted runner artifact drift without duplicate launch, checkpoint replacement, or process signals", - () => verifyLiveRunnerAdoption(false, true), - 15_000, -); - -it( - "rejects a live runner whose provider identity mismatches the compacted checkpoint", - () => verifyLiveRunnerAdoption(true), - 30_000, -); - -it("binds buffered mid-goal items only after the authenticated recovery snapshot", () => verifyLiveRunnerAdoption(false, false, true), 30_000); - -it("keeps an adopted runner alive when the detached controller finalizer closes", () => verifyLiveRunnerAdoption(false, false, true, true), 30_000); - -it("adopts an opening session without a checkpoint instead of bootstrapping a duplicate provider", () => verifyLiveRunnerAdoption(false, false, false, false, true), 30_000); - it("cancels startup readiness before releasing the route that settles its ready callback", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-ready-release-cycle-")); let releaseReady!: () => void; @@ -9470,52 +5432,6 @@ it("rejects the notification stream promptly when runnerd exits after accepting } }, 30_000); -it("persists an active provider as settled before bounded suspension", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-active-suspension-"), - ); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, "--linger-after-turn-start"), - stateDirectory, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("initialize", {}); - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - await bundle.transport.request("turn/start", { - input: [{ type: "text", text: "Wait for another instruction." }], - }); - const notifications = bundle.transport - .notifications() - [Symbol.asyncIterator](); - await expectTurnStarted(notifications); - await bundle.transport.close(); - - const providerState = JSON.parse( - await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - "utf8", - ), - ) as Record; - expect(providerState).toMatchObject({ - lifecycle: "prepared", - activeProviderTurnId: null, - }); - } finally { - await bundle.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - - it.each(["claude", "codex"] as const)("keeps the explicitly assigned gateway in the %s runner environment", (agent) => { const gateway = { PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned", @@ -9550,7 +5466,6 @@ it("resolves explicit skills to the remote provider home and rejects unassigned expect(() => resolveRunnerdCodexSkillInputs([skill], null, "/runner/codex-home")).toThrow("assigned runtime skill"); }); - it("admits only exact live Pi turn controls", () => { expect(parseAcpxTurnControlCapabilities(undefined, "pi")).toEqual({ steering: false, queuedFollowUp: false }); expect(parseAcpxTurnControlCapabilities({ steering: true, queuedFollowUp: true }, "pi")).toEqual({ steering: true, queuedFollowUp: true }); @@ -9843,7 +5758,6 @@ it.each([ }); }, 30_000); - it.each([ { provider: "codex", acpxAgent: "cursor", acpxMode: "plan" }, { provider: "acpx", acpxAgent: "copilot", acpxMode: "" }, @@ -9856,11 +5770,11 @@ it.each([ it.each([undefined, "medium", "minimal", "xhigh", null])("rejects non-exact Pi transport thinking level %s before spawn", piThinkingLevel => { expect(() => createCapabilityRunnerdCodexTransport({ provider: "acpx", acpxAgent: "pi", piThinkingLevel: piThinkingLevel as never })).toThrow(/thinking/); }); + it("rejects Pi thinking settings on a different transport provider", () => { expect(() => createCapabilityRunnerdCodexTransport({ provider: "codex", piThinkingLevel: "low" })).toThrow(/only supported/); }); - describe("cold process admission budget", () => { it("bounds Pi cold startup and recovery at 60 seconds", () => { expect(coldAdmissionTimeoutMs("acpx", "pi", true)).toBe(60_000); @@ -9929,7 +5843,6 @@ it("reports a generic failure when an ACPX rejection does not match Runner's clo expect((failure as Error).message).not.toContain("Private provider explanation."); }); - it("shares Pi's absolute cold deadline across recovery barriers and rejects a late ACK", async () => { vi.useFakeTimers(); try { @@ -9978,6 +5891,7 @@ it("aborts startup promptly on close while allowing the owned cleanup command to await drain; } finally { vi.useRealTimers(); } }); + it("launches and closes native Codex without materializing or editing the preconfigured home", async () => { const root = await mkdtemp(join(tmpdir(), "runner-preconfigured-launch-")); const home = join(root, "user-home"); @@ -10005,19 +5919,3 @@ it("launches and closes native Codex without materializing or editing the precon await rm(root, { force: true, recursive: true }); } }, 30_000); - - -it.each(["before", "after"] as const)("settles a saved tool outcome with the handler installed %s recovery startup", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); - -it.each(["completion-before", "completion-after"] as const)("recovers a pending finish without an ordinary-tool lookup (%s)", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); - -it("exposes semantic recovery before cancellation when the authority has been revoked", () => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "denied-before-cancel"), 30_000); - -it.each(["before", "after", "completion-before", "completion-after"] as const)("allows cleanup-only adoption to cancel without replaying a pending semantic input (%s)", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order, false, "cleanup-only"), 30_000); - -it.each(["missing", "false"] as const)("refuses cleanup settlement when the exact stop receipt has %s provider exit proof", fault => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "cleanup-only", fault), 30_000); diff --git a/packages/paperclip-runner/test/support/runnerd-transport.ts b/packages/paperclip-runner/test/support/runnerd-transport.ts new file mode 100644 index 00000000000..171414f2415 --- /dev/null +++ b/packages/paperclip-runner/test/support/runnerd-transport.ts @@ -0,0 +1,88 @@ +import { join, resolve } from "node:path"; +import { expect } from "vitest"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + nativeRuntimePromptDigest, + type NativeRuntimeContextSnapshot, +} from "../../src/contracts/runtime-context.js"; +import { defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "../../src/live/runnerd-codex-transport.js"; + +export const defaultCapabilityRunnerdBinary = () => + process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + qualifiedCapabilityRunnerdBinary(); + +export async function expectTurnStarted( + notifications: AsyncIterator<{ method: string }>, +) { + for (let index = 0; index < 32; index += 1) { + const next = await notifications.next(); + expect(next.done).not.toBe(true); + if (next.value.method === "turn/started") return; + // Startup ownership and capability facts can precede the active turn. + expect(next.value.method).toBe("paperclip/canonicalProviderEvent"); + } + throw new Error( + "provider turn did not start within the bounded notification prefix", + ); +} + +export const fakeCodex = resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", +); + +export function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { + return [ + "--state-file", + join(stateDirectory, "fake-codex-state.json"), + ...args, + ]; +} + +export function assignedRuntimeContext( + skillRoot: string, + instructionRoot: string, +): NativeRuntimeContextSnapshot { + const digest = "0".repeat(64); + const value = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath: "AGENTS.md", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: instructionRoot, + fileCount: 1, + totalBytes: 1, + }, + }, + skills: [ + { + key: "company/assigned", + runtimeName: "assigned", + versionId: "version-1", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: skillRoot, + fileCount: 1, + totalBytes: 1, + }, + }, + ], + mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" }, + } satisfies Omit; + return { + ...value, + aggregateDigest: canonicalNativeRuntimeContextDigest(value), + }; +} diff --git a/packages/paperclip-runner/vitest.config.ts b/packages/paperclip-runner/vitest.config.ts index 3514fa1ea01..4c5817aec31 100644 --- a/packages/paperclip-runner/vitest.config.ts +++ b/packages/paperclip-runner/vitest.config.ts @@ -1,7 +1,9 @@ +import { availableParallelism } from "node:os"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { configDefaults, defineConfig } from "vitest/config"; +import { RunnerSequencer } from "./scripts/vitest-sequencer.mjs"; const packageRoot = fileURLToPath(new URL(".", import.meta.url)); @@ -17,6 +19,8 @@ export default defineConfig({ ? { reporters: [...configDefaults.reporters, resolve(packageRoot, "../../scripts/ci-profile/vitest-reporter.mjs")] } : {}), environment: "node", + ...(process.env.GITHUB_ACTIONS === "true" ? { maxWorkers: Math.min(4, availableParallelism()) } : {}), + sequence: { sequencer: RunnerSequencer }, include: [ "src/**/*.test.ts", "src/**/*.test.tsx", @@ -27,6 +31,7 @@ export default defineConfig({ "examples/scenario-explorer/src/**/*.test.tsx", "scripts/local-runner-browser-server.test.mjs", "scripts/live-console-browser-server.test.mjs", + "scripts/vitest-sequencer.test.mjs", ], }, }); diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 3ca0bc25840..a507f3034f8 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -966,7 +966,9 @@ test('native dependency caches are target-bound and unmerged master callers cann const cache = producer.split(' - name: Cache native Rust dependencies\n')[1].split('\n - ')[0]; assert.match(cache, /Swatinem\/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6/); assert.match(cache, /workspaces: \$\{\{ steps.native_rust_workspace.outputs.path \}\} -> target/); - assert.match(cache, /shared-key: release-native-v2-\$\{\{ matrix.host \}\}-\$\{\{ matrix.target \}\}-\$\{\{ hashFiles\('packages\/paperclip-runner\/protocol\/\*\*', 'packages\/paperclip-runner\/src\/\*\*', 'packages\/paperclip-runner\/\*.json'\) \}\}/); + const key = cache.match(/^\s*shared-key: (.+)$/m)?.[1]; + assert.ok(key?.startsWith("release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', ")); + assert.ok(key?.endsWith("-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }}")); for (const option of ['cache-workspace-crates', 'cache-bin', 'cache-on-failure']) { assert.match(cache, new RegExp(option + ': false')); } diff --git a/scripts/ci-dependency-cache.mjs b/scripts/ci-dependency-cache.mjs index b447c01d362..cca636cfb45 100644 --- a/scripts/ci-dependency-cache.mjs +++ b/scripts/ci-dependency-cache.mjs @@ -7,11 +7,19 @@ import { readPluginsUnder } from "./link-plugin-dev-sdk.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const postinstallHooks = new Map([ + [".", "node scripts/link-plugin-dev-sdk.mjs"], + ["server", "node scripts/install-copilot-assets.mjs"], + ["packages/paperclip-runner", "node scripts/install-copilot-assets.mjs"], +]); + export function cachePlan({ files, read, scope, packageDirs }) { const dependencyInputs = files.filter((file) => /(^|\/)package\.json$/.test(file) || ["pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc", ".pnpmfile.cjs", "pnpmfile.cjs"].includes(file) || file.startsWith("scripts/") + || file.startsWith("server/scripts/") + || file.startsWith("packages/paperclip-runner/scripts/") || file.startsWith("patches/")); const digest = (inputs, salt) => { const hash = createHash("sha256").update(JSON.stringify(salt)); @@ -28,8 +36,8 @@ export function cachePlan({ files, read, scope, packageDirs }) { if (manifest === null) return false; const scripts = JSON.parse(manifest).scripts ?? {}; return ['preinstall', 'install', 'postinstall', 'prepare'].every((hook) => - scripts[hook] === undefined || (directory === '.' && hook === 'postinstall' - && scripts[hook] === 'node scripts/link-plugin-dev-sdk.mjs')); + scripts[hook] === undefined || (hook === 'postinstall' + && scripts[hook] === postinstallHooks.get(directory))); }); const buildInputs = files.filter((file) => file.startsWith("packages/shared/") || file.startsWith("packages/plugins/sdk/") From 630ed6624c71c26617a1e0f3c5e97384ece2caae Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:35:43 -0600 Subject: [PATCH 20/28] test(e2e): add verified dependency continuation QA (#18901) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip lets people manage work assigned to AI agents. > - Task dependencies must hold assigned work until its blockers are complete. > - The reusable Product E2E set needs a browser journey for that full path. > - The journey must verify saved records, automatic wake identity, and the result after reload. > - The original implementation passed staging QA, but its automated review workspace has an export hold. > - This replacement integrates current master and earns a separate normal review while preserving the original attempt. ## Linked Issues or Issue Description Refs #18789, the original dependency QA pull request. Related: #7542 and #17524 for dependency wake and explicit resume behavior; #18782 for the merged lifecycle registration layout. **What existing behavior does this improve?** The Product E2E catalog can verify the complete blocked-task continuation path against a configured native Cloud target. **Current behavior** The main catalog lacks this full browser journey. The original PR's automated review task has a retained workspace export hold (`native_workspace_sync_out_retry_exhausted`), with neither Retry nor Continue available. Its task, files, checks, and failed export attempts remain preserved. **Proposed behavior** Add the same independently checked `task-dependency-resume` journey through this replacement. Its new commit normally merges the actual fetched master `4e29f001396c9fc76075850165ccfa3a6743e3ef` into verified source `e53dd51407b901b14bd582eeb2071fabe6955d9d`. That integration contains 39 real upstream path updates. The replacement must pass fresh CI and automated review on its own head. **Reason and benefit** Daily QA can detect premature work, a missing or duplicate continuation, and a result from the wrong run. A separate review workspace lets the normal review process inspect the current integration while the original held workspace stays intact. **Breaking changes** The change adds an explicit journey. Existing journeys and their registrations remain present. ## What Changed - Add a browser journey that creates a human-owned blocker and dependent task, saves their relation, selects Blocked, and assigns the configured native agent. - Require a measured ten-second blocked wait, a board-owned blocker completion, and exactly one automatic successful native run. - Independently verify the saved wake source and blocker identity, configured provider/model/environment, committed native phase, document revision authorship, and reloaded result. - Return `unsupported` before observations or task creation when the native provider or model identity is missing or blank. - Attempt both evidence writes, preserve an existing exact error object and outcome, and fail otherwise successful work when required evidence cannot be saved. - Grade runtime identity, document authorship, reload visibility, and retained history from their own evidence bound to the same company, task, agent, and run. All eight checks remain required. - Keep the random review reference in the blocker description. Save allowlisted private observations and read failures through the existing evidence pipeline. - Register the case at the distinct dependency anchors in the catalog, types, checks, source hash, dispatcher, features, and documentation. The delta against fetched master is exactly the original 15 QA paths. The fixture, contract tests, shared registrations, and dispatch remain unchanged. The browser actions and observations are byte-identical to `d41558b5`; only evidence finalization changes in that flow. Six shared-file AST comparisons still match fetched master after removing only the dependency registrations. All upstream integration changes remain preserved. ## Verification Prior acceptance remains bound to its actual source: - [Staging QA run 41731870131](https://pages.paperclip.ing/evals-qa-smoke-41731870131-1/) passed all eight checks and all five phases, with closed cleanup, on `dc9ea42d46d3015c010d0532b48e0da973d3611a`. - [Original full CI run 41745149524](https://github.com/paperclipai/paperclip/actions/runs/41745149524) passed on `e53dd51407b901b14bd582eeb2071fabe6955d9d`: 48 successful jobs and one skip on attempt 1. - The original failed staging result and the original held review remain intact. These prior passes are not new checks for this replacement. Passed locally on the current replacement source with Node 24.22.2 and pinned pnpm 9.15.4: - One complete `install --frozen-lockfile --ignore-scripts`, followed by the maintained `node scripts/link-plugin-dev-sdk.mjs` for links inside this checkout. - `node scripts/ensure-plugin-build-deps.mjs` and `pnpm --filter @paperclipai/db run build`. - `node scripts/typecheck-test-projects.mjs --low-memory tests/runner-e2e`, including the current Runner TypeScript build, fresh server declarations, and E2E typecheck. - 239 focused controls across eight files. These cover missing/null/empty/blank execution identity, real filesystem write failures, synchronous writer failures, frozen original errors, and independent defect attribution. - 5,302 comparisons against the prior grader, with zero aggregate acceptance changes or new throws. The sealed actual staging observations from run 41731870131 still produce the same eight passing checks. This is finite calibration of retained evidence, not a new staging run. - Credential-free discovery: 22 unique cases, including dependency and lifecycle. - Source-preservation checks and whitespace checks for both the normal integration and the replacement delta. The focused command is: ```sh node node_modules/vitest/vitest.mjs run --config tests/runner-e2e/vitest.config.ts tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts tests/runner-e2e/product-journeys/task-dependency-resume.test.ts tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts tests/runner-e2e/product-journeys/contracts.test.ts tests/runner-e2e/product-journeys/observation-read.test.ts tests/runner-e2e/catalog.test.ts tests/runner-e2e/journey-features.test.ts ``` The old generated cache failed current-source equality and was not copied. Canonical current outputs were built and sealed with their input hashes, including CLI, UI, and provider inputs. No Rust build, QA dispatch, model call, or action on the held workspace was performed for this replacement. Review follow-up: the first replacement head `b4408c5732e1800f7ccf66c639cf382b5ff2f01a` passed full CI run [41749940687](https://github.com/paperclipai/paperclip/actions/runs/41749940687), with 48 successful jobs and one skip. The native review then found the missing execution-identity preflight. The preflight now rejects that setup before work. Completed current-head checks are recorded below. The current E2E typecheck reused the same canonical declarations only after all 11,273 build inputs, two installed toolchain files, and 6,747 generated files matched their saved hashes. The five changed case files are outside that build-input set. One sandbox test run passed 216 controls and lacked the report subprocess output. A direct `tsx` diagnostic confirmed a temporary pipe `listen EPERM`. The unchanged 217 controls then passed with ordinary local IPC permissions. Both results remain recorded. The `d41558b5807e609032bbadddb103d6b0a7db4a12` attempt [41753398775](https://github.com/paperclipai/paperclip/actions/runs/41753398775) is retained: 45 successful jobs, one skip, two failed server shards, and a failed aggregate. The runtime readiness case refused connections within its ten-second window; the issue-deadline and large repository cases hit their existing test limits. Their causes remain unknown. A source-matched Linux CI run executed and passed all three cases, and one unchanged local runtime case passed. No timeout or assertion changed and no old-head diagnostic recheck was issued. The native review then identified error masking during evidence persistence and cumulative diagnostic flags. The current head fixes those two owned-case issues. Its completed current-head checks are recorded below. Seventeen of twenty new regression controls failed before these fixes; all now pass. Two additional controls exercise synchronous writer failures. Both original review and CI evidence remain intact. The eight required checks, one-run limit, 500-cent configured budget, and bounded ten-second observation are unchanged. Fresh full CI, current-head Greptile 5/5 with no open findings, and every actual review context must complete before merge. Prior statuses are not copied or overridden. Current-head verification is complete for `d9bc055f1f6609463679dc78fdcb96034d2c3727`: - [Full CI run 41758005698](https://github.com/paperclipai/paperclip/actions/runs/41758005698), attempt 2, completed successfully at 02:26:19 UTC on November 4: 48 successful jobs and one skip. - Attempt 1 lost one exact Spot runner. AWS confirmed `Server.SpotInstanceTermination` and `instance-terminated-no-capacity`; its log recorded the shutdown. That failed job and the failed aggregate remain preserved. - One normal failed-jobs recheck ran only the interrupted static checks (02:19:13–02:25:54 UTC) and aggregate (02:26:13–02:26:18 UTC). The other 46 successful start/finish pairs remained unchanged. The skipped selector was not an execution. - Greptile and Paperclip Review both scored this exact head 5/5. There are no unresolved review findings. ## Risks - The staging pass remains attributed to `dc9ea42`; final admission must retain that source provenance and the current preservation proof. - The case uses one paid work run when executed. Agent setup usage remains separate from task-run accounting. - The ten-second blocked observation proves a finite wait. Missing, incomplete, foreign, premature, or duplicate evidence fails the independent checks. - The original held workspace still needs its own disposition. This PR leaves it preserved for its owner. ## Model Used OpenAI GPT-6 through Codex, with reasoning, code editing, and code execution. The exact model ID and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/evals.md | 2 + tests/runner-e2e/PRODUCT-JOURNEYS.md | 1 + tests/runner-e2e/catalog.test.ts | 1 + tests/runner-e2e/journey-features.ts | 1 + tests/runner-e2e/product-journeys/catalog.ts | 1 + tests/runner-e2e/product-journeys/launch.ts | 6 + .../product-journeys/success-contract.ts | 2 + .../task-dependency-resume-contract.test.ts | 46 +++ .../task-dependency-resume-evidence.test.ts | 123 ++++++++ .../task-dependency-resume-evidence.ts | 131 ++++++++ .../task-dependency-resume-fixture.ts | 38 +++ .../task-dependency-resume.md | 64 ++++ .../task-dependency-resume.test.ts | 285 ++++++++++++++++++ .../task-dependency-resume.ts | 243 +++++++++++++++ tests/runner-e2e/product-journeys/types.ts | 1 + 15 files changed, 945 insertions(+) create mode 100644 tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts create mode 100644 tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts create mode 100644 tests/runner-e2e/product-journeys/task-dependency-resume.md create mode 100644 tests/runner-e2e/product-journeys/task-dependency-resume.test.ts create mode 100644 tests/runner-e2e/product-journeys/task-dependency-resume.ts diff --git a/doc/evals.md b/doc/evals.md index e2db0f49f9e..8a57f31089e 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -113,6 +113,8 @@ reply with the late requirement, and persistence after reload. Its attached and managed targets share the same story and permit one steered run or one normal successor. Registration and calibration are separate from a live staging pass. +The explicit [dependency continuation journey](../tests/runner-e2e/product-journeys/task-dependency-resume.md) checks a finite blocked wait and one automatic native continuation after a browser-completed blocker. Local evidence controls are separate from a deployed QA pass. + [Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit deployment targets and starting-state recipes. The shared task-properties journey and task-document create/edit/reopen journey use no model; Cloud diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 6fb291d0ac7..1b73f1e566a 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -30,6 +30,7 @@ never start a server or depend on a tenant filesystem. | `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | +| `task-dependency-resume` | Native agent; one automatic work run; two browser-created tasks | Keep an assigned dependent blocked for a measured ten seconds; complete its human-owned blocker through the UI; require one automatic native wake, run-authored saved document, and reloaded result. See [contract](product-journeys/task-dependency-resume.md). | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | | `chat-continuity` | Fresh owned native agent with no prior work runs; Agent Chat enabled; fresh owned user chat; three provider turns and one synthetic reset | Recall prompt-hidden context after reopening, then use `/new` to advance the same saved conversation to a fresh native session, preserve history and verify the final reply after reload. See [CHAT-CONTINUITY.md](product-journeys/CHAT-CONTINUITY.md) | | `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 4ab07867b7f..e5e598a5e78 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -135,6 +135,7 @@ describe("runner E2E catalog", () => { "file-input-answer", "task-create", "project-task-context", + "task-dependency-resume", "ask-question", "chat-continuity", "question-answer-resume", diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index b964103c45d..1816d62bd96 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -21,6 +21,7 @@ function associate(suites: readonly string[], cases: Record = { "task-create": ["browser-created-task-persisted", "no-agent-work"], "task-document-persistence": ["browser-document-created", "document-revision-history-preserved", "document-reopened-after-reload", "no-agent-work"], "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], + "task-dependency-resume": ["dependency-browser-created-and-linked", "dependency-unresolved-prevented-execution", "dependency-blocker-completed-by-board", "dependency-one-automatic-resume", "dependency-output-authored-by-run", "dependency-result-reopened", "dependency-history-preserved", "configured-model-and-environment"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], "chat-continuity": ["chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts new file mode 100644 index 00000000000..5c997d832be --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { parseTargetConfig, productJourneys } from "./catalog.js"; +import { agentJourneyRunCountValid, journeyRequiredChecks } from "./success-contract.js"; +import { TASK_DEPENDENCY_RESUME_CHECKS } from "./task-dependency-resume-evidence.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import type { RunnerE2EResult } from "../types.js"; + +const caseId = "task-dependency-resume"; +const configuration = () => ({ version: 1, id: "qa-fixture", mode: "existing-cloud", executionEnvironment: "daytona", + adapterModule: "/private/qa-provider.mjs", journeys: [caseId], baseline: { features: {}, connections: [] }, + limits: { durationMs: 600_000, modelBudgetCents: 500, maxRuns: 1 } }); +function retained(): RunnerE2EResult { + return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "fixture", caseId, + executionId: `product-journeys.qa-fixture.configured-agent.daytona.${caseId}`, profileId: "configured-agent", environmentId: "daytona", + provider: "opencode", model: "fixture-model", runtimeMode: "native", status: "passed", cleanup: "passed", attempt: 1, + startedAt: "2020-01-01T00:00:00Z", finishedAt: "2020-01-01T00:01:00Z", durationMs: 60_000, runIds: ["run"], + productJourney: { outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: (["authentication", "setup", "journey", "collection", "cleanup"] as const).map(phase => ({ phase, outcome: "passed" })), + checks: TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: true })), deployment: { kind: "cloud", targetId: "qa-fixture", + origin: "https://qa.example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: null, configurationHash: "fixture" } } }; +} +describe("dependency journey registration and retained acceptance", () => { + it("is explicit and requires exactly one bounded configured native run", () => { + expect(productJourneys.find(row => row.id === caseId)).toMatchObject({ paid: true, requires: ["company", "board-user", "task-write", "agent"] }); + expect(parseTargetConfig(configuration()).journeys).toEqual([caseId]); + expect(journeyRequiredChecks[caseId]).toEqual(TASK_DEPENDENCY_RESUME_CHECKS); + expect(agentJourneyRunCountValid(caseId, 1)).toBe(true); + for (const count of [0, 2]) { + const value = configuration(); value.limits.maxRuns = count; + expect(() => parseTargetConfig(value)).toThrow("agent_journeys_require_bounded_runs"); + expect(agentJourneyRunCountValid(caseId, count)).toBe(false); + } + expect(() => parseTargetConfig({ ...configuration(), limits: { ...configuration().limits, modelBudgetCents: 0 } })).toThrow(); + }); + it("accepts a complete retained measurement and rejects partial checks, runtime or cleanup", () => { + expect(() => validateRetainedRunnerResult(retained())).not.toThrow(); + for (const change of [ + (r: RunnerE2EResult) => { r.productJourney!.checks.pop(); }, + (r: RunnerE2EResult) => { r.productJourney!.checks[0]!.passed = false; }, + (r: RunnerE2EResult) => { r.runIds = []; }, + (r: RunnerE2EResult) => { r.runIds = ["run", "extra"]; }, + (r: RunnerE2EResult) => { r.runtimeMode = "none"; }, + (r: RunnerE2EResult) => { r.cleanup = "failed"; }, + ]) { const row = retained(); change(row); expect(() => validateRetainedRunnerResult(row)).toThrow(); } + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts new file mode 100644 index 00000000000..c10c27f6896 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { dependencyOutputMatches, gradeTaskDependencyResume, TASK_DEPENDENCY_RESUME_CHECKS, type DependencyEvidence } from "./task-dependency-resume-evidence.js"; +import { dependencyEvidenceFixture } from "./task-dependency-resume-fixture.js"; + +describe("task dependency resume evidence", () => { + it("accepts a finite blocked observation followed by one automatic, attributed native result", () => { + const checks = gradeTaskDependencyResume(dependencyEvidenceFixture()); + expect(checks).toEqual(TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: true }))); + }); + const changeRuns = (f: DependencyEvidence, mutate: (run: Record) => void) => { + for (const point of [f.finished, f.reopened]) for (const run of [...point.dependentRuns, ...point.agentRuns]) mutate(run); + }; + it.each([ + ["provider", (f: DependencyEvidence) => changeRuns(f, run => { run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }), "configured-model-and-environment"], + ["model", (f: DependencyEvidence) => changeRuns(f, run => { run.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }), "configured-model-and-environment"], + ["environment", (f: DependencyEvidence) => changeRuns(f, run => { run.contextSnapshot.paperclipEnvironment.id = "other"; }), "configured-model-and-environment"], + ["runtime mode", (f: DependencyEvidence) => changeRuns(f, run => { run.runtimeMode = "legacy"; }), "configured-model-and-environment"], + ["uncommitted phase", (f: DependencyEvidence) => changeRuns(f, run => { run.nativePhase = "saving"; }), "configured-model-and-environment"], + ["document author", (f: DependencyEvidence) => { f.finished.document!.createdByUserId = "user"; }, "dependency-output-authored-by-run"], + ["revision run", (f: DependencyEvidence) => { f.finished.revision!.createdByRunId = "foreign"; }, "dependency-output-authored-by-run"], + ["reload visibility", (f: DependencyEvidence) => { f.reopened.resultVisible = false; }, "dependency-result-reopened"], + ["activity coverage", (f: DependencyEvidence) => { f.reopened.activityCoverage.agent = false; }, "dependency-history-preserved"], + ["extra child work", (f: DependencyEvidence) => { f.reopened.activity.push({ runId: "run", action: "issue.created" }); }, "dependency-history-preserved"], + ["retained completion", (f: DependencyEvidence) => { f.reopened.blocker.completedAt = "2020-01-01T00:00:22.000Z"; }, "dependency-history-preserved"], + ] as const)("attributes the %s defect to its own evidence check", (_name, mutate, failedId) => { + const f = dependencyEvidenceFixture(); mutate(f); + expect(gradeTaskDependencyResume(f).filter(check => !check.passed).map(check => check.id)).toEqual([failedId]); + }); + const mutants: Array<[string, (value: DependencyEvidence) => void]> = [ + ["missing observation", f => { f.waiting = []; }], + ["short quiet window", f => { f.waiting[1]!.observedAt--; }], + ["non-monotonic observation", f => { f.waiting.reverse(); }], + ["quiet window before creation", f => { f.created.observedAt = f.waiting[0]!.observedAt + 1; }], + ["release before final quiet read", f => { f.releaseRequestedAt = f.waiting[1]!.observedAt - 1; }], + ["blocker not completed", f => { f.released.blocker.status = "todo"; }], + ["wrong board actor", f => { f.reopened.activity[0]!.actorId = "foreign"; }], + ["agent claimed board completion", f => { f.reopened.activity[0]!.actorType = "agent"; }], + ["missing board completion audit", f => { f.reopened.activity = []; }], + ["duplicate board completion", f => { f.reopened.activity.push({ ...f.reopened.activity[0] }); }], + ["wrong blocker relation", f => { f.waiting[0]!.dependent.blockedByIssueIds = ["foreign"]; }], + ["removed relation after completion", f => { f.reopened.dependent.blockedByIssueIds = []; }], + ["parked backlog instead of blocked", f => { f.waiting[0]!.dependent.status = "backlog"; }], + ["premature execution", f => { f.waiting[0]!.dependentRuns = [f.finished.dependentRuns[0]!]; }], + ["unknown work on owned agent", f => { f.waiting[0]!.agentRuns = [f.finished.agentRuns[0]!]; }], + ["blocker ran an agent", f => { f.reopened.blockerRuns = [f.finished.dependentRuns[0]!]; }], + ["dependency never resumed", f => { f.finished.dependent.status = "blocked"; }], + ["duplicate native run", f => { f.reopened.dependentRuns.push({ ...f.reopened.dependentRuns[0], id: "extra" }); }], + ["native run preceded blocker completion", f => { f.finished.dependentRuns[0]!.startedAt = "2020-01-01T00:00:19.000Z"; }], + ["manual run instead of automatic resume", f => { f.finished.dependentRuns[0]!.invocationSource = "on_demand"; }], + ["wrong wake reason", f => { f.finished.dependentRuns[0]!.contextSnapshot.wakeReason = "issue_assigned"; }], + ["mutation name used as wake source", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) r.contextSnapshot.source = "issue.blocker_done"; }], + ["missing saved wake source", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) delete r.contextSnapshot.source; }], + ["wrong resolved blocker", f => { f.finished.dependentRuns[0]!.contextSnapshot.resolvedBlockerIssueId = "foreign"; }], + ["conflicting native issue binding", f => { f.finished.dependentRuns[0]!.issueId = "foreign"; }], + ["reopened run changed status", f => { f.reopened.dependentRuns[0]!.status = "running"; }], + ["reopened native work still owns checkout", f => { f.reopened.dependent.executionRunId = "run"; }], + ["wrong provider", f => { f.finished.dependentRuns[0]!.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["wrong model", f => { f.finished.dependentRuns[0]!.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["wrong environment", f => { f.finished.dependentRuns[0]!.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["uncommitted native phase", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) r.nativePhase = "saving"; }], + ["missing native phase", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) delete r.nativePhase; }], + ["incomplete activity coverage", f => { f.reopened.activityCoverage.agent = false; }], + ["wrong saved reference", f => { f.finished.document!.body = '{"reference":"wrong","blockerIdentifier":"QA-1"}'; }], + ["wrong document task", f => { f.finished.document!.issueId = "foreign"; }], + ["board-authored document", f => { f.finished.document!.createdByUserId = "user"; }], + ["revision from another run", f => { f.finished.revision!.createdByRunId = "foreign"; }], + ["missing saved result", f => { delete f.finished.document; }], + ["missing revision", f => { delete f.finished.revision; }], + ["result not visible after reload", f => { f.reopened.resultVisible = false; }], + ["replaced document after reload", f => { f.reopened.document!.id = "replacement"; }], + ["reference leaked in task prompt", f => { f.scope.prompt += f.scope.reference; }], + ["foreign creator", f => { f.created.dependent.createdByUserId = "foreign"; }], + ["unrequested child work", f => { f.reopened.activity.push({ runId: "run", action: "issue.created" }); }], + ]; + it.each(mutants)("rejects %s", (_name, mutate) => { + const f = dependencyEvidenceFixture(); mutate(f); + expect(gradeTaskDependencyResume(f).some(check => !check.passed)).toBe(true); + }); + it("missing input never becomes a pass", () => { expect(gradeTaskDependencyResume(undefined).every(c => !c.passed)).toBe(true); }); + it.each([null, {}, { summary: "Compacted result" }])("accepts the public run projection without duplicate finalization metadata: %j", resultJson => { + const f = dependencyEvidenceFixture(); + for (const point of [f.finished, f.reopened]) for (const run of [...point.dependentRuns, ...point.agentRuns]) run.resultJson = resultJson; + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + }); + const withHistory = () => { + const f = dependencyEvidenceFixture(); + const prior = { ...structuredClone(f.finished.agentRuns[0]!), id: "prior", nativeIssueId: "prior-task", + contextSnapshot: { issueId: "prior-task" }, status: "failed", startedAt: "2019-12-31T23:00:00Z" }; + f.startingAgentRuns = [prior]; + for (const p of [f.created, ...f.waiting, f.released, f.finished, f.reopened]) p.agentRuns.push(structuredClone(prior)); + return f; + }; + it("retains completed earlier campaign work while measuring only new runs", () => { + const f = withHistory(); + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + expect(f.reopened.agentRuns.map(run => run.id)).toEqual(["run", "prior"]); + }); + it.each([ + ["missing baseline", (f: DependencyEvidence) => { delete (f as Partial).startingAgentRuns; }], + ["foreign baseline", (f: DependencyEvidence) => { f.startingAgentRuns[0]!.agentId = "foreign"; }], + ["active baseline", (f: DependencyEvidence) => { f.startingAgentRuns[0]!.status = "running"; }], + ["baseline run disappeared", (f: DependencyEvidence) => { f.waiting[0]!.agentRuns = []; }], + ["duplicate baseline ID", (f: DependencyEvidence) => { f.startingAgentRuns.push(f.startingAgentRuns[0]!); }], + ["premature run hidden as prior work", (f: DependencyEvidence) => { f.startingAgentRuns.push(structuredClone(f.finished.dependentRuns[0]!)); }], + ["new unrelated run while blocked", (f: DependencyEvidence) => { f.waiting[0]!.agentRuns.push({ ...structuredClone(f.startingAgentRuns[0]!), id: "extra" }); }], + ] as const)("rejects %s", (_name, mutate) => { + const f = withHistory(); mutate(f); + expect(gradeTaskDependencyResume(f).some(check => !check.passed)).toBe(true); + }); + it("has no global activity length cap after complete scoped observation", () => { + const f = dependencyEvidenceFixture(); + f.reopened.activity.push(...Array.from({ length: 600 }, (_, i) => ({ id: `prior-${i}`, companyId: f.scope.companyId, + entityType: "issue", entityId: f.scope.dependentId, action: "issue.observed" }))); + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + }); + it("accepts plain or fenced JSON but rejects extra keys and prose", () => { + const s = dependencyEvidenceFixture().scope, body = JSON.stringify({ reference: s.reference, blockerIdentifier: s.blockerIdentifier }); + expect(dependencyOutputMatches(body, s)).toBe(true); + expect(dependencyOutputMatches(`\`\`\`json\n${body}\n\`\`\``, s)).toBe(true); + expect(dependencyOutputMatches(JSON.stringify({ reference: s.reference, blockerIdentifier: s.blockerIdentifier, extra: true }), s)).toBe(false); + expect(dependencyOutputMatches(`Completed ${body}`, s)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts new file mode 100644 index 00000000000..d212a1f3ce2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts @@ -0,0 +1,131 @@ +type Row = Record; +export const DEPENDENCY_IDLE_MS = 10_000; +export const TASK_DEPENDENCY_RESUME_CHECKS = [ + "dependency-browser-created-and-linked", "dependency-unresolved-prevented-execution", + "dependency-blocker-completed-by-board", "dependency-one-automatic-resume", + "dependency-output-authored-by-run", "dependency-result-reopened", + "dependency-history-preserved", "configured-model-and-environment", +] as const; + +export interface DependencyScope { + companyId: string; userId: string; agentId: string; environmentId: string; provider: string; model: string; + blockerId: string; blockerIdentifier: string; blockerTitle: string; blockerDescription: string; + dependentId: string; dependentTitle: string; prompt: string; reference: string; documentKey: string; +} +export interface DependencyPoint { + observedAt: number; blocker: Row; dependent: Row; + blockerRuns: Row[]; dependentRuns: Row[]; agentRuns: Row[]; activity: Row[]; + activityCoverage: { blocker: boolean; dependent: boolean; agent: boolean }; + document?: Row; revision?: Row; resultVisible?: boolean; +} +export interface DependencyEvidence { + scope: DependencyScope; startingAgentRuns: Row[]; created: DependencyPoint; waiting: DependencyPoint[]; + releaseRequestedAt: number; released: DependencyPoint; finished: DependencyPoint; reopened: DependencyPoint; +} +const nonempty = (value: unknown): value is string => typeof value === "string" && value.length > 0; +const date = (value: unknown) => typeof value === "string" ? Date.parse(value) : NaN; +const exactId = (rows: unknown, id: string) => Array.isArray(rows) && rows.length === 1 && rows[0] === id; +const runFacts = (run: Row) => JSON.stringify([run.id, run.companyId, run.agentId, dependencyRunIssueId(run), run.status, + run.runtimeMode, run.nativePhase, run.invocationSource, run.triggerDetail, run.startedAt, + run.contextSnapshot?.wakeReason, run.contextSnapshot?.source, run.contextSnapshot?.resolvedBlockerIssueId, + run.contextSnapshot?.blockerIssueIds, run.contextSnapshot?.paperclipEnvironment?.id, + run.runnerProfileJson?.nativeExecutionInput?.provider?.kind, run.runnerProfileJson?.nativeExecutionInput?.provider?.model]); +export function dependencyNewAgentRuns(runs: Row[], startingRuns: Row[]) { + const priorIds = new Set(startingRuns.map(run => run.id)); + return runs.filter(run => !priorIds.has(run.id)); +} +export function dependencyRunIssueId(run: Row) { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(value => value != null); + return ids.length && ids.every(nonempty) && new Set(ids).size === 1 ? ids[0] as string : undefined; +} +export function dependencyOutputMatches(body: unknown, scope: Pick) { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const value = JSON.parse(json); + return Boolean(value && !Array.isArray(value) && Object.keys(value).sort().join(",") === "blockerIdentifier,reference" + && value.reference === scope.reference && value.blockerIdentifier === scope.blockerIdentifier); + } catch { return false; } +} + +export function gradeTaskDependencyResume(value: DependencyEvidence | undefined) { + const fail = () => TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: false })); + if (!value?.scope || !Array.isArray(value.startingAgentRuns) || !value.created || !Array.isArray(value.waiting) || value.waiting.length < 2 + || !value.released || !value.finished || !value.reopened) return fail(); + const { scope: s, startingAgentRuns, created, waiting, released, finished, reopened } = value; + const points = [created, ...waiting, released, finished, reopened]; + if (Object.values(s).some(item => !nonempty(item)) || s.blockerId === s.dependentId || s.prompt.includes(s.reference) + || !s.blockerDescription.includes(s.reference) + || points.some(p => !p?.blocker || !p.dependent || !Number.isFinite(p.observedAt) + || [p.blockerRuns, p.dependentRuns, p.agentRuns, p.activity].some(rows => !Array.isArray(rows)))) return fail(); + const startingIds = startingAgentRuns.map(run => run.id); + const historyComplete = startingIds.every(nonempty) && new Set(startingIds).size === startingIds.length + && startingAgentRuns.every(run => run.companyId === s.companyId && run.agentId === s.agentId + && ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status) + && ![s.blockerId, s.dependentId].includes(dependencyRunIssueId(run) ?? "")) + && points.every(p => new Set(p.agentRuns.map(run => run.id)).size === p.agentRuns.length + && p.agentRuns.every(run => nonempty(run.id) && run.companyId === s.companyId && run.agentId === s.agentId) + && startingAgentRuns.every(prior => p.agentRuns.some(run => runFacts(run) === runFacts(prior)))); + const newRuns = (point: DependencyPoint) => dependencyNewAgentRuns(point.agentRuns, startingAgentRuns); + const scoped = historyComplete && points.every(p => p.blocker.id === s.blockerId && p.dependent.id === s.dependentId + && p.blocker.companyId === s.companyId && p.dependent.companyId === s.companyId + && p.blocker.title === s.blockerTitle && p.dependent.title === s.dependentTitle + && p.blocker.description === s.blockerDescription && p.dependent.description === s.prompt + && p.blocker.createdByUserId === s.userId && p.dependent.createdByUserId === s.userId + && !p.blocker.createdByAgentId && !p.dependent.createdByAgentId + && p.blocker.assigneeUserId === s.userId && p.blocker.assigneeAgentId === null); + const linked = scoped && [created.blocker, created.dependent].every(task => task.status === "todo") + && created.dependent.assigneeUserId === s.userId && created.dependent.assigneeAgentId === null + && [created.blockerRuns, created.dependentRuns, newRuns(created)].every(rows => rows.length === 0) + && points.slice(1).every(p => p.dependent.assigneeAgentId === s.agentId && p.dependent.assigneeUserId === null + && exactId(p.dependent.blockedByIssueIds, s.blockerId)); + const held = linked && waiting.every(p => p.blocker.status === "todo" && p.dependent.status === "blocked" + && !p.dependent.executionRunId && !p.dependent.checkoutRunId + && [p.blockerRuns, p.dependentRuns, newRuns(p)].every(rows => rows.length === 0)) + && waiting.every((p, index) => index === 0 || p.observedAt >= waiting[index - 1]!.observedAt) + && waiting[waiting.length - 1]!.observedAt - waiting[0]!.observedAt >= DEPENDENCY_IDLE_MS + && waiting[0]!.observedAt >= created.observedAt + && Number.isFinite(value.releaseRequestedAt) && value.releaseRequestedAt >= waiting[waiting.length - 1]!.observedAt + && released.observedAt >= value.releaseRequestedAt && finished.observedAt >= released.observedAt && reopened.observedAt >= finished.observedAt; + const completedByBoard = held && [released, finished, reopened].every(p => p.blocker.status === "done" + && Number.isFinite(date(p.blocker.completedAt))) + && reopened.activity.filter(row => row.companyId === s.companyId && row.entityType === "issue" + && row.entityId === s.blockerId && row.action === "issue.updated" && row.actorType === "user" + && row.actorId === s.userId && row.details?.status === "done").length === 1; + const run = finished.dependentRuns[0]; + const oneRun = completedByBoard && finished.dependent.status === "done" && reopened.dependent.status === "done" + && !finished.dependent.executionRunId && !finished.dependent.checkoutRunId + && !reopened.dependent.executionRunId && !reopened.dependent.checkoutRunId + && finished.dependentRuns.length === 1 && reopened.dependentRuns.length === 1 + && newRuns(finished).length === 1 && newRuns(reopened).length === 1 + && nonempty(run?.id) && [reopened.dependentRuns[0], newRuns(finished)[0], newRuns(reopened)[0]].every(row => runFacts(row) === runFacts(run)) + && run.companyId === s.companyId && run.agentId === s.agentId && dependencyRunIssueId(run) === s.dependentId + && run.status === "succeeded" && run.invocationSource === "automation" && run.triggerDetail === "system" + && run.contextSnapshot?.wakeReason === "issue_blockers_resolved" && run.contextSnapshot?.source === "issue.blockers_resolved" + && run.contextSnapshot?.resolvedBlockerIssueId === s.blockerId && exactId(run.contextSnapshot?.blockerIssueIds, s.blockerId) + && Number.isFinite(date(run.startedAt)) && date(run.startedAt) >= date(released.blocker.completedAt); + const boundRun = scoped && [run, reopened.dependentRuns[0], newRuns(finished)[0], newRuns(reopened)[0]].every(row => + nonempty(row?.id) && row.id === run?.id && row.companyId === s.companyId && row.agentId === s.agentId + && dependencyRunIssueId(row) === s.dependentId); + const runtime = boundRun && run?.runtimeMode === "native" && run.nativePhase === "committed" + && run.runnerProfileJson?.nativeExecutionInput?.provider?.kind === s.provider + && run.runnerProfileJson?.nativeExecutionInput?.provider?.model === s.model + && run.contextSnapshot?.paperclipEnvironment?.id === s.environmentId; + const documentMatches = (p: DependencyPoint) => p.document && p.revision + && nonempty(p.document.id) && p.document.companyId === s.companyId && p.document.issueId === s.dependentId + && p.document.key === s.documentKey && p.document.createdByAgentId === s.agentId && p.document.createdByUserId === null + && p.document.latestRevisionId === p.revision.id && p.revision.documentId === p.document.id + && p.revision.createdByAgentId === s.agentId && p.revision.createdByUserId === null && p.revision.createdByRunId === run?.id + && dependencyOutputMatches(p.document.body, s) && dependencyOutputMatches(p.revision.body, s); + const authored = boundRun && documentMatches(finished); + const resultReopened = boundRun && documentMatches(reopened) && finished.document && reopened.resultVisible === true + && reopened.document!.id === finished.document.id && reopened.document!.latestRevisionId === finished.document.latestRevisionId + && reopened.document!.body === finished.document.body; + const history = boundRun && points.every(p => p.blockerRuns.length === 0) + && points.every(p => p.activityCoverage?.blocker === true && p.activityCoverage?.dependent === true && p.activityCoverage?.agent === true) + && !reopened.activity.some(row => row.runId === run?.id + && ["issue.created", "issue.child_created", "project.created", "agent.created", "agent.hire_created"].includes(row.action)) + && reopened.blocker.completedAt === released.blocker.completedAt; + const values = [linked, held, completedByBoard, oneRun, authored, resultReopened, history, runtime]; + return TASK_DEPENDENCY_RESUME_CHECKS.map((id, index) => ({ id, passed: Boolean(values[index]) })); +} diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts new file mode 100644 index 00000000000..984459110b0 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts @@ -0,0 +1,38 @@ +import type { DependencyEvidence, DependencyPoint } from "./task-dependency-resume-evidence.js"; + +export function dependencyEvidenceFixture(): DependencyEvidence { + const start = Date.parse("2020-01-01T00:00:00.000Z"); + const scope = { companyId: "company", userId: "user", agentId: "agent", environmentId: "environment", provider: "opencode", model: "fixture-model", + blockerId: "blocker", blockerIdentifier: "QA-1", blockerTitle: "Review gate", blockerDescription: "The review reference is secret-reference.", + dependentId: "dependent", dependentTitle: "Continue after review", prompt: "Read QA-1 after its review and write dependency-proof.", reference: "secret-reference", documentKey: "dependency-proof" }; + const point = (observedAt: number): DependencyPoint => ({ observedAt, + blocker: { id: scope.blockerId, identifier: scope.blockerIdentifier, companyId: scope.companyId, title: scope.blockerTitle, + description: scope.blockerDescription, status: "todo", createdByUserId: scope.userId, createdByAgentId: null, + assigneeUserId: scope.userId, assigneeAgentId: null }, + dependent: { id: scope.dependentId, identifier: "QA-2", companyId: scope.companyId, title: scope.dependentTitle, + description: scope.prompt, status: "blocked", createdByUserId: scope.userId, createdByAgentId: null, + assigneeUserId: null, assigneeAgentId: scope.agentId, blockedByIssueIds: [scope.blockerId], executionRunId: null, checkoutRunId: null }, + blockerRuns: [], dependentRuns: [], agentRuns: [], activity: [], activityCoverage: { blocker: true, dependent: true, agent: true } }); + const created = point(start); + Object.assign(created.dependent, { status: "todo", assigneeAgentId: null, assigneeUserId: scope.userId, blockedByIssueIds: [] }); + const released = point(start + 20_000); Object.assign(released.blocker, { status: "done", completedAt: "2020-01-01T00:00:20.000Z" }); + const run = { id: "run", companyId: scope.companyId, agentId: scope.agentId, nativeIssueId: scope.dependentId, + status: "succeeded", runtimeMode: "native", nativePhase: "committed", invocationSource: "automation", triggerDetail: "system", + startedAt: "2020-01-01T00:00:21.000Z", + contextSnapshot: { issueId: scope.dependentId, taskId: scope.dependentId, wakeReason: "issue_blockers_resolved", source: "issue.blockers_resolved", + resolvedBlockerIssueId: scope.blockerId, blockerIssueIds: [scope.blockerId], paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } } }; + const document = { id: "document", companyId: scope.companyId, issueId: scope.dependentId, key: scope.documentKey, + body: JSON.stringify({ reference: scope.reference, blockerIdentifier: scope.blockerIdentifier }), latestRevisionId: "revision", + createdByAgentId: scope.agentId, createdByUserId: null }; + const revision = { id: "revision", documentId: document.id, body: document.body, createdByAgentId: scope.agentId, + createdByUserId: null, createdByRunId: run.id }; + const finished = structuredClone(released); finished.observedAt = start + 30_000; finished.dependent.status = "done"; + finished.dependentRuns = [structuredClone(run)]; finished.agentRuns = [structuredClone(run)]; + finished.document = document; finished.revision = revision; + const reopened = structuredClone(finished); reopened.observedAt = start + 31_000; reopened.resultVisible = true; + reopened.activity = [{ id: "completion", companyId: scope.companyId, actorType: "user", actorId: scope.userId, + action: "issue.updated", entityType: "issue", entityId: scope.blockerId, details: { status: "done" } }]; + return { scope, startingAgentRuns: [], created, waiting: [point(start + 1_000), point(start + 11_000)], releaseRequestedAt: start + 19_000, + released, finished, reopened }; +} diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.md b/tests/runner-e2e/product-journeys/task-dependency-resume.md new file mode 100644 index 00000000000..f8f18dee135 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.md @@ -0,0 +1,64 @@ +# Blocked task resumes after its human-owned blocker + +`task-dependency-resume` is an explicit Product E2E journey for an authenticated +existing Cloud target or a target provider with a qualified native agent. It uses +the ordinary task UI and public observation APIs. Its registration is not evidence +of a local or deployed pass. + +The browser creates two tasks assigned to the signed-in person. The blocker holds +a random review reference absent from the dependent's prompt. On the dependent, +the person adds the blocker, selects Blocked, and assigns the disposable agent. +Closed properties panes are opened through their visible toolbar control; mounted +inert or accessibility-hidden rows cannot substitute for an interactive pane. +Status choices are bound to the matching labeled glyph inside the open picker. +The saved dependency, blocked status, and empty task/agent run histories must +remain observable for at least ten seconds. This is a finite measured wait, +not an unlimited claim that no execution can ever occur. + +The person completes the blocker through the task's status control. Paperclip must +then start exactly one native run automatically. Its durable wake must name the +same resolved blocker, reason `issue_blockers_resolved`, and saved source +`issue.blockers_resolved`; `blocker_done` is the producer's separate mutation +payload. Manual +Run or a follow-up message cannot substitute. The run starts after the saved +blocker completion, uses the configured provider/model/environment, commits its +result through the durable committed native phase, and completes the dependent. +The public API may omit or compact `resultJson`; duplicate finalization metadata +in that optional object is not required. A task document named `dependency-proof` +contains the review reference and blocker identifier. Its current revision must +be authored by this agent and run. Reloading the task must show that same saved +revision and preserve the two tasks' descriptions, dependency, and board-owned +blocker completion audit. No agent work is permitted on the human-owned blocker. + +The case requires `company`, `board-user`, `task-write`, and `agent`. It declares +one work run, uses the existing bounded case budget, and shares the original +campaign deadline. The native agent must have nonblank provider and model +identifiers before any observation read or task creation. An incomplete execution +identity returns `unsupported` without task or agent work. +It saves the agent's starting run history and retains those +records while checking only newly appearing runs for this case. Earlier terminal +campaign work stays in the accounting and does not count as new execution. +Owned-task activity reads are complete and uncapped; the separate agent-filtered +activity read fails closed if its 500-row bound is reached. The agent run list +likewise fails closed at the public API's 1000-row cap. Unrelated company history +does not enter these completeness checks. +Agent setup usage is outside task-run accounting; missing +provider prices remain unknown. The owning target provider controls normal +cancellation, termination, retained history, and evidence cleanup. It never +changes a shared source agent, company budget, or execution environment. + +The private evidence folder contains `task-dependency-observations.json` and +`observation-read-failures.json`, with allowlisted owned task, run, audit, and +document facts. The normal launcher captures the final owned task screenshot. +Both evidence writes are attempted. A write failure retains an existing journey +error and its outcome; a previously successful journey fails if required evidence +cannot be saved. Diagnostics name only the affected evidence file. +Runtime identity, document authorship, reload visibility, and retained history +use their own observations bound to the same company, task, agent, and run. One +failed control does not mark unrelated controls failed. All eight checks remain +required for acceptance. +There are no direct database writes, private runner hooks, manually dispatched +resumption wakes, retry loops for failed model work, or fixture-authored outputs. +Independent controls reject premature or duplicate execution, a never-resumed +or manually started task, foreign/changed runtime identity, missing completion +audit, wrong output or revision authorship, and missing or shortened observations. diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts new file mode 100644 index 00000000000..fa694af7d1d --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts @@ -0,0 +1,285 @@ +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; +import { dependencyEvidenceFixture } from "./task-dependency-resume-fixture.js"; +import * as observedReader from "./observation-read.js"; + +const controls = vi.hoisted(() => ({ create: vi.fn() })); +vi.mock("../user-actions.js", () => ({ createTaskThroughUi: (...args: unknown[]) => controls.create(...args) })); +vi.mock("./task-document-flow.js", () => ({ openDocumentAuthoring: vi.fn(async () => {}) })); +vi.mock("@playwright/test", () => ({ expect: Object.assign((locator: any) => ({ + async toBeVisible() { expect(await locator.isVisible()).toBe(true); }, + async toBeInViewport() { expect(await locator.isVisible()).toBe(true); }, + async toContainText(text: string) { expect(await locator.textContent()).toContain(text); }, +}), { poll: (read: () => Promise) => ({ + async toBe(value: unknown) { expect(await read()).toBe(value); }, + async toEqual(value: unknown) { expect(await read()).toEqual(value); }, +}) }) })); +import { taskDependencyResume } from "./task-dependency-resume.js"; + +interface FixtureOptions { priorRuns?: number; unrelatedActivity?: number; ownedActivity?: number; agentActivity?: number; missingActivity?: boolean; activePrior?: boolean; + closedPanel?: "properties" | "toggle"; missingProperty?: boolean; missingStatusChoice?: boolean } +async function fixture(premature = false, options: FixtureOptions = {}) { + const directory = await mkdtemp(path.join(os.tmpdir(), "dependency-flow-")); + const f = dependencyEvidenceFixture(), s = f.scope; + const blocker = { ...f.created.blocker }, dependent = { ...f.created.dependent }; + const actions: string[] = [], owned: any[] = [], writes: unknown[] = [], reads: string[] = []; + let completed = false, assigned = false, panelOpen = !options.closedPanel; + const reference = () => /review reference is ([a-f0-9]{32})/.exec(blocker.description)?.[1]; + const output = () => JSON.stringify({ reference: reference(), blockerIdentifier: s.blockerIdentifier }); + const runs = () => assigned && (completed || premature) ? [structuredClone(f.finished.dependentRuns[0]!)] : []; + const priorRuns = Array.from({ length: options.priorRuns ?? 0 }, (_, i) => ({ ...structuredClone(f.finished.agentRuns[0]!), + id: `prior-${i}`, nativeIssueId: `prior-task-${i}`, contextSnapshot: { issueId: `prior-task-${i}` }, + startedAt: "2019-12-31T23:00:00Z", status: options.activePrior ? "running" : i % 2 ? "failed" : "succeeded" })); + const activity = (count: number, entityId: string, agentId?: string) => Array.from({ length: count }, (_, i) => ({ + id: `${entityId}-activity-${i}`, companyId: s.companyId, entityType: "issue", entityId, agentId, + actorType: agentId ? "agent" : "user", actorId: agentId ?? s.userId, action: "issue.observed" })); + const taskDocument = () => ({ ...f.finished.document, body: output() }); + const revision = () => ({ ...f.finished.revision, body: output() }); + controls.create.mockImplementation(async (input: any) => { + await input.beforeDispatch?.(); + const row = input.title.includes(" blocker ") ? blocker : dependent; + Object.assign(row, { title: input.title, description: input.prompt }); + actions.push(`create:${row.id}`); + return { issueId: row.id, submittedAtMs: Date.now() }; + }); + const locator = (key: string): any => ({ + key, + filter(value: any) { + if (key === '[data-slot="popover-content"]' && value?.has) { + return locator(options.missingStatusChoice ? "missing-status-choice" : value.has.key); + } + return this; + }, first() { return this; }, last() { return this; }, + locator() { return this; }, + getByRole(_role: string, options?: any) { return locator(options?.name ? String(options.name) : key); }, + async isVisible() { + if (key.includes("data-property-row")) return panelOpen && !options.missingProperty; + if (key === "Toggle side panel") return !panelOpen && options.closedPanel === "toggle"; + if (key.includes("Show properties")) return !panelOpen && options.closedPanel === "properties"; + return true; + }, + async scrollIntoViewIfNeeded() { actions.push(`scroll:${key}`); }, + async fill(value: string) { actions.push(`fill:${key}:${value}`); }, + async click() { + actions.push(`click:${key}`); + if (key === "Toggle side panel" || key.includes("Show properties")) panelOpen = true; + if (key === "missing-status-choice") throw new Error("missing_status_choice"); + if (key === "blocker-choice") dependent.blockedByIssueIds = [blocker.id]; + if (key === "Blocked") dependent.status = "blocked"; + if (key === "agent-choice") { assigned = true; dependent.assigneeAgentId = s.agentId; dependent.assigneeUserId = null; } + if (key === "Done") { + completed = true; blocker.status = "done"; blocker.completedAt = "2020-01-01T00:00:20.000Z"; + dependent.status = "done"; + } + }, + async textContent() { return output(); }, + }); + const page: any = { + locator(selector: string) { return locator(selector); }, + getByRole(_role: string, options?: any) { + if (_role === "button" && options?.exact && ["Blocked", "Done"].includes(options.name)) return locator("missing-status-choice"); + const l = locator(options?.name ? String(options.name) : "buttons"); + l.filter = (options: any) => options?.hasText === blocker.title ? locator("blocker-choice") + : options?.hasText === "Fixture agent" ? locator("agent-choice") : l; + return l; + }, + getByPlaceholder(name: string) { return locator(name); }, + getByTestId(name: string) { return locator(name); }, + async goto(url: string) { actions.push(`goto:${url}`); }, async reload() { actions.push("reload"); }, + }; + const api: any = { origin: "https://qa.example.test", async get(route: string) { + reads.push(route); + if (route === `/api/issues/${s.blockerId}`) return structuredClone(blocker); + if (route === `/api/issues/${s.dependentId}`) return structuredClone(dependent); + if (route === `/api/issues/${s.blockerId}/runs`) return []; + if (route === `/api/issues/${s.dependentId}/runs`) return runs().map(r => ({ runId: r.id })); + if (route.includes("/heartbeat-runs?")) return [...runs(), ...structuredClone(priorRuns)]; + if (route === "/api/heartbeat-runs/run") return runs()[0]; + if (route === `/api/issues/${s.blockerId}/activity`) return completed ? structuredClone(f.reopened.activity) : []; + if (route === `/api/issues/${s.dependentId}/activity`) return activity(options.ownedActivity ?? 0, s.dependentId); + if (route === `/api/companies/${s.companyId}/activity?agentId=${s.agentId}&limit=500`) { + return options.missingActivity ? undefined : activity(options.agentActivity ?? 0, "prior-task", s.agentId); + } + if (route === `/api/companies/${s.companyId}/activity?limit=500`) return activity(options.unrelatedActivity ?? 0, "unrelated-task").slice(0, 500); + if (route === `/api/issues/${s.dependentId}/documents`) return completed ? [taskDocument()] : []; + if (route.endsWith("/documents/dependency-proof")) return taskDocument(); + if (route.endsWith("/revisions/revision")) return revision(); + throw new Error(`unexpected_read:${route}`); + }, async post(...args: unknown[]) { writes.push(args); throw new Error("unexpected_mutation"); }, + async patch(...args: unknown[]) { writes.push(args); throw new Error("unexpected_mutation"); } }; + const context = { page, api, company: { id: s.companyId, issuePrefix: "QA" }, user: { id: s.userId }, environmentId: s.environmentId, + capabilities: ["company", "board-user", "task-write", "agent"], agent: { id: s.agentId, name: "Fixture agent", provider: s.provider, model: s.model, runtimeMode: "native" } } as JourneyContext; + const session = { own: vi.fn(async resource => { owned.push(resource); }) } as unknown as TargetSession; + const evidence = { checks: [] } as unknown as JourneyEvidence; + return { directory, context, session, evidence, actions, owned, writes, reads }; +} +afterEach(() => { vi.useRealTimers(); controls.create.mockReset(); }); +describe("dependency browser flow orchestration", () => { + const evidenceFiles = ["task-dependency-observations.json", "observation-read-failures.json"] as const; + const writerFaults = ["observations", "reads", "both"] as const; + const obstructWriters = async (directory: string, fault: typeof writerFaults[number]) => { + const names = fault === "both" ? [...evidenceFiles] : [evidenceFiles[fault === "observations" ? 0 : 1]]; + await Promise.all(names.map(name => mkdir(path.join(directory, name)))); + return names; + }; + it.each(writerFaults.flatMap(fault => (["unsupported", "failed"] as const).map(outcome => [fault, outcome] as const)))( + "preserves the original %s writer failure outcome %s and attempts both evidence files", async (fault, outcome) => { + const f = await fixture(), blocked = await obstructWriters(f.directory, fault); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + const original = Object.freeze(new JourneyError("failed", "dependency_observation_control_failed")); + if (outcome === "unsupported") f.context.agent!.provider = ""; + else vi.spyOn(f.context.api, "get").mockRejectedValue(original); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + if (outcome === "unsupported") await expect(operation).rejects.toMatchObject({ outcome, code: "dependency_execution_identity_unavailable" }); + else await expect(operation).rejects.toBe(original); + for (const name of evidenceFiles.filter(name => !blocked.includes(name))) { + expect(JSON.parse(await readFile(path.join(f.directory, name), "utf8"))).toBeTypeOf("object"); + } + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", blocked); + expect(JSON.stringify(logging.mock.calls)).not.toContain(f.directory); + expect(f.actions).toEqual([]); expect(f.owned).toEqual([]); expect(f.writes).toEqual([]); + } finally { logging.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(["unsupported", "failed"] as const)("awaits the other evidence file when the read writer throws synchronously after %s", async outcome => { + const f = await fixture(), createReader = observedReader.createObservedReader; + const original = Object.freeze(new JourneyError("failed", "dependency_observation_control_failed")); + const failingWrite = vi.fn(() => { throw new Error("private writer diagnostic"); }); + const reader = vi.spyOn(observedReader, "createObservedReader").mockImplementation((...args) => ({ + ...createReader(...args), preserve: failingWrite, + })); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + if (outcome === "unsupported") f.context.agent!.model = ""; + else vi.spyOn(f.context.api, "get").mockRejectedValue(original); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + if (outcome === "unsupported") await expect(operation).rejects.toMatchObject({ outcome, code: "dependency_execution_identity_unavailable" }); + else await expect(operation).rejects.toBe(original); + expect(failingWrite).toHaveBeenCalledOnce(); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", + outcome === "unsupported" ? "prerequisites" : "observe_starting_runs"); + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", ["observation-read-failures.json"]); + expect(JSON.stringify(logging.mock.calls)).not.toContain("private writer diagnostic"); + } finally { logging.mockRestore(); reader.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(writerFaults)("fails a successful journey when its %s evidence writer fails", async fault => { + const f = await fixture(), blocked = await obstructWriters(f.directory, fault); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const rejection = expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toMatchObject({ + outcome: "failed", code: "dependency_evidence_preservation_failed", + }); + await vi.advanceTimersByTimeAsync(11_000); await rejection; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", blocked); + for (const name of evidenceFiles.filter(name => !blocked.includes(name))) { + expect(JSON.parse(await readFile(path.join(f.directory, name), "utf8"))).toBeTypeOf("object"); + } + expect(f.writes).toEqual([]); + } finally { logging.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each((["provider", "model"] as const).flatMap(field => [ + [field, "missing", undefined], [field, "null", null], [field, "empty", ""], [field, "blank", " \t "], + ] as const))("rejects a %s identity that is %s before observation or task work", async (field, _kind, value) => { + const f = await fixture(); + Object.assign(f.context.agent!, { [field]: value }); + const get = vi.spyOn(f.context.api, "get").mockRejectedValue(new Error("preflight_read_attempted")); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toMatchObject({ + outcome: "unsupported", code: "dependency_execution_identity_unavailable", + }); + expect(get).not.toHaveBeenCalled(); expect(controls.create).not.toHaveBeenCalled(); + expect(f.actions).toEqual([]); expect(f.owned).toEqual([]); expect(f.writes).toEqual([]); + expect(f.evidence.checks).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", "prerequisites"); + expect(JSON.parse(await readFile(path.join(f.directory, "observation-read-failures.json"), "utf8"))).toEqual({ failures: [] }); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["empty history", {}], + ["completed earlier campaign runs", { priorRuns: 3 }], + ["more than 100 earlier campaign runs", { priorRuns: 150 }], + ["more than 500 unrelated company events", { unrelatedActivity: 800 }], + ["more than 500 owned task events", { ownedActivity: 600 }], + ["prior agent events", { priorRuns: 2, agentActivity: 300 }], + ] as const)("uses browser actions with %s and never writes output through API observations", async (_name, options) => { + const f = await fixture(false, options); vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + await vi.advanceTimersByTimeAsync(11_000); await operation; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.writes).toEqual([]); + expect(f.actions.indexOf("click:Blocked")).toBeLessThan(f.actions.indexOf("click:agent-choice")); + expect(f.actions.indexOf("click:agent-choice")).toBeLessThan(f.actions.indexOf("click:Done")); + expect(f.owned).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind: "issue", id: "blocker", journeyId: "task-dependency-resume" }), + expect.objectContaining({ kind: "issue", id: "dependent", journeyId: "task-dependency-resume" }), + ])); + const saved = JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8")); + expect(saved.stage).toBe("complete"); expect(saved.scope.prompt).not.toContain(saved.scope.reference); + expect(saved.waiting.at(-1).observedAt - saved.waiting[0].observedAt).toBeGreaterThanOrEqual(10_000); + expect(saved.startingAgentRuns).toHaveLength("priorRuns" in options ? options.priorRuns : 0); + expect(saved.reopened.agentRuns).toHaveLength(saved.startingAgentRuns.length + 1); + expect(f.reads.some(route => route.startsWith("/api/heartbeat-runs/prior-"))).toBe(false); + expect(f.reads).toContain("/api/issues/blocker/activity"); expect(f.reads).toContain("/api/issues/dependent/activity"); + expect(f.reads).not.toContain("/api/companies/company/activity?limit=500"); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(["properties", "toggle"] as const)("opens a closed properties pane through its %s control before linking", async closedPanel => { + const f = await fixture(false, { closedPanel }); vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + await vi.advanceTimersByTimeAsync(11_000); await operation; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const opener = f.actions.findIndex(action => action.startsWith("click:") && (action.includes("Show properties") || action === "click:Toggle side panel")); + expect(opener).toBeGreaterThanOrEqual(0); expect(opener).toBeLessThan(f.actions.indexOf("click:blocker-choice")); + expect(f.actions.some(action => action.startsWith("scroll:[data-property-row]:not([inert] *):not([aria-hidden=\"true\"] *)"))).toBe(true); + expect(f.writes).toEqual([]); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["missing property", { missingProperty: true }, "link_blocker"], + ["missing matching status glyph", { missingStatusChoice: true }, "set_dependent_blocked"], + ] as const)("fails closed on %s before agent assignment or blocker completion", async (_name, options, stage) => { + const f = await fixture(false, options); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow(); + expect(f.actions).not.toContain("click:agent-choice"); expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", stage); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["truncated agent activity", { agentActivity: 500 }, "dependency_activity_coverage_unavailable"], + ["missing agent activity", { missingActivity: true }, "dependency_activity_coverage_unavailable"], + ["active earlier work", { priorRuns: 1, activePrior: true }, "dependency_starting_agent_not_idle"], + ["saturated run history", { priorRuns: 1000 }, "dependency_run_coverage_unavailable"], + ] as const)("rejects %s before completing the blocker", async (_name, options, code) => { + const f = await fixture(false, options); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow(code); + expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it("premature work fails before the browser completes the blocker and preserves its observations", async () => { + const f = await fixture(true); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow("dependency_ran_before_blocker_completed"); + expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", "observe_blocked_wait"); + expect(JSON.parse(await readFile(path.join(f.directory, "observation-read-failures.json"), "utf8"))).toEqual({ failures: [] }); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.ts b/tests/runner-e2e/product-journeys/task-dependency-resume.ts new file mode 100644 index 00000000000..ed13e309878 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.ts @@ -0,0 +1,243 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { HEARTBEAT_RUN_LIST_MAX_LIMIT } from "../../../packages/shared/src/constants.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { DEPENDENCY_IDLE_MS, dependencyNewAgentRuns, gradeTaskDependencyResume, type DependencyEvidence, type DependencyPoint, type DependencyScope } from "./task-dependency-resume-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "task-dependency-resume" as const; +const taskSnapshot = (task: Row) => ({ id: task.id, companyId: task.companyId, identifier: task.identifier, + title: task.title, description: task.description, status: task.status, assigneeAgentId: task.assigneeAgentId, + assigneeUserId: task.assigneeUserId, createdByUserId: task.createdByUserId, createdByAgentId: task.createdByAgentId, + completedAt: task.completedAt, executionRunId: task.executionRunId, checkoutRunId: task.checkoutRunId, + blockedByIssueIds: task.blockedByIssueIds ?? task.blockedBy?.map((row: Row) => row.id) ?? [] }); +const runSnapshot = (run: Row) => ({ id: run.id, companyId: run.companyId, agentId: run.agentId, + issueId: run.issueId, nativeIssueId: run.nativeIssueId, status: run.status, runtimeMode: run.runtimeMode, + nativePhase: run.nativePhase, invocationSource: run.invocationSource, triggerDetail: run.triggerDetail, startedAt: run.startedAt, + contextSnapshot: { issueId: run.contextSnapshot?.issueId, taskId: run.contextSnapshot?.taskId, + source: run.contextSnapshot?.source, wakeReason: run.contextSnapshot?.wakeReason, + resolvedBlockerIssueId: run.contextSnapshot?.resolvedBlockerIssueId, blockerIssueIds: run.contextSnapshot?.blockerIssueIds, + paperclipEnvironment: { id: run.contextSnapshot?.paperclipEnvironment?.id } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: run.runnerProfileJson?.nativeExecutionInput?.provider?.kind, + model: run.runnerProfileJson?.nativeExecutionInput?.provider?.model } } } }); +const documentSnapshot = (row: Row) => Object.fromEntries(["id", "companyId", "issueId", "key", "body", "latestRevisionId", + "documentId", "createdByAgentId", "createdByUserId", "createdByRunId"].map(key => [key, row[key]])); + +async function properties(context: JourneyContext, label: string) { + const { page } = context; + const row = page.locator('[data-property-row]:not([inert] *):not([aria-hidden="true"] *)') + .filter({ has: page.locator(`[data-property-label="${label}"]`) }).filter({ visible: true }); + const toggle = page.getByRole("button", { name: "Toggle side panel", exact: true, pressed: false }).filter({ visible: true }); + const propertiesButton = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).filter({ visible: true }); + await expect.poll(async () => await row.isVisible() || await toggle.isVisible() || await propertiesButton.isVisible()).toBe(true); + if (!await row.isVisible()) { + if (await toggle.isVisible()) await toggle.click(); + else await propertiesButton.click(); + } + await expect(row).toBeVisible(); + await row.scrollIntoViewIfNeeded(); + await expect(row).toBeInViewport(); + return row; +} +async function setStatus(context: JourneyContext, label: string) { + const { page } = context; + await page.getByTestId("issue-detail-header").filter({ visible: true }) + .getByRole("button", { name: /^Change status \(current:/ }).click(); + await page.locator('[data-slot="popover-content"]').filter({ visible: true }) + .getByRole("button").filter({ has: page.getByRole("img", { name: label, exact: true }) }).click(); +} + +export async function taskDependencyResume(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, + options: { deadlineAt: number; directory: string }) { + const { api, page, company, user, agent } = context; + const read = createObservedReader(api, options.deadlineAt); + const reference = randomBytes(16).toString("hex"); + let scope: DependencyScope | undefined, stage = "prerequisites", journeyFailed = false; + let latest: DependencyPoint | undefined; + const observations: Partial = { waiting: [] }; + const preserve = async () => { + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + await writeFile(path.join(options.directory, "task-dependency-observations.json"), + JSON.stringify({ schema: "paperclip.task-dependency-observations/v1", stage, ...observations, latest, scope }, null, 2), { mode: 0o600 }); + }; + const url = (identifier: string) => `${api.origin}/${company.issuePrefix}/issues/${identifier}`; + try { + for (const capability of ["company", "board-user", "task-write", "agent"]) { + if (!context.capabilities.includes(capability)) throw new JourneyError("unsupported", `missing_${capability}`); + } + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); + if ([agent.provider, agent.model].some(value => typeof value !== "string" || !value.trim())) { + throw new JourneyError("unsupported", "dependency_execution_identity_unavailable"); + } + const summaries = async (route: string, limit?: number) => { + const rows = await read.get(route); + if (!Array.isArray(rows) || limit !== undefined && rows.length >= limit) throw new JourneyError("failed", "dependency_run_coverage_unavailable"); + const ids = rows.map(row => row.runId ?? row.id); + if (ids.some(value => typeof value !== "string" || !value) || new Set(ids).size !== ids.length) throw new JourneyError("failed", "dependency_run_identity_invalid"); + return rows.map((row, index) => ({ ...row, id: ids[index] })); + }; + const runDetail = async (runId: string) => { + const run = await read.get(`/api/heartbeat-runs/${runId}`); + if (run.id !== runId) throw new JourneyError("failed", "dependency_run_identity_invalid"); + return runSnapshot(run); + }; + const snapshots = async (route: string) => Promise.all((await summaries(route)).map(row => runDetail(row.id))); + const agentRoute = `/api/companies/${company.id}/heartbeat-runs?agentId=${agent.id}&limit=${HEARTBEAT_RUN_LIST_MAX_LIMIT}`; + stage = "observe_starting_runs"; + const startingRuns = observations.startingAgentRuns = (await summaries(agentRoute, HEARTBEAT_RUN_LIST_MAX_LIMIT)).map(runSnapshot); + if (startingRuns.some(run => run.companyId !== company.id || run.agentId !== agent.id + || !["succeeded", "failed", "cancelled", "timed_out"].includes(run.status))) { + throw new JourneyError("failed", "dependency_starting_agent_not_idle"); + } + const startingIds = new Set(startingRuns.map(run => run.id)); + const agentSnapshots = async () => Promise.all((await summaries(agentRoute, HEARTBEAT_RUN_LIST_MAX_LIMIT)) + .map(row => startingIds.has(row.id) ? runSnapshot(row) : runDetail(row.id))); + stage = "create_blocker"; + const blockerTitle = `Journey QA dependency blocker ${campaignId}`; + const blockerDescription = `Human review gate. The review reference is ${reference}. Keep this task for my review; no agent work is required.`; + const blocker = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: blockerTitle, prompt: blockerDescription, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: blockerTitle, journeyId: id }) }); + await session.own({ kind: "issue", name: blockerTitle, id: blocker.issueId, journeyId: id }); + const savedBlocker = await read.get(`/api/issues/${blocker.issueId}`); + if (typeof savedBlocker.identifier !== "string" || !savedBlocker.identifier) throw new JourneyError("failed", "dependency_blocker_receipt_invalid"); + const dependentTitle = `Journey QA dependency dependent ${campaignId}`, documentKey = "dependency-proof"; + const prompt = `After the blocker ${savedBlocker.identifier} is completed, read its description and save a task document with key ${documentKey}. ` + + `Its body must be JSON with exactly reference (the review reference in the blocker description) and blockerIdentifier ("${savedBlocker.identifier}"). ` + + "Then finish this task. No other tasks, agents or projects are needed."; + stage = "create_dependent"; + const dependent = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: dependentTitle, prompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: dependentTitle, journeyId: id }) }); + await session.own({ kind: "issue", name: dependentTitle, id: dependent.issueId, journeyId: id }); + scope = { companyId: company.id, userId: user.id, agentId: agent.id, environmentId: context.environmentId, + provider: agent.provider, model: agent.model, blockerId: blocker.issueId, blockerIdentifier: savedBlocker.identifier, + blockerTitle, blockerDescription, dependentId: dependent.issueId, dependentTitle, prompt, reference, documentKey }; + const observe = async (): Promise => { + const [currentBlocker, currentDependent, blockerRuns, dependentRuns, agentRuns, blockerActivity, dependentActivity, agentActivity] = await Promise.all([ + read.get(`/api/issues/${blocker.issueId}`), read.get(`/api/issues/${dependent.issueId}`), + snapshots(`/api/issues/${blocker.issueId}/runs`), snapshots(`/api/issues/${dependent.issueId}/runs`), + agentSnapshots(), read.get(`/api/issues/${blocker.issueId}/activity`), read.get(`/api/issues/${dependent.issueId}/activity`), + read.get(`/api/companies/${company.id}/activity?agentId=${agent.id}&limit=500`), + ]); + if ([blockerActivity, dependentActivity, agentActivity].some(rows => !Array.isArray(rows)) || agentActivity.length >= 500) { + throw new JourneyError("failed", "dependency_activity_coverage_unavailable"); + } + const newAgentRuns = dependencyNewAgentRuns(agentRuns, startingRuns); + const activity = [...new Map([...blockerActivity, ...dependentActivity, ...agentActivity].map(row => [row.id, row])).values()]; + if (activity.some(row => typeof row.id !== "string" || !row.id || row.companyId !== company.id) + || agentActivity.some(row => row.agentId !== agent.id)) throw new JourneyError("failed", "dependency_activity_identity_invalid"); + const point: DependencyPoint = { observedAt: Date.now(), blocker: taskSnapshot(currentBlocker), dependent: taskSnapshot(currentDependent), + blockerRuns, dependentRuns, agentRuns, activityCoverage: { blocker: true, dependent: true, agent: true }, + activity: activity.filter(row => [blocker.issueId, dependent.issueId].includes(row.entityId) + || newAgentRuns.some(run => run.id === row.runId)).map(row => ({ id: row.id, companyId: row.companyId, actorType: row.actorType, + actorId: row.actorId, runId: row.runId, action: row.action, entityType: row.entityType, entityId: row.entityId, + createdAt: row.createdAt, details: { status: row.details?.status } })) }; + latest = point; + if (dependentRuns.length > 1 || newAgentRuns.length > 1 || blockerRuns.length + || [...dependentRuns, ...newAgentRuns].some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) { + throw new JourneyError("failed", "dependency_unexpected_or_failed_work"); + } + if (currentDependent.status === "done") { + const documents = await read.get(`/api/issues/${dependent.issueId}/documents`); + const candidate = documents.find(row => row.key === documentKey); + if (candidate) { + const document = await read.get(`/api/issues/${dependent.issueId}/documents/${documentKey}`); + point.document = documentSnapshot(document); + point.revision = documentSnapshot(await read.get(`/api/issues/${dependent.issueId}/documents/${documentKey}/revisions/${document.latestRevisionId}`)); + } + } + return point; + }; + observations.created = await observe(); + if ([observations.created.blocker, observations.created.dependent].some(task => task.companyId !== company.id + || task.assigneeUserId !== user.id || task.assigneeAgentId !== null || task.createdByUserId !== user.id + || task.createdByAgentId || task.status !== "todo") || observations.created.blocker.id !== blocker.issueId + || observations.created.dependent.id !== dependent.issueId || observations.created.blockerRuns.length + || observations.created.dependentRuns.length || dependencyNewAgentRuns(observations.created.agentRuns, startingRuns).length) { + throw new JourneyError("failed", "dependency_created_task_binding_mismatch"); + } + stage = "link_blocker"; + const dependentSaved = observations.created.dependent; + await page.goto(url(dependentSaved.identifier ?? dependent.issueId), { waitUntil: "domcontentloaded" }); + const blockedBy = await properties(context, "Blocked by"); + await blockedBy.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByRole("textbox", { name: "Search tasks to add as blockers", exact: true }).fill(blockerTitle); + await page.getByRole("button").filter({ hasText: blockerTitle }).last().click(); + await expect.poll(async () => (taskSnapshot(await read.get(`/api/issues/${dependent.issueId}`)).blockedByIssueIds)).toEqual([blocker.issueId]); + stage = "set_dependent_blocked"; + await setStatus(context, "Blocked"); + await expect.poll(async () => (await read.get(`/api/issues/${dependent.issueId}`)).status).toBe("blocked"); + stage = "assign_blocked_task"; + const assignee = await properties(context, "Assignee"); + await assignee.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByPlaceholder("Search assignees...", { exact: true }).fill(agent.name); + await page.getByRole("button").filter({ hasText: agent.name }).last().click(); + await expect.poll(async () => (await read.get(`/api/issues/${dependent.issueId}`)).assigneeAgentId).toBe(agent.id); + await page.reload(); + stage = "observe_blocked_wait"; + const waiting = observations.waiting!; + waiting.push(await observe()); + if (waiting[0]!.dependentRuns.length || dependencyNewAgentRuns(waiting[0]!.agentRuns, startingRuns).length || waiting[0]!.dependent.status !== "blocked" + || waiting[0]!.blocker.status !== "todo") throw new JourneyError("failed", "dependency_ran_before_blocker_completed"); + const until = waiting[0]!.observedAt + DEPENDENCY_IDLE_MS; + while (Date.now() < until) { + if (Date.now() + 1000 >= options.deadlineAt) throw new JourneyError("failed", "dependency_wait_deadline"); + await new Promise(resolve => setTimeout(resolve, Math.min(1000, until - Date.now()))); + const point = await observe(); waiting.push(point); + if (point.dependentRuns.length || dependencyNewAgentRuns(point.agentRuns, startingRuns).length || point.dependent.status !== "blocked" || point.blocker.status !== "todo") { + throw new JourneyError("failed", "dependency_ran_before_blocker_completed"); + } + } + if (waiting[waiting.length - 1]!.observedAt < until) waiting.push(await observe()); + stage = "complete_blocker"; + await page.goto(url(savedBlocker.identifier), { waitUntil: "domcontentloaded" }); + observations.releaseRequestedAt = Date.now(); + await setStatus(context, "Done"); + await expect.poll(async () => (await read.get(`/api/issues/${blocker.issueId}`)).status).toBe("done"); + observations.released = await observe(); + stage = "await_automatic_resume"; + while (Date.now() < options.deadlineAt) { + const point = await observe(); observations.finished = point; + if (point.dependent.status === "done" && point.dependentRuns.length === 1 && point.dependentRuns[0]?.status === "succeeded" + && point.dependentRuns[0]?.nativePhase === "committed" && point.document && point.revision + && !point.dependent.executionRunId && !point.dependent.checkoutRunId) break; + await new Promise(resolve => setTimeout(resolve, 1000)); + } + if (!observations.finished || observations.finished.dependent.status !== "done" || observations.finished.dependentRuns[0]?.status !== "succeeded" + || observations.finished.dependentRuns[0]?.nativePhase !== "committed" || !observations.finished.document || !observations.finished.revision) { + throw new JourneyError("failed", "dependency_resume_deadline"); + } + stage = "reopen_result"; + await page.goto(url(dependentSaved.identifier ?? dependent.issueId), { waitUntil: "domcontentloaded" }); + await page.reload(); await openDocumentAuthoring(context); + const document = page.locator(`#document-${documentKey}`); + await expect(document).toContainText(reference); + observations.reopened = await observe(); + observations.reopened.resultVisible = await document.isVisible() && (await document.textContent())?.includes(reference) === true; + const checks = gradeTaskDependencyResume({ ...observations, scope } as DependencyEvidence); + evidence.checks.push(...checks); + const failed = checks.find(check => !check.passed); + if (failed) throw new JourneyError("failed", failed.id.replaceAll("-", "_")); + stage = "complete"; + } catch (error) { + journeyFailed = true; + throw error; + } finally { + const writers = [ + { label: "task-dependency-observations.json", write: preserve }, + { label: "observation-read-failures.json", write: () => read.preserve(options.directory) }, + ]; + const results = await Promise.allSettled(writers.map(writer => Promise.resolve().then(writer.write))); + const failedWriters = results.flatMap((result, index) => result.status === "rejected" ? [writers[index]!.label] : []); + if (failedWriters.length) { + console.error("dependency_evidence_preservation_failed", failedWriters); + if (!journeyFailed) throw new JourneyError("failed", "dependency_evidence_preservation_failed"); + } + } +} diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index cc92860f48b..2a776353237 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -7,6 +7,7 @@ export type JourneyId = | "task-properties" | "task-create" | "project-task-context" + | "task-dependency-resume" | "task-document-persistence" | "ask-question" | "chat-continuity" From 9e61fdebf9e634a1537624d6c6ad08ef58e58f96 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:00:36 -0600 Subject: [PATCH 21/28] test(evals): pin approval-link fix for case 564 (#18909) ## Thinking Path > - Paperclip manages work through agents and explicit approval gates. > - The hiring revision case requires a usable link to the saved pending proposal. > - A rendered unprefixed approval link currently misses the company redirect. > - The reviewed fix adds one existing redirect route. It changes no approval authority. > - The TestOps workflow reads its candidate source from a protected master pin. > - This PR pins that one-fix candidate so one remote case can test the original journey. ## Linked Issues or Issue Description Refs #17858. Refs #18780. The source candidate applies only the approval-detail redirect from the existing fix. The core fix PR remains unmerged. This PR changes the evaluation source pin. ## What Changed - Pin candidate `abbde061c7ec9181021858f7b1af21a48637f97f`, a direct child of the prior `fd5e3b1e` candidate. - Update the matching launch snapshot target and the expected-pin test. - Keep all 244 case rows, original inputs, clocks, model, and private evidence rules unchanged. ## Verification - The candidate differs from its parent in one line in `ui/src/App.tsx`. - The owning browser regression follows a real rendered link to a saved pending proposal. It fails before the route fix and passes after it. The 18 App tests, UI build, typecheck, and token gates passed. - The candidate token gates passed. The existing source SDK input check reports 506 unchanged inputs and unchanged runtime pins. No SDK registry or certificate was renewed. - The three existing TestOps contract, adapter, and manifest test files passed: 47 tests, zero failures. - The first local pin invocation lacked the YAML dependency. It is retained as a setup failure. The invocation with the existing locked dependencies passed without assertion changes. - Exact-head normal PR CI [41763017485](https://github.com/paperclipai/paperclip/actions/runs/41763017485) completed successfully: 48 successful jobs and one selector skip. Fresh Greptile and Paperclip reviews both report 5/5 with no findings; all review threads are resolved. The optional Superagent security check is neutral, so no security assessment PASS is claimed. No new provider run or TestOps PASS is claimed. The intended next check is only `questions-and-approvals.formal-hire-revision` on existing plan 102 and job 100. ## Risks The route regression does not prove the full agent journey passes. The first remote run will test that remaining gap. The candidate keeps the prior catalog application base plus this one route fix. It does not refresh the whole runtime or qualify all 244 cases. The existing broader qualification holds remain. Automatic model retries stay zero and the authorized concurrency cap stays eight. ## Model Used OpenAI `gpt-6.1-sol` through Codex. Capabilities used: source analysis, code integration, and local validation. The host does not expose the context-window limit. The model under evaluation remains OpenCode/OpenRouter `deepseek/deepseek-v4.1-flash-1017`. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- tests/runner-e2e/chat-full-catalog-testops-manifest.json | 4 ++-- tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index 66cef83f915..255f181243b 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1,6 +1,6 @@ { "schema": "paperclip.chat-full-catalog-testops-manifest/v1", - "targetSha": "fd5e3b1e5a379524c0d1651ed8273e75ffd86541", + "targetSha": "abbde061c7ec9181021858f7b1af21a48637f97f", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "profile": "runner-opencode", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017", @@ -1578,7 +1578,7 @@ }, "candidateLauncherSnapshot": { "path": "fixtures/chat-repair-candidate-launcher.txt", - "targetSha": "fd5e3b1e5a379524c0d1651ed8273e75ffd86541", + "targetSha": "abbde061c7ec9181021858f7b1af21a48637f97f", "sourcePath": "tests/runner-e2e/launch.ts", "gitBlobOid": "dba1467e211611f37937e9e1fc4c0c5be4329cbc" } diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index 9dca3e5607f..015a41207e6 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -103,7 +103,7 @@ test('repair source metadata binds the corrected candidate and explicit changed const repair = manifest.source.repairMetadata; assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); assert.equal(repair.previousTargetSha, 'ab2019e976076ec0eb60b026844f3d9211cb0208'); - assert.equal(manifest.targetSha, 'fd5e3b1e5a379524c0d1651ed8273e75ffd86541'); + assert.equal(manifest.targetSha, 'abbde061c7ec9181021858f7b1af21a48637f97f'); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); assert.equal(repair.candidateDescriptorSnapshot, 'fixtures/chat-repair-candidate-descriptors.json'); const descriptorBytes = readFileSync(new URL(repair.candidateDescriptorSnapshot, import.meta.url)); From 27a825da2f960d4042f2fd91bab85112b0d99acf Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Tue, 3 Nov 2026 19:00:47 -0800 Subject: [PATCH 22/28] feat(telemetry): track the server OS with the registered contract (#18897) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Its first-party telemetry uses a generated event contract and typed helpers. > - The existing events do not identify the server's OS family. > - A proposed event is dropped until the generated contract registers it. > - This PR syncs the registered contract and emits the OS family when an enabled server telemetry client starts. > - The benefit is a way to compare Windows, Linux, and macOS installations while keeping machine details out of the event. ## Linked Issues or Issue Description **Problem or motivation** We cannot compare Paperclip installations by host OS. Counting task or install events does not provide this dimension. **Proposed solution** Send one `server.started` event with Node's raw `process.platform` as `os_platform` each time the server initializes its enabled telemetry client. Compare distinct install identifiers per platform, rather than restart event totals. **Alternatives considered** A proposal-only emitter would send no data. A separate later contract-sync PR would leave the feature incomplete. This PR includes the canonical generated SDK and promotes the emitter together. CPU architecture and OS versions are outside this feature's scope. **Roadmap alignment** This is a narrow telemetry addition to existing server startup, with no new user flow. The full contract copy also adopts the three existing connector proposals described in #13578; their schemas are already registered. The generated file stays an exact copy, so their helpers and tests must be promoted in the same change. ## What Changed - Export `trackServerStarted` and call it once when server telemetry initializes. Repeated initialization returns the same client and emits no duplicate event. - Copy the canonical generated telemetry SDK byte for byte from schema source commit `5369e6b`. Remove the startup proposal marker. Normal opt-out and CI suppression apply before initialization. - Promote the three connector events included in the canonical SDK. Derive enum types from the generated map and preserve raw values. Update their real-client transport and disabled-state tests. - Assign startup to the existing `operational_enum_count` retention class. Document OS privacy, container/WSL interpretation, and distinct-install counting. - Add real-client startup transport tests and server initialization tests, including file opt-out, environment opt-out, and CI suppression. - Keep startup running when telemetry state storage fails. Publish the client and start periodic flushing only after the startup event succeeds. On failure, clean up the client, disable telemetry for the process, and log a generic warning without the storage path or underlying error. ## Verification - `pnpm --filter @paperclipai/shared typecheck`: passed. - `pnpm --filter @paperclipai/shared build`: passed. - `pnpm --filter @paperclipai/shared exec vitest run`: 98 files, 1,055 tests passed. - `pnpm --filter @paperclipai/shared exec vitest run src/telemetry`: 6 files, 39 tests passed. - `pnpm --filter @paperclipai/server exec vitest run src/__tests__/telemetry-startup.test.ts src/__tests__/shared-telemetry-events.test.ts`: 2 files, 19 tests passed. - Exact generated artifact comparison, `git diff --check`, and local secret/PII review passed. - Proposal inventory contains no startup or connector proposals. The repository extractor fails with TypeScript 7's missing compiler API; rerunning it with the installed TypeScript 5.9 compiler API passed. No extractor source change is included. - Initial local server typecheck attempts found missing workspace dependencies and a missing plugin SDK link in this fresh checkout. Dependencies and the local SDK link were repaired; `pnpm --filter @paperclipai/server typecheck` then passed. Full repository typecheck, tests, and build are covered by the PR CI rather than claimed as local passes. Greptile found that a telemetry state-file error could abort startup. New regression tests reproduced this on the previous head with both an injected permission error and the real state store blocked by a file. Both now pass, including no published client, no periodic flushing, no send, and no retry after failure. Shared telemetry tests and server typecheck were rerun after the fix. Full PR CI passed on `a41540b7899bc8a0aedbd78d72f8b69ad1a4b318`, including typecheck, build, all selected test shards, and the final verify gate. [Successful workflow attempt 2](https://github.com/paperclipai/paperclip/actions/runs/41752039595/attempts/2). Attempt 1 was interrupted when the server (1/12) runner received a shutdown signal; its incomplete result was retained in the PR discussion. The same-commit shard retry and dependent gate passed. Greptile reviewed this exact head at 5/5 with all threads resolved. ## Risks This change starts sending one OS family value per server startup when telemetry is enabled. It sends no OS version, architecture, hostname, path, prompt, or machine identifier beyond the existing envelope. Containers and WSL report the runtime's OS. Event totals reflect restart frequency, so use distinct install identifiers for OS adoption analysis. The complete generated SDK also enables the previously registered connector proposals. Those events keep their existing approved payloads; no connection names, credentials, tool arguments, or results are added. Privacy review is requested for both newly active paths. Deploy the registered schema before merging this app change. Live production delivery has not been tested. No schema migration is needed. Revert this PR to stop the new startup emitter and restore the prior contract. ## Model Used - Original implementation: Anthropic Claude Opus 5.5, as recorded in the original commit. - Completion: OpenAI `gpt-6.1-sol`, xhigh reasoning, through Codex in T3 Code; shell, Git, and automated tests. Context window size is not exposed by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have described the issue in-PR following the feature issue template - [x] The final change and description contain no private issue links or proposal markers - [x] The existing PR branch is preserved to retain its review history - [x] I have run the targeted tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Paperclip --- packages/shared/src/telemetry/README.md | 18 +++ ...vents.test.ts => connector-events.test.ts} | 33 +++-- packages/shared/src/telemetry/events.ts | 87 ++++++----- .../generated/paperclip-telemetry.ts | 140 +++++++++++++++++- packages/shared/src/telemetry/index.ts | 1 + packages/shared/src/telemetry/retention.ts | 1 + .../telemetry/server-started-event.test.ts | 69 +++++++++ .../src/__tests__/telemetry-startup.test.ts | 113 ++++++++++++++ server/src/telemetry.ts | 30 ++-- 9 files changed, 424 insertions(+), 68 deletions(-) rename packages/shared/src/telemetry/{proposed-connector-events.test.ts => connector-events.test.ts} (75%) create mode 100644 packages/shared/src/telemetry/server-started-event.test.ts create mode 100644 server/src/__tests__/telemetry-startup.test.ts diff --git a/packages/shared/src/telemetry/README.md b/packages/shared/src/telemetry/README.md index da0403c260b..e792827b9f1 100644 --- a/packages/shared/src/telemetry/README.md +++ b/packages/shared/src/telemetry/README.md @@ -65,6 +65,24 @@ provider-supplied error name that may contain a credential. This is a privacy boundary, not enum canonicalization. Test the value reaching the telemetry helper as well as the separate crash-reporting and local logging sinks. +## Server Startup Privacy + +The startup helper emits the raw Node.js operating system family once when an +enabled server telemetry client initializes. It omits OS versions, CPU +architecture, hostnames, and other machine details. The usual telemetry opt-out +and CI suppression apply before client initialization. If state storage fails +at startup, the server disables telemetry for that process and continues to run. +The warning contains no storage path or underlying error. Container and WSL servers +report their runtime platform; this does not identify the underlying physical +host's OS. Count distinct installation identifiers by platform to compare +installations, since restart event counts also reflect restart frequency. + +Contract syncs may also register existing proposals. Remove each adopted +proposal marker and update its helper and transport tests in the same change. +The connector helpers preserve raw enum values and send only catalog connector +keys and the approved lifecycle and invocation dimensions. Connection names, +credentials, tool arguments, and tool results stay out of those events. + ## Interaction Resolver Events `interaction.created` records the interaction kind and whether the create diff --git a/packages/shared/src/telemetry/proposed-connector-events.test.ts b/packages/shared/src/telemetry/connector-events.test.ts similarity index 75% rename from packages/shared/src/telemetry/proposed-connector-events.test.ts rename to packages/shared/src/telemetry/connector-events.test.ts index a8a6826e9de..c060017c7a6 100644 --- a/packages/shared/src/telemetry/proposed-connector-events.test.ts +++ b/packages/shared/src/telemetry/connector-events.test.ts @@ -28,7 +28,7 @@ function makeClient(config?: { enabled?: boolean }) { }; } -function trackAllProposedConnectorEvents(client: TelemetryClient) { +function trackAllConnectorEvents(client: TelemetryClient) { trackConnectionCreated(client, { connector_key: "github", transport: "mcp_remote", @@ -56,28 +56,33 @@ function trackAllProposedConnectorEvents(client: TelemetryClient) { }); } -describe("proposed connector events against the real TelemetryClient", () => { +describe("registered connector events against the real TelemetryClient", () => { afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); + vi.unstubAllGlobals(); }); - it("cannot queue or send: state stays untouched and nothing hits the wire", async () => { + it("queues and sends the three registered connector events", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client, stateFactory } = makeClient(); - trackAllProposedConnectorEvents(client); + trackAllConnectorEvents(client); await client.flush(); - expect(stateFactory).not.toHaveBeenCalled(); - expect(fetch).not.toHaveBeenCalled(); + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledTimes(1); + const body = JSON.parse(String(vi.mocked(fetch).mock.calls[0]?.[1]?.body)); + expect(body.events.map((event: { name: string }) => event.name)).toEqual([ + "connection.created", "connection.updated", "connection.invoked", + ]); }); - it("differential control: a registered event through the same client does send", async () => { + it("batches connector events with other registered events", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client } = makeClient(); - trackAllProposedConnectorEvents(client); + trackAllConnectorEvents(client); client.track("project.created", {}); await client.flush(); @@ -86,15 +91,18 @@ describe("proposed connector events against the real TelemetryClient", () => { String((vi.mocked(fetch).mock.calls[0]?.[1] as RequestInit).body), ); expect(body.events).toEqual([ + expect.objectContaining({ name: "connection.created" }), + expect.objectContaining({ name: "connection.updated" }), + expect.objectContaining({ name: "connection.invoked" }), expect.objectContaining({ name: "project.created" }), ]); }); - it("reports the three connector event names as unregistered", () => { + it("reports the three connector event names as registered", () => { const { client } = makeClient(); - expect(client.isRegisteredEventName("connection.created")).toBe(false); - expect(client.isRegisteredEventName("connection.updated")).toBe(false); - expect(client.isRegisteredEventName("connection.invoked")).toBe(false); + expect(client.isRegisteredEventName("connection.created")).toBe(true); + expect(client.isRegisteredEventName("connection.updated")).toBe(true); + expect(client.isRegisteredEventName("connection.invoked")).toBe(true); expect(client.isRegisteredEventName("project.created")).toBe(true); }); @@ -102,6 +110,7 @@ describe("proposed connector events against the real TelemetryClient", () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client, stateFactory } = makeClient({ enabled: false }); + trackAllConnectorEvents(client); client.track("project.created", {}); await client.flush(); diff --git a/packages/shared/src/telemetry/events.ts b/packages/shared/src/telemetry/events.ts index 71f78bb2f3f..78f2031313f 100644 --- a/packages/shared/src/telemetry/events.ts +++ b/packages/shared/src/telemetry/events.ts @@ -156,70 +156,69 @@ export function trackAgentTaskRun( export function trackConnectionCreated( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - auth_kind: RawDimension<"oauth" | "api_key" | "none">; - setup_flow: RawDimension<"gallery" | "api" | "example">; - status: RawDimension<"draft" | "active" | "disabled" | "archived">; + connector_key: string; + transport: RawDimension; + auth_kind: RawDimension; + setup_flow: RawDimension; + status: RawDimension; enabled: boolean; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure which catalog connectors installations create connections for - "connection.created", - dims, - ); + client.track("connection.created", { + ...dims, + transport: asEventDimension(dims.transport), + auth_kind: asEventDimension(dims.auth_kind), + setup_flow: asEventDimension(dims.setup_flow), + status: asEventDimension(dims.status), + }); } export function trackConnectionUpdated( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - auth_kind: RawDimension<"oauth" | "api_key" | "none">; - change_source: RawDimension< - | "api" - | "gallery" - | "oauth_callback" - | "credential_refresh" - | "archive" - | "example" - >; - previous_status: RawDimension<"draft" | "active" | "disabled" | "archived">; - status: RawDimension<"draft" | "active" | "disabled" | "archived">; + connector_key: string; + transport: RawDimension; + auth_kind: RawDimension; + change_source: RawDimension; + previous_status: RawDimension; + status: RawDimension; previous_enabled: boolean; enabled: boolean; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure connector lifecycle transitions (configured, paused, archived) after creation - "connection.updated", - dims, - ); + client.track("connection.updated", { + ...dims, + transport: asEventDimension(dims.transport), + auth_kind: asEventDimension(dims.auth_kind), + change_source: asEventDimension(dims.change_source), + previous_status: asEventDimension(dims.previous_status), + status: asEventDimension(dims.status), + }); } export function trackConnectionInvoked( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - status: RawDimension< - | "succeeded" - | "failed" - | "denied" - | "cancelled" - | "timed_out" - | "rate_limited" - >; - origin: RawDimension<"setup_test" | "agent" | "user" | "system" | "plugin">; + connector_key: string; + transport: RawDimension; + status: RawDimension; + origin: RawDimension; duration_seconds?: number; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure whether connected connectors are successfully used and where invocations fail; records completed invocation attempts and their terminal status, never invocation starts - "connection.invoked", - dims, - ); + client.track("connection.invoked", { + ...dims, + transport: asEventDimension(dims.transport), + status: asEventDimension(dims.status), + origin: asEventDimension(dims.origin), + }); +} + +export function trackServerStarted( + client: TelemetryClient, + dims: { os_platform: RawDimension }, +): void { + client.track("server.started", { os_platform: asEventDimension(dims.os_platform) }); } export function trackErrorHandlerCrash( diff --git a/packages/shared/src/telemetry/generated/paperclip-telemetry.ts b/packages/shared/src/telemetry/generated/paperclip-telemetry.ts index 0502b6e2a1c..b44d37d3428 100644 --- a/packages/shared/src/telemetry/generated/paperclip-telemetry.ts +++ b/packages/shared/src/telemetry/generated/paperclip-telemetry.ts @@ -37,6 +37,34 @@ source_ref?: string source_ref_hashed?: boolean } +export interface PaperclipConnectionCreatedDimensions { +auth_kind: ("oauth" | "api_key" | "none") +connector_key: string +enabled: boolean +setup_flow: ("gallery" | "api" | "example") +status: ("draft" | "active" | "disabled" | "archived") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + +export interface PaperclipConnectionInvokedDimensions { +connector_key: string +duration_seconds?: number +origin: ("setup_test" | "agent" | "user" | "system" | "plugin") +status: ("succeeded" | "failed" | "denied" | "cancelled" | "timed_out" | "rate_limited") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + +export interface PaperclipConnectionUpdatedDimensions { +auth_kind: ("oauth" | "api_key" | "none") +change_source: ("api" | "gallery" | "oauth_callback" | "credential_refresh" | "archive" | "example") +connector_key: string +enabled: boolean +previous_enabled: boolean +previous_status: ("draft" | "active" | "disabled" | "archived") +status: ("draft" | "active" | "disabled" | "archived") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + export interface PaperclipErrorHandlerCrashDimensions { error_code: string } @@ -58,7 +86,6 @@ interaction_kind: ("suggest_tasks" | "ask_user_questions" | "request_confirmatio used_deprecated_resolver_policy_alias: boolean } -/** Emit only after the complete resolution transaction commits, including response provenance. */ export interface PaperclipInteractionResolvedDimensions { interaction_kind: ("suggest_tasks" | "ask_user_questions" | "request_confirmation" | "request_checkbox_confirmation" | "request_item_verdicts" | "other") status: ("accepted" | "rejected" | "answered" | "cancelled" | "expired" | "failed" | "other") @@ -95,9 +122,11 @@ source: ("schedule" | "manual" | "api" | "webhook" | "other") status: ("received" | "coalesced" | "skipped" | "issue_created" | "completed" | "failed" | "other") } +export interface PaperclipServerStartedDimensions { +os_platform: ("aix" | "android" | "darwin" | "freebsd" | "haiku" | "linux" | "openbsd" | "sunos" | "win32" | "cygwin" | "netbsd" | "other") +} + export interface PaperclipSkillImportedDimensions { -/** skill_ref may identify a known public catalog source only. Omit it for - * GitHub-synced sources, which may be private; never emit repository paths or names. */ source_type: ("local_path" | "github" | "url" | "catalog" | "skills_sh" | "unknown") skill_ref?: string } @@ -108,6 +137,9 @@ export type PaperclipEventName = | "agent.task_completed" | "agent.task_run" | "company.imported" + | "connection.created" + | "connection.invoked" + | "connection.updated" | "error.handler_crash" | "goal.created" | "install.completed" @@ -117,6 +149,7 @@ export type PaperclipEventName = | "project.created" | "routine.created" | "routine.run" + | "server.started" | "skill.imported"; export interface EventDimensionsMap { @@ -125,6 +158,9 @@ export interface EventDimensionsMap { "agent.task_completed": PaperclipAgentTaskCompletedDimensions; "agent.task_run": PaperclipAgentTaskRunDimensions; "company.imported": PaperclipCompanyImportedDimensions; + "connection.created": PaperclipConnectionCreatedDimensions; + "connection.invoked": PaperclipConnectionInvokedDimensions; + "connection.updated": PaperclipConnectionUpdatedDimensions; "error.handler_crash": PaperclipErrorHandlerCrashDimensions; "goal.created": PaperclipGoalCreatedDimensions; "install.completed": PaperclipInstallCompletedDimensions; @@ -134,6 +170,7 @@ export interface EventDimensionsMap { "project.created": PaperclipProjectCreatedDimensions; "routine.created": PaperclipRoutineCreatedDimensions; "routine.run": PaperclipRoutineRunDimensions; + "server.started": PaperclipServerStartedDimensions; "skill.imported": PaperclipSkillImportedDimensions; } @@ -143,6 +180,9 @@ export const PAPERCLIP_EVENTS = { "agent.task_completed": "agent.task_completed", "agent.task_run": "agent.task_run", "company.imported": "company.imported", + "connection.created": "connection.created", + "connection.invoked": "connection.invoked", + "connection.updated": "connection.updated", "error.handler_crash": "error.handler_crash", "goal.created": "goal.created", "install.completed": "install.completed", @@ -152,6 +192,7 @@ export const PAPERCLIP_EVENTS = { "project.created": "project.created", "routine.created": "routine.created", "routine.run": "routine.run", + "server.started": "server.started", "skill.imported": "skill.imported", } as const; @@ -277,6 +318,83 @@ export const PAPERCLIP_ENUM_DESCRIPTIONS = { "unknown": "Source type could not be classified." } }, + "connection.created": { + "auth_kind": { + "oauth": "Connection authenticates with an OAuth flow.", + "api_key": "Connection authenticates with a stored API key.", + "none": "Connection requires no authentication." + }, + "setup_flow": { + "gallery": "Connection was created through the app-gallery setup flow.", + "api": "Connection was created through the connections API.", + "example": "Connection was created or upserted as a bundled example connection." + }, + "status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, + "connection.invoked": { + "origin": { + "setup_test": "A human ran the connection's setup test, identified by durable invocation columns: user actor with no run, issue, or gateway reference and a connection id.", + "agent": "An agent run invoked the tool.", + "user": "A human invoked the tool outside the setup-test shape.", + "system": "A system-initiated process invoked the tool.", + "plugin": "A plugin invoked the tool." + }, + "status": { + "succeeded": "The tool invocation completed successfully.", + "failed": "The tool invocation reached a terminal failure.", + "denied": "The tool invocation was denied by policy or by a human approval decision.", + "cancelled": "The tool invocation was cancelled before completion. Declared in the application status enum; no current application code path writes this value.", + "timed_out": "The tool invocation exceeded its time limit.", + "rate_limited": "The tool invocation was rejected by rate limiting." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, + "connection.updated": { + "auth_kind": { + "oauth": "Connection authenticates with an OAuth flow.", + "api_key": "Connection authenticates with a stored API key.", + "none": "Connection requires no authentication." + }, + "change_source": { + "api": "A connections API update changed the persisted lifecycle state.", + "gallery": "An app-gallery setup step (finish, OAuth access finalization, or reconnect; Apps gallery and inline task cards) changed the persisted lifecycle state.", + "oauth_callback": "An OAuth callback completion (provider OAuth, Paperclip Cloud connector, or Vercel Connect) changed the persisted lifecycle state.", + "credential_refresh": "A credential refresh changed the persisted lifecycle state.", + "archive": "An archive operation changed the persisted lifecycle state.", + "example": "An example-connection upsert changed the persisted lifecycle state." + }, + "previous_status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, "goal.created": { "goal_level": { "company": "Goal applies at company scope.", @@ -402,6 +520,22 @@ export const PAPERCLIP_ENUM_DESCRIPTIONS = { "other": "Fallback when the status is unknown or not represented by the tracked enum." } }, + "server.started": { + "os_platform": { + "aix": "Server host runs IBM AIX.", + "android": "Server host runs Android.", + "darwin": "Server host runs macOS.", + "freebsd": "Server host runs FreeBSD.", + "haiku": "Server host runs Haiku.", + "linux": "Server host runs Linux, including WSL and container hosts.", + "openbsd": "Server host runs OpenBSD.", + "sunos": "Server host runs SunOS, Solaris, or illumos.", + "win32": "Server host runs Windows, both 32-bit and 64-bit.", + "cygwin": "Server host runs under Cygwin on Windows.", + "netbsd": "Server host runs NetBSD.", + "other": "Fallback when the platform is unknown or not represented by the tracked enum." + } + }, "skill.imported": { "source_type": { "local_path": "Import source came from a filesystem path on the operator's machine.", diff --git a/packages/shared/src/telemetry/index.ts b/packages/shared/src/telemetry/index.ts index 61f945ad834..d6436ffd5ba 100644 --- a/packages/shared/src/telemetry/index.ts +++ b/packages/shared/src/telemetry/index.ts @@ -21,6 +21,7 @@ export { trackConnectionCreated, trackConnectionUpdated, trackConnectionInvoked, + trackServerStarted, } from "./events.js"; export type { TelemetryConfig, diff --git a/packages/shared/src/telemetry/retention.ts b/packages/shared/src/telemetry/retention.ts index 543e1085298..023004d4e49 100644 --- a/packages/shared/src/telemetry/retention.ts +++ b/packages/shared/src/telemetry/retention.ts @@ -47,4 +47,5 @@ export const EVENT_RETENTION_CLASS: Partial> = { "codex.credential_health": "operational_enum_count", "interaction.created": "operational_enum_count", "interaction.resolved": "operational_enum_count", + "server.started": "operational_enum_count", }; diff --git a/packages/shared/src/telemetry/server-started-event.test.ts b/packages/shared/src/telemetry/server-started-event.test.ts new file mode 100644 index 00000000000..3d0e0e2289b --- /dev/null +++ b/packages/shared/src/telemetry/server-started-event.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TelemetryClient } from "./client.js"; +import { trackServerStarted } from "./events.js"; +import type { TelemetryState } from "./types.js"; + +const TEST_STATE: TelemetryState = { + installId: "test-install", + salt: "test-salt", + createdAt: "2026-01-01T00:00:00Z", + firstSeenVersion: "0.0.0", +}; + +function makeClient(enabled = true) { + const stateFactory = vi.fn(() => TEST_STATE); + return { + client: new TelemetryClient( + { enabled, endpoint: "http://localhost:9999/ingest" }, + stateFactory, + "0.0.0-test", + () => 0.5, + ), + stateFactory, + }; +} + +describe("registered server.started event", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it("passes the raw process.platform value through without normalization", () => { + const { client } = makeClient(); + const track = vi.spyOn(client, "track"); + + trackServerStarted(client, { os_platform: "win32" }); + trackServerStarted(client, { os_platform: "some-future-platform" }); + + expect(track.mock.calls).toEqual([ + ["server.started", { os_platform: "win32" }], + ["server.started", { os_platform: "some-future-platform" }], + ]); + }); + + it("sends the registered event with only the raw OS family", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); + const { client, stateFactory } = makeClient(); + + expect(client.isRegisteredEventName("server.started")).toBe(true); + trackServerStarted(client, { os_platform: process.platform }); + await client.flush(); + + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledTimes(1); + const body = JSON.parse(String(vi.mocked(fetch).mock.calls[0]?.[1]?.body)); + expect(body.events).toEqual([ + { name: "server.started", occurredAt: expect.any(String), dimensions: { os_platform: process.platform } }, + ]); + }); + + it("leaves state and network untouched when disabled", async () => { + vi.stubGlobal("fetch", vi.fn()); + const { client, stateFactory } = makeClient(false); + trackServerStarted(client, { os_platform: process.platform }); + await client.flush(); + expect(stateFactory).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/__tests__/telemetry-startup.test.ts b/server/src/__tests__/telemetry-startup.test.ts new file mode 100644 index 00000000000..b132405f681 --- /dev/null +++ b/server/src/__tests__/telemetry-startup.test.ts @@ -0,0 +1,113 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const stateFactory = vi.hoisted(() => vi.fn(() => ({ + installId: "test-install", + salt: "test-salt", + createdAt: "2026-01-01T00:00:00Z", + firstSeenVersion: "0.0.0", +}))); + +vi.mock("@paperclipai/shared/telemetry", async (importOriginal) => ({ + ...await importOriginal(), + loadOrCreateState: stateFactory, +})); +vi.mock("../home-paths.js", () => ({ resolvePaperclipInstanceRoot: () => "/unused-test-instance" })); +vi.mock("../version.js", () => ({ serverVersion: "0.0.0-test" })); + +describe("server telemetry startup", () => { + const fixtureDirs: string[] = []; + beforeEach(() => { + vi.resetModules(); + stateFactory.mockClear(); + for (const key of ["CI", "CONTINUOUS_INTEGRATION", "BUILD_NUMBER", "GITHUB_ACTIONS", "GITLAB_CI", "PAPERCLIP_TELEMETRY_DISABLED", "DO_NOT_TRACK"]) { + vi.stubEnv(key, ""); + } + vi.stubEnv("PAPERCLIP_TELEMETRY_ENDPOINT", "http://localhost:9999/ingest"); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); + }); + + afterEach(async () => { + const { getTelemetryClient } = await import("../telemetry.js"); + getTelemetryClient()?.stop(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + for (const dir of fixtureDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + it("emits the OS family exactly once even when initialization is repeated", async () => { + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + const client = initTelemetry({ enabled: true }); + expect(client).not.toBeNull(); + expect(initTelemetry({ enabled: true })).toBe(client); + expect(getTelemetryClient()).toBe(client); + await client!.flush(); + + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledTimes(1); + const body = JSON.parse(String(vi.mocked(fetch).mock.calls[0]?.[1]?.body)); + expect(body.events).toEqual([ + { name: "server.started", occurredAt: expect.any(String), dimensions: { os_platform: process.platform } }, + ]); + }); + + it("disables telemetry after a state failure without aborting startup or retrying", async () => { + stateFactory.mockImplementationOnce(() => { + throw Object.assign(new Error("private telemetry path cannot be written"), { code: "EACCES" }); + }); + const warning = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { TelemetryClient } = await import("@paperclipai/shared/telemetry"); + const periodicFlush = vi.spyOn(TelemetryClient.prototype, "startPeriodicFlush"); + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(periodicFlush).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + expect(warning.mock.calls).toEqual([ + ["[telemetry] Initialization failed; telemetry is disabled for this process."], + ]); + }); + + it("continues startup when the real state store cannot create its directory", async () => { + const fixtureDir = mkdtempSync(path.join(tmpdir(), "paperclip-telemetry-startup-")); + fixtureDirs.push(fixtureDir); + const blockedStateDir = path.join(fixtureDir, "telemetry"); + writeFileSync(blockedStateDir, "this file prevents directory creation"); + const { loadOrCreateState } = await vi.importActual( + "@paperclipai/shared/telemetry", + ); + stateFactory.mockImplementationOnce(() => loadOrCreateState(blockedStateDir, "0.0.0-test")); + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("does not initialize state or send when disabled in file configuration", async () => { + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + expect(initTelemetry({ enabled: false })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(stateFactory).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }); + + it.each(["PAPERCLIP_TELEMETRY_DISABLED", "DO_NOT_TRACK", "CI", "GITHUB_ACTIONS"])( + "respects %s before creating the client or state", + async (key) => { + vi.stubEnv(key, "1"); + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(stateFactory).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/server/src/telemetry.ts b/server/src/telemetry.ts index eb2b28749e6..bfa7c1930bb 100644 --- a/server/src/telemetry.ts +++ b/server/src/telemetry.ts @@ -3,26 +3,38 @@ import { TelemetryClient, resolveTelemetryConfig, loadOrCreateState, + trackServerStarted, } from "@paperclipai/shared/telemetry"; import { resolvePaperclipInstanceRoot } from "./home-paths.js"; import { serverVersion } from "./version.js"; let client: TelemetryClient | null = null; +let initializationFailed = false; export function initTelemetry(fileConfig?: { enabled?: boolean }): TelemetryClient | null { - if (client) return client; + if (client || initializationFailed) return client; const config = resolveTelemetryConfig(fileConfig); if (!config.enabled) return null; - const stateDir = path.join(resolvePaperclipInstanceRoot(), "telemetry"); - client = new TelemetryClient( - config, - () => loadOrCreateState(stateDir, serverVersion), - serverVersion, - ); - client.startPeriodicFlush(60_000); - return client; + let pendingClient: TelemetryClient | null = null; + try { + const stateDir = path.join(resolvePaperclipInstanceRoot(), "telemetry"); + pendingClient = new TelemetryClient( + config, + () => loadOrCreateState(stateDir, serverVersion), + serverVersion, + ); + trackServerStarted(pendingClient, { os_platform: process.platform }); + pendingClient.startPeriodicFlush(60_000); + client = pendingClient; + return client; + } catch { + pendingClient?.stop(); + initializationFailed = true; + console.warn("[telemetry] Initialization failed; telemetry is disabled for this process."); + return null; + } } export function getTelemetryClient(): TelemetryClient | null { From ce7766dc11c4515190d48d395f26a90bba900f8a Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:06:50 -0600 Subject: [PATCH 23/28] fix(recovery): resume retries with remote workspace storage (#18910) ## Thinking Path > - Paperclip manages work performed by AI agents. > - Native recovery must resume an interrupted run under its saved ownership. > - Remote workspace storage records that ownership in `runnerProfileJson.remoteWorkspace`. > - Recovery recognized only the older workspace record and plugin task leases. > - A retained remote PID could therefore leave a due retry without a dispatch. > - This PR routes the new record through the existing remote recovery verifier. > - Checkpoint, lease, process and single-executor checks remain authoritative. ## Linked Issues or Issue Description **What happened?** After a native sandbox turn returned without a semantic result, same-run recovery was saved with a retry deadline. The run retained its sandbox PID and the new remote workspace reference. The recovery selector skipped it. Later task attempts could then fail on the unfinished workspace writer. **What did you expect to happen?** The due retry must enter existing remote checkpoint and ownership verification. Recovery must resume the original run once or expose the real verification blocker. **Steps to reproduce** Persist a retryable native failure with a remote PID, a prepared `paperclip.remote-workspace-run.v1` reference, an existing lease and checkpoint, then run heartbeat recovery. The added PostgreSQL cases reproduce this shape. **Deployment mode** Native Paperclip Runner in a sandbox with remote workspace storage. Related: #18841 handles legacy workspace recovery during storage migration. #18908 handles failed native preparation. This repair covers a later, provider-started retry. ## What Changed - Recognize the current remote workspace record when classifying process location and selecting remote retries. - Extend the existing database and real-process fixtures to cover the new record, including a retained PID without a process group. - Reuse existing restart recovery. Add no new retry loop, ownership bypass, migration or workflow change. ## Verification - Before the repair: three new behavioral cases failed. Two retries never dispatched; a missing checkpoint never reached its visible recovery hold. - After the repair: all 82 heartbeat recovery tests pass. - All 57 neighboring resumption-binding, restart-recovery and remote-probe tests pass. - Safety cases retain retry deadlines, live controller ownership, active execution, missing-checkpoint rejection, single dispatch and zero host signaling of the sandbox PID. - Local platform: macOS arm64, Node 24.22.2, PostgreSQL 18.1. The tests use disposable databases and an owned live process as the remote PID collision control. Provider execution is outside this routing regression. - Full `pnpm -r typecheck` and `pnpm build` pass at `34aa2d08d38e8afec7f8d98ce984a74551caeda6`. - Current-head GitHub checks: 53 successful; `ci / verify` passed. Paperclip Review completed neutral with a 5/5 assessment and no findings. Two Storybook jobs were skipped by scope; the external security scan was neutral. No failures, pending checks, or merge conflicts remain. - Greptile: 5/5 at the same commit, with no review threads. - The duplicate full local `pnpm test:run` was interrupted after current-head CI passed. Its first two chat shards passed (574 cases); it has no complete local full-suite verdict. This is separate from the passing targeted recovery suites and required CI test lanes. - Staging diagnosis used read-only run, finalization, workspace and lease evidence. The fixed revision has not been deployed. The original task's useful continuation remains unverified. - Initial local attempts stopped before test execution because generated SDK output and PostgreSQL library aliases were missing. Those setup failures are retained separately from the behavioral red/green evidence. ## Risks A wrong remote classification could send a run to the wrong ownership path. The existing verifier still checks the saved checkpoint, controller lease and target. Tests require one dispatch, preserve process identifiers, and prove that recovery never inspects or signals a coincident host PID. Legacy workspace and plugin task coverage remains green. This repairs missed continuation; it does not establish the cause of the earlier missing semantic result. ## Model Used OpenAI GPT-6 through Codex, with reasoning and tool execution. The exact model snapshot and context window were not exposed. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- .../src/services/heartbeat/recovery.test.ts | 28 +++++++++++++------ server/src/services/heartbeat/recovery.ts | 6 ++-- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index 73a665bb53d..d109fcec13c 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -310,7 +310,7 @@ describePostgres("heartbeat recovery module ownership", () => { } finally { try { await f.cleanup(); } finally { await child.stop(); } } }); - async function remoteRetryFixture(pid: number, target: "workspace" | "plugin_task" = "workspace") { + async function remoteRetryFixture(pid: number, target: "workspace" | "remote_workspace" | "plugin_task" = "workspace") { const fixture = await nativeProcessFixture(pid, "remote", "failed"); const sessionId = randomUUID(); const [lease] = await db.insert(environmentLeases).values({ companyId: fixture.run.companyId, @@ -319,13 +319,17 @@ describePostgres("heartbeat recovery module ownership", () => { }).returning(); await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, fixture.run.agentId)); await db.update(issues).set({ executionRunId: fixture.run.id }).where(eq(issues.id, fixture.run.issueId!)); - await db.update(heartbeatRuns).set({ nativeSessionId: sessionId, runnerProfileJson: { + await db.update(heartbeatRuns).set({ nativeSessionId: sessionId, + ...(target === "remote_workspace" ? { processGroupId: null } : {}), runnerProfileJson: { sessionCheckpoint: { identity: { runId: fixture.run.id, companyId: fixture.run.companyId, issueId: fixture.run.issueId, agentId: fixture.run.agentId, sessionId }, providerSessionId: "existing-provider" }, ...(target === "workspace" ? { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", workspaceId: randomUUID(), leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, resourceDisposition: "keep_running" } } : {}), + ...(target === "remote_workspace" ? { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", + workspaceId: randomUUID(), epoch: 3, leaseId: lease.id, providerLeaseId: lease.providerLeaseId, + remoteCwd: "/work", state: "prepared", snapshotId: randomUUID() } } : {}), } }).where(eq(heartbeatRuns.id, fixture.run.id)); await db.insert(nativeRunnerCheckpoints).values({ runId: fixture.run.id, companyId: fixture.run.companyId, issueId: fixture.run.issueId!, part: "opaque-checkpoint-record", material: { fixture: true } }); @@ -335,9 +339,12 @@ describePostgres("heartbeat recovery module ownership", () => { return { ...fixture, lease }; } - it.each(["failed", "running"] as const)("routes a due remote %s retry through verified recovery without local PID inspection", async status => { + it.each((["workspace", "remote_workspace"] as const).flatMap(target => + (["failed", "running"] as const).map(status => ({ target, status })), + ))("routes a due $target $status retry through verified recovery without local PID inspection", async ({ target, status }) => { const child = await ownedProcess(); - const f = await remoteRetryFixture(child.pid); + const f = await remoteRetryFixture(child.pid, target); + const processGroupId = target === "remote_workspace" ? null : child.pid; const originalKill = process.kill.bind(process); const kill = vi.spyOn(process, "kill").mockImplementation(originalKill); try { @@ -351,7 +358,7 @@ describePostgres("heartbeat recovery module ownership", () => { remote: { providerLeaseId: f.lease.providerLeaseId, remoteCwd: "/work" } }), })); expect(native.claims).toHaveBeenCalledWith(expect.objectContaining({ runIds: [f.run.id] })); - expect(await f.current()).toMatchObject({ status: "running", processPid: child.pid, processGroupId: child.pid }); + expect(await f.current()).toMatchObject({ status: "running", processPid: child.pid, processGroupId }); expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]); expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false); await recovery.reapOrphanedRuns(); @@ -378,10 +385,13 @@ describePostgres("heartbeat recovery module ownership", () => { } finally { try { await f.cleanup(); } finally { await child.stop(); } } }); - it.each(["future_retry", "live_owner", "missing_checkpoint", "active_execution"] as const)( - "does not execute a remote retry without current authority: %s", async condition => { + it.each((["workspace", "remote_workspace"] as const).flatMap(target => + (["future_retry", "live_owner", "missing_checkpoint", "active_execution"] as const).map(condition => ({ target, condition })), + ))( + "does not execute a $target retry without current authority: $condition", async ({ target, condition }) => { const child = await ownedProcess(); - const f = await remoteRetryFixture(child.pid); + const f = await remoteRetryFixture(child.pid, target); + const processGroupId = target === "remote_workspace" ? null : child.pid; try { if (condition === "future_retry") await db.update(nativeRunFinalizations) .set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id)); @@ -397,7 +407,7 @@ describePostgres("heartbeat recovery module ownership", () => { await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns(); expect(f.deps.executeRun).not.toHaveBeenCalled(); expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled(); - expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid }); + expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId }); if (condition === "future_retry") expect(native.claims).not.toHaveBeenCalled(); if (condition === "missing_checkpoint") { const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id)); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 2912b041065..739a7584444 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1862,7 +1862,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD } async function nativeProcessIsLocal(run: HeartbeatRun) { - if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) return false; + const profile = parseObject(run.runnerProfileJson); + if (profile.nativeWorkspaceSync != null || profile.remoteWorkspace != null) return false; const [identity] = await db.select({ eventType: heartbeatRunEvents.eventType, agentId: heartbeatRunEvents.agentId, @@ -2068,7 +2069,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD if (!await nativeProcessIsLocal(run)) { if (nextAttemptAt && nextAttemptAt > now) continue; if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) continue; - if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) { + const profile = parseObject(run.runnerProfileJson); + if (profile.nativeWorkspaceSync != null || profile.remoteWorkspace != null) { remoteRetryRunIds.push(run.id); } else { const [taskLease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( From 0884e73b46ee064b3d242e467035dde6c32e5cb5 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:13:40 -0600 Subject: [PATCH 24/28] test(e2e): verify saved assigned skill use (#18783) ## Thinking Path > - Paperclip lets people manage agent work and reusable company skills. > - A saved assigned skill must affect a task's actual execution. > - Existing QA coverage creates and edits skills. > - The daily target needs coverage for ordinary saved-skill use. > - The task receives its receipt data only through one saved skill. > - The admitted immutable version, native read and saved document establish the outcome. > - This PR adds one bounded journey for that gap. > - A shared lease test also needs a stable clock to assert its exact timeout. ## Linked Issues or Issue Description Refs: #18305. Supporting repair #18885 merged as `057898530c702ec14c1e14d163e96dfb045cd8c0`. It closes the recovery-conflict prerequisite. This branch does not duplicate that production patch. **What existing behavior does this improve?** Product E2E coverage for ordinary assigned company skill use on an existing deployment target. **Current behavior** The deployment journey catalog can create and edit a company skill. It has no separate case that requires a native task to load its assigned saved skill and use data available only inside that source. **Proposed behavior** Create one synthetic managed company skill through the public API. Assign its canonical key with the normal Latest selection. Create the task through the production browser composer. Require one native work run, the exact created immutable version in its admitted skill-version map, a successful native source read or load, and an agent-authored saved receipt. Reload and verify the result, revision and original source. ## What Changed - Add `saved-skill-use` with one native work run and seven required checks. The first check is `saved-skill-assignment-bound`. - Use ordinary `desiredSkills: [canonicalKey]` assignment. Require the returned stored assignment to have `versionId: null`. This works with Beta skills off and reads or changes no experimental setting. - Bind the creation receipt, pre-dispatch source read and version file inventory. Require a managed skill with the exact company key. Require the run's authoritative `skillSelectionSnapshot: true` and admitted version map to match the created immutable version. - Keep the random receipt reference and numeric inputs only in the saved source. The task prompt names the skill without supplying its answer. - Require exact company, task, agent, run, turn and runner bindings for a complete successful native source receipt. Accept the deployed OpenCode read wrapper only when both the saved assignment and native execution input bind its runtime directory to the created immutable skill version. Also accept the first-party assigned-skill read with a provider-independent canonical key/version binding from both the saved API assignment and admitted native input. Require the exact full source, checksum, byte count and complete page. Keep the materialized directory-name guard specific to the OpenCode filesystem path. Reject claims, shell output, foreign or incomplete reads and wrong skill loads. - Add a narrow Codex reader for the retained controller `semantic_tool.input` and `semantic_tool.result` pair. Require the actual native provider, runner, session, call, turn and item; the admitted immutable key/version; the read-only `read_assigned_skill` operation; and its successful active-task receipt. Recompute the input digest and the exact complete seven-field read-result digest from the saved source. Raw dynamic or MCP names grant no authority. Keep the builtin OpenCode/filesystem reader body unchanged. - Require authoritative native commit before qualifying the saved receipt. Independently check its value and same-run document revision authorship. Reload and verify the result and unchanged source. Retain those existing readback receipts in the private checkpoint for independent verification. - Keep the exact raw campaign ID in the journal, API name, frontmatter and saved name. A lost successful creation response remains recoverable by name. - Register discovery, source fingerprints, closed checks and existing feature associations. Update the case and fixture docs. Feature-map files are unchanged. - Stabilize one existing shared sandbox lease regression with a scoped `Date.now` spy and `finally` restoration. Keep its exact timeout, no-destroy, no-replacement and recorded lease assertions. Change no production service code. ## Verification - **Genuine QA staging qualification passed on this exact `2fb654f7be0bac3bfeb4ec6b7a2eb7a06b8de7e4` head**: [workflow41763421118](https://github.com/paperclipai/paperclip-cloud/actions/runs/41763421118), [TestOps launch2257/result4034](https://paperclip.testops.cloud/launch/2257/tree/4034?treeId=0), [sanitized report](https://pages.paperclip.ing/evals-qa-smoke-41763421118-1/). All seven required checks and all five phases passed, with exactly one configured OpenCode/DeepSeek native run succeeded and committed. Owned cleanup closed. A separate source-pinned replay verified25 original-evidence controls and16 rejection controls without altering originals. Automatic #testops PASS was independently read back. Both earlier failed QA grades remain unchanged. - Current-head CI `41758727042` attempt 2 passed with 48 successful jobs and one skip. The coordinator authorized one unchanged-source failed-jobs diagnostic. Start and end timestamps prove three fresh executions (Runner package, server shard 7 and aggregate verification), 45 retained successful executions and one skipped non-execution. The prepared-context test passed in 2,534 ms. All 28 repository rehome tests passed. The original failed attempt remains sealed. Its timeout causes are still unknown. No source, timeout or assertion changed, and no reliability fix is claimed. - Fresh exact-head Greptile review is 5/5 with zero unresolved threads. All 58 current check-runs and the status rollup are terminal and clean. The native Paperclip review ended neutral and incomplete, so it is not a validated assessment. The PR remains mergeable after the dependency case merged in #18901. The source remains frozen. A coordinator-only qualification draft is prepared with approval false. A genuine QA staging pass is still required before daily enrollment. - Current head `2fb654f7be0bac3bfeb4ec6b7a2eb7a06b8de7e4`: 269 focused tests across eight files passed. These include 42 Codex producer, converter and false-positive controls. The canonical E2E typecheck passed with its own declaration prerequisites. Credential-free discovery lists 24 journeys. Fresh exact-head Greptile review passed at 5/5 with zero unresolved threads. CI `41758727042` attempt 1 failed. It completed 45 successful jobs, one skip and three failed jobs. The primary failures were one existing OpenCode prepared-context test at its unchanged 5-second timeout and three existing repository rehome tests at their unchanged 15-second timeouts. The third failed job aggregates those results. The timeout causes are unknown. The original attempt remains sealed. A direct passing twin ran the prepared-context test in 2,533 ms. All package, script, CLI, UI, workflow and root inputs match between the actual tested merges. Three server files differ, including `task-retry.ts` inside the broader rehome import graph; no full-server equality or causal fix is claimed. The later single diagnostic pass is recorded above. The native Paperclip review ended neutral with an incomplete execution, so it is not a validated assessment. No live QA pass is claimed. - The valid review on `e1a474` found that Codex displays its first-party tool as dynamic. Source inspection confirmed that its raw name has no authority. The retained controller semantic input/result pair supplies the real operation authority. Core `09162fe893b464dd2ccdafd14bd3ef1611202a86` and current source have identical relevant envelope, digest, result-unwrapping and immutable-read implementation regions. - Three real-producer positive controls failed against the old grader. All 42 now pass. The tests call the actual immutable-file reader, production protocol digest builders, Codex display converter and API redaction. They validate the event schema and cover omitted defaults, the exact runtime alias and Unicode bytes. Missing, duplicate, reordered, failed, foreign, MCP, unbound, partial-page, wrong-digest, wrong-key/version and nonce-only evidence fails. Generic dynamic output fails even with copied full text and invented read-only markers. - The original builtin reader body and source-only fixture function are byte-identical to `e1a474`. All seven checks, the one-run limit, 500-cent guard, ordinary Latest assignment and Beta-off behavior remain unchanged. The independent verifier is pinned to this new head. Its synthetic semantic receipt passes 24 negative controls and its filesystem receipt passes 16. Older helpers and both original failed QA seals remain unchanged. - Earlier integrated head `e1a47438b1c7e51a5bb2896a4ec9b6ed6560318b`: 376 focused checks across eleven files passed, including Saved, settings and Chat overlap controls. All seven canonical server type layers and exact-head E2E types passed. All four lease controls passed. Credential-free discovery lists 24 catalog journeys. No new staging run has occurred. - Normal integrations consume actual master `fbb7055b9020e505cd1bf903b1b1c301ba8f3eef` and actual Chat squash `d7283e875772ca8c684dc86e471e6b48d543b7b1`. Preserve all five Saved case/parser files byte-for-byte and retain settings and Chat registrations. The only manual resolutions were two additive handler/type anchors. All seven checks, one-run limit and 500-cent guard remain unchanged. The independent verifier is pinned to this exact head. - Earlier dbdf CI passed after one authorized failed-jobs recheck of a confirmed Spot-terminated Build runner. Timestamp comparison proves two fresh executions (Build and verify), 46 retained successful executions and one skip. The original failed attempt remains sealed. Its native Paperclip Review completed neutral with an incomplete execution and is not a validated assessment. - New master entries made the earlier dbdf PR conflict. The frozen qualification guard correctly blocked dispatch before any pin or receipt. Preserve its unapproved draft as history; it did not authorize a QA run. This normal integration resolves the observed conflict. Fresh current-head CI/review and one genuine staging pass remain required. - Earlier merged head `dbdf1f3cbeaacc7cf9a2a457618035c945bd09f0`: 227 focused checks passed across seven files. All seven canonical server type layers passed. The E2E typecheck passed using the declarations built in this same checkout on this exact head. All four existing clock and lease controls passed. Credential-free discovery lists 22 catalog journeys, including both saved-skill use and task lifecycle. These are local checks, not a live staging qualification. - Normal merge of master `8771606fd87f69ed4057779e3dce3a9bca8088e6` retains the reviewed task-lifecycle case and its Cursor fixture repair from #18782. Resolve six shared journey registrations by preserving both cases. The five saved-skill case files remain byte-for-byte equal to reviewed `ff3814`. All seven required checks, one-run limit, 500-cent bound and ordinary Beta-off Latest assignment remain unchanged. No feature-map delta was added. - The independent sealed-run verifier is now pinned to this exact merged head. Synthetic semantic evidence passed with 24 negative controls. Synthetic filesystem evidence passed with 16 negative controls. Foreign document authorship and missing reopen/source readback still fail. The old `ff3814` verifier and both original failed QA seals remain unchanged. - Common generated outputs were copied only after all 4,225 package, script and root inputs matched, installed lock/TypeScript hashes matched, and all 4,955 source output paths and hashes passed before and after copy. No server declarations or editable source/module links were copied. The server declarations were rebuilt locally. A full frozen workspace install disabled lifecycle scripts and used the same locked packages. No Rust build ran locally. - Previous `ff3814` CI `41733864320` exposed a real wrapped PostgreSQL uniqueness error in a recovery server test. That production defect is independently fixed and tested in #18885. The later supporting PR merged separately. This branch does not duplicate its production patch or regrade the original failure. Preserve the original failed result and the separate Spot interruption. New full CI and a fresh exact-head 5/5 review remain required. - Previously, **225 tests passed across six files** in the focused evidence, flow, contract, catalog and report set after the semantic-read correction. The prior 181-test proof and all original failed attempts remain retained. This correction changes only the saved-skill evidence helper, flow argument, their tests and case documentation. The shared clock repair remains unchanged. - E2E TypeScript check passed on Node **24.22.2**. - Earlier credential-free discovery listed 21 cases and exposes the ordinary saved-skill journey without a Beta prerequisite. These local checks make no provider or model calls. - Source inspection of Core `09162fe893b464dd2ccdafd14bd3ef1611202a86` confirms key-only selection becomes a stored null Latest selection. Managed execution resolves its current immutable version and records the admitted map in `contextSnapshot`. The public run API preserves that map independently of optional result presentation data. - Seven invalid admitted-version fixtures wrongly qualified without the new guard. The regressions now reject them before the read or document can qualify. Beta-off and Beta-on ordinary request controls use the production sync validator. Wrong assignments, missing null selection, source drift, creation-version mismatch and another skill's complete native output cannot pass. - Existing native commit, null/compact result data, read provenance, nonce, saved-result, reload and lost-creation-response controls remain required. Their original red results are retained. - The earlier experimental version-pin variant remains unqualified. Its missing-prerequisite regression and the denied target-setting attempt remain separate failure evidence. This case qualifies ordinary assignment and the version actually admitted by the server. - Normal merge of master `5f1b59ba2c9adfb40a681bf419a8b7e7837d1125` includes the reviewed inbox snapshot fix #18798 and App/Shell access-fixture fix #18799. The merge has no conflicts. All 17 journey source, helper, registration and documentation files remain byte-identical to reviewed head `eab260955ff43eeacfdafc9b75dfc9b116f0eb79`. - Previous CI `41672686614` failed two UI test shards with obsolete user-ID fixtures; its aggregate verify job also failed. The merged access-fixture repair addresses those failures. This is a new source CI run; the failed run is retained. - Dependency output reuse follows exact path and byte checks for 10,977 tracked build inputs and 3,411 generated outputs. The first copy guard detected 12 bundles regenerated after its initial manifest and stopped before any destination mutation. The stable replacement manifest passed all checks before copying. Final focused tests, E2E typecheck and discovery passed after the verified copy. No editable source or module link was shared. - An earlier integration found stale generated Cursor Cloud declarations. Canonical dependency builds refreshed them, then typecheck passed. The original typecheck failure remains recorded. - The initial report support failure came from a blocked `tsx` IPC pipe before report execution. The unchanged test passed with normal local IPC permission. The original failure and diagnostic remain recorded. - `git diff --check` passes. Full workspace recursive typecheck, unit run and build remain assigned to hosted CI and coordinator verification. Full CI `41679720041` passed on the prior `f3f288a54177f2b9a7b225b1d1ab35440dfc597c` source. A clean 5/5 review passed on `0f445cf1c622afb25a1af92398784209df542696`. Fresh CI and review are required after the shared clock fixture repair. - The first real QA staging attempt on `f3f288a54177f2b9a7b225b1d1ab35440dfc597c` failed `saved-skill-native-read-attributed`. Assignment, immutable admitted version and cleanup passed. The test stopped before document, reload and source-after observations. The original failed attempt and its complete evidence remain sealed. - The original successful builtin `read` event contained the full saved procedure. OpenCode returned its file wrapper with an unknown normalized operation, null target and a hashed runtime skill name. The old parser rejected that shape. A synthetic regression failed before this correction. The corrected parser recognizes that exact retained read in memory, with no change to the original grade. Wrong paths, copied shell text, missing starts, mismatched calls, failed/truncated reads and wrong version/name bindings still fail. A later staging attempt is described below. Neither failed original was regraded. - Original CI `41703535552` failed one shared server test and its dependent verification job. The lease-resume test expected an RPC timeout of 31,234 ms, but the runtime correctly supplied 31,233 ms after one elapsed millisecond. The test and runtime service were byte-identical to the previous passing source. The failed run remains recorded. - A scoped advancing clock reproduced that exact 31,233-versus-31,234 failure locally. A stable clock around only the acquisition call now preserves the exact original assertion. The repaired test and three related sandbox lifecycle controls pass on disposable embedded Postgres. No production timeout, lease preservation, cleanup or replacement behavior changed. - The first local reproduction stopped before test execution because six same-package native Postgres library aliases were missing. Their exact existing versioned libraries matched the canonical dependency. Restoring only those internal relative aliases allowed the separate deterministic red and green measurements. This prerequisite failure remains recorded. - The second real staging attempt on `02582aadd23fc128ae29daa69482113cf0f95ad1` also failed `saved-skill-native-read-attributed`. Authentication, setup, assignment, source-only input, collection and cleanup passed. It completed one native run. Later document, reload and source-after checks were not executed. The complete original remains sealed as failed. - That attempt used the ordinary first-party `paperclip_read_assigned_skill` tool. Its complete native receipt contained the exact canonical skill key, created version, `SKILL.md` bytes, SHA-256, byte count and null next-page offset. Core verifies the pinned manifest and file checksum before returning that receipt. The old grader recognized only filesystem and native skill-load output. An actual-shape regression failed before this correction. The corrected parser recognizes the retained read in memory. Wrong keys, versions, source bytes, hashes, partial pages, semantic failures, copied shell output and foreign or unpaired native calls still fail. No model prompt, product source or original grade changed. A live pass on this corrected source remains required. - CI `41708636059` on the previous `02582` source passed at attempt 3. The first two interrupted attempts remain recorded. AWS evidence identified Spot capacity termination for the second Rust runner. The targeted retry executed Rust and aggregate verification; 46 successful jobs retained their exact prior execution timestamps and one job remained skipped. New metadata IDs do not establish new executions. Fresh full CI and review remain required for this source. - Earlier CI `41730743279` passed on `1ee86838ea528edd8c60fb9ddd858faf3c83e783`. A later review finding correctly showed that the semantic read still depended on an OpenCode-only directory-name helper. That finding remains recorded. Three local Codex, ACPX Cursor and Cursor Cloud flow controls reproduced the rejection before this correction. - The semantic path now binds the canonical key and immutable version through the common native runtime skill selection contract. Its checksum, complete source, page and native call attribution checks stay unchanged. Missing, duplicate, wrong-key and foreign-version API/native assignment controls still fail for all three provider configurations. The OpenCode filesystem wrapper still requires its exact provider and materialized directory name. These local controls test the existing canonical receipt shape; they do not claim live provider qualification or add a guessed provider encoding. Fresh CI and review are required after this correction. - This public PR does not enroll the case in the daily plan or change protected QA configuration. ## Risks The target must supply a disposable native agent with normal company skill-write access and cleanup. It must preserve the authoritative admitted version map and provide a complete source read or named native skill-load receipt. Missing evidence fails the case even when the receipt document has the right value. The OpenCode wrapper path requires matching assignment and native input records for the same immutable version; it does not accept a generic targetless read. The builtin first-party path requires the same version binding and exact full-source receipt. Codex must provide its authoritative controller pair and a digest that commits to that exact complete result. Partial pages, unknown operations and semantic failures do not qualify. Local flow controls simulate the target API and browser boundaries. They do not establish live Cloud qualification. Experimental assignment pins need separate qualification. The journey changes only its owned skill, assignment and task. It changes no shared agent, setting, model, connection or budget. ## Model Used OpenAI GPT-6 through Codex with reasoning and code execution. The runtime does not expose a more specific model identifier or context-window size. Local checks made no product-model calls. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/evals.md | 2 + .../src/__tests__/environment-runtime.test.ts | 28 +- tests/runner-e2e/FIXTURES.md | 8 + tests/runner-e2e/PRODUCT-JOURNEYS.md | 1 + tests/runner-e2e/SECURITY.md | 7 + tests/runner-e2e/catalog.test.ts | 1 + tests/runner-e2e/journey-features.ts | 1 + .../product-journeys/agent-flows.ts | 3 +- tests/runner-e2e/product-journeys/catalog.ts | 1 + .../product-journeys/contracts.test.ts | 26 ++ tests/runner-e2e/product-journeys/launch.ts | 7 +- .../saved-skill-use-codex.test.ts | 157 ++++++++ .../saved-skill-use-evidence.test.ts | 311 +++++++++++++++ .../saved-skill-use-evidence.ts | 238 +++++++++++ .../saved-skill-use-flow.test.ts | 370 ++++++++++++++++++ .../product-journeys/saved-skill-use-flow.ts | 129 ++++++ .../product-journeys/saved-skill-use.md | 97 +++++ .../product-journeys/success-contract.ts | 2 + tests/runner-e2e/product-journeys/types.ts | 3 +- 19 files changed, 1378 insertions(+), 14 deletions(-) create mode 100644 tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts create mode 100644 tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts create mode 100644 tests/runner-e2e/product-journeys/saved-skill-use-flow.ts create mode 100644 tests/runner-e2e/product-journeys/saved-skill-use.md diff --git a/doc/evals.md b/doc/evals.md index 8a57f31089e..f59035c2843 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -125,6 +125,8 @@ It requires a fresh owned agent with no prior work runs and uses three provider Cloud fleet operations and trusted CI remain private. Local eval gateway qualification does not qualify Cloud routes, and these selections stay outside `--all`. +The deployment-targeted [saved skill journey](../tests/runner-e2e/product-journeys/saved-skill-use.md) checks one company skill assigned by key, its exact admitted immutable version, a bound native source read/load, and a run-authored saved result from skill-only inputs. It uses the reusable Product journey target and remains unqualified until a real target run passes. + The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) contains arithmetic and receipt smoke cells for a same-task round trip among three native agents using real case/task APIs, saved documents, handoff receipts, and browser readback. Receipt smoke has diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 21738faa5c0..9231fb7a089 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -3808,17 +3808,23 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); - await expect(runtimeWithPlugin.acquireRunLease({ - companyId, - environment, - issueId: null, - agentId, - heartbeatRunId: runId, - persistedExecutionWorkspace: { - id: executionWorkspaceId, - mode: "shared_workspace", - }, - })).rejects.toThrow("the lease was preserved and no replacement was created"); + // Keep the exact configured budget stable across deadline creation and RPC dispatch. + const now = vi.spyOn(Date, "now").mockReturnValue(Date.now()); + try { + await expect(runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + agentId, + heartbeatRunId: runId, + persistedExecutionWorkspace: { + id: executionWorkspaceId, + mode: "shared_workspace", + }, + })).rejects.toThrow("the lease was preserved and no replacement was created"); + } finally { + now.mockRestore(); + } expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentResumeLease", expect.objectContaining({ driverKey: "fake-plugin", diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 6e653494ad6..ce803748eb6 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -1,5 +1,13 @@ # Runner E2E fixture authoring +The deployment-targeted [saved skill journey](product-journeys/saved-skill-use.md) +creates and assigns one campaign-owned synthetic skill by key through public APIs, then +creates the measured task through the browser. The saved outcome must come from +one native run. Setup never supplies the output document or its native read +receipt. The ordinary Latest assignment, immutable created version and admitted +run version must agree. Skill ownership is journaled before creation and cleaned by the target +provider with the other owned resources. + [Chat feature evals](CHAT-FEATURES.md) load two authored YAML requests and seed only company, agents and project context through public APIs. Both agents use production role instructions; evaluator outcomes are never injected. Real browser diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 1b73f1e566a..eec3849f077 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -26,6 +26,7 @@ never start a server or depend on a tenant filesystem. | `repository-continuation` | Native agent; two work runs; a pinned public read-only repository | Verify independent Git bundle history and workspace bytes across two turns, preserve the original random file, and reopen registered outputs after reload. See [repository-continuation.md](product-journeys/repository-continuation.md) | | `routine-scheduled-run` | Native agent; one work run; an owned project and routine | Browser-create a routine, enable its minute schedule, verify one scheduled invocation and run-attributed saved output, then pause and observe the next schedule boundary with no new work | | `company-access-isolation` | Two registered QA Cloud workspaces; disposable company-bound agent credentials; zero model runs | Switch through the real tenant Cloud menu and return. Require allowed task reads and task creation, foreign read/create denial in both directions, exact canonical creator evidence, and key revocation. See [COMPANY-ACCESS.md](product-journeys/COMPANY-ACCESS.md) | +| `saved-skill-use` | Native agent with skill-write capability; one work run; one synthetic company skill assigned by key using Latest | Load the assigned saved skill, use inputs available only in its source, and verify a same-run-authored task document after reload. Bind the immutable created version to the run's admitted skill-version map. Require a complete bound native source read/load and preserve the exact source and version. Works with Beta skills off. See [saved-skill-use.md](product-journeys/saved-skill-use.md) | | `file-input-answer` | Native agent; one work run; synthetic CSV uploaded through the composer | Verify original uploaded bytes, independently calculated accepted-row count and units, a run-attributed task document after reload, and preservation of the input attachment. The task prompt contains neither the random file reference nor the answers | | `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 5abc39cb853..1c6969ebcf7 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -1,5 +1,12 @@ # Runner E2E security for a public repository +The deployment-targeted `saved-skill-use` journey creates one synthetic skill on +the campaign-owned agent. Its random receipt reference and numeric inputs occur +only in the saved skill source. Exact source, immutable version, ordinary assignment, native read +events and run-authored document revisions remain in the private bundle. The +existing public projection exports closed checks and the authenticated owned +task screenshot. The journey changes no shared agent, instance flag or budget. + Native `cursor-cloud` acceptance uses a separate callback-only loopback listener. Its external HTTPS tunnel must never target the local_trusted operator app. Credential-free ingress checks require `/` and `/api/health` to return 404 and diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index e5e598a5e78..3729583453d 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -127,6 +127,7 @@ describe("runner E2E catalog", () => { it("exports deployment journeys separately from the model matrix and default selection", () => { expect(productJourneys.map(journey => journey.id)).toEqual([ + "saved-skill-use", "task-edit-lifecycle", "company-access-isolation", "repository-continuation", diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 1816d62bd96..157f2ffce49 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,7 @@ function associate(suites: readonly string[], cases: Record; + | "repository-continuation" + | "saved-skill-use">; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 7467ae5d6bc..95e51b6f072 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -2,6 +2,7 @@ import { agentJourneyRuns } from "./success-contract.js"; export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ + { id: "saved-skill-use", label: "Use a saved assigned skill and verify its run-authored result", requires: ["company", "board-user", "task-write", "agent", "skill-write"], capabilities: ["skills/policy", "skills/versions", "documents-artifacts/documents"], paid: true }, { id: "task-edit-lifecycle", label: "Edit task properties, complete and reopen with saved context", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/properties", "tasks/completion"], paid: false }, { id: "company-access-isolation", label: "Switch QA workspaces and verify restricted agent isolation", requires: ["company", "board-user", "task-write", "company-access"], capabilities: ["companies/selection", "access/resources"], paid: false }, { id: "repository-continuation", label: "Continue repository work with the same saved workspace", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["projects/task-context", "documents-artifacts/attachments"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts index b3d3bd7b441..a702406ee61 100644 --- a/tests/runner-e2e/product-journeys/contracts.test.ts +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -39,6 +39,32 @@ function connectReceipt(): OnboardingConnectEvidence { }; } describe("deployment journey contracts", () => { + it("discovers an ordinary assigned-skill journey without an experimental pin prerequisite", () => { + const discovered = productJourneys.find(journey => journey.id === "saved-skill-use")!; + expect(discovered).not.toHaveProperty("requiredSettings"); + expect(journeyRequiredChecks[discovered.id]?.[0]).toBe("saved-skill-assignment-bound"); + const config = parseTargetConfig({ ...local, journeys: ["saved-skill-use"], + limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 1 } }); + expect(preparationPreview(config).journeys).toContainEqual(discovered); + expect(config.baseline.features).toEqual({}); + }); + it("requires one native saved-skill run and every closed source/use/result check", () => { + const config = { ...local, id: "qa", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["saved-skill-use"], limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 1 } }; + expect(parseTargetConfig(config).journeys).toEqual(["saved-skill-use"]); + for (const maxRuns of [0, 2]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow("agent_journeys_require_bounded_runs"); + const retained = result(); + Object.assign(retained, { caseId: "saved-skill-use", provider: "opencode", model: "configured-model", runtimeMode: "native", runIds: ["run"] }); + retained.productJourney!.checks = journeyRequiredChecks[retained.caseId]!.map(id => ({ id, passed: true })); + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + for (const check of retained.productJourney!.checks) { + const missing = structuredClone(retained); + missing.productJourney!.checks = missing.productJourney!.checks.filter(row => row.id !== check.id); + expect(() => validateRetainedRunnerResult(missing)).toThrow("result.productJourney"); + } + retained.runIds = []; + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.agentJourneyRun"); + }); it("accepts historical results without Connect diagnostics and optional closed receipts", () => { const retained = result(); retained.caseId = "cloud-onboarding"; diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 35337dcc765..930112167c5 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -23,6 +23,7 @@ import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; import { taskDependencyResume } from "./task-dependency-resume.js"; import { runInputFile } from "./input-file-flow.js"; +import { runSavedSkillUse } from "./saved-skill-use-flow.js"; import { runAgentSettings } from "./agent-settings-flow.js"; import { companyAccessIsolation } from "./company-access-flow.js"; import { inboxTriage } from "./inbox-triage-flow.js"; @@ -118,6 +119,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); for (const name of [ + "saved-skill-use-flow.ts", + "saved-skill-use-evidence.ts", "task-lifecycle.ts", "task-lifecycle-evidence.ts", "../../../packages/shared/src/validators/issue.ts", @@ -287,7 +290,9 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.provider = ready.agent.provider; result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; - if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, + if (id === "saved-skill-use") await runSavedSkillUse(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "agent-settings-persistence") await runAgentSettings(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts new file mode 100644 index 00000000000..1bbf1b32a7f --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts @@ -0,0 +1,157 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromCodex } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { createPrpSemanticToolInputEnvelope, createPrpSemanticToolResultEnvelope, digestPrpSemanticContent } from "../../../packages/paperclip-runner/src/protocol/semantic-tool-receipts.js"; +import { validatePrpEvent } from "../../../packages/paperclip-runner/src/protocol/replay-contract.js"; +import { redactEventPayload } from "../../../server/src/redaction.js"; +import { readAssignedSkill, RUNNER_BRIDGE_READ_OPERATIONS } from "../../../server/src/services/native-runtime/runner-bridge-tools.js"; +import { savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts } from "./saved-skill-use-evidence.js"; + +type Row = Record; +const binding = { companyId: "company", agentId: "agent", runId: "run" }; +const fixture = savedSkillFixture("codex-campaign", "62ff548dfec748da8fef6d5fbf6cc23d"); +const key = `company/${binding.companyId}/${fixture.slug}`, runtimeName = `${fixture.slug}--0123456789`; +const correlation = { runId: "run", normalizedSessionId: "native-session", turnId: "turn", itemId: "item" }; +const callId = "assigned-read"; +const hash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); + +function event(seq: number, eventType: string, payload: Row, itemId = correlation.itemId): Row { + return { ...binding, seq, eventType, protocolSchemaVersion: 1, sourceSeq: seq, + sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + payload: redactEventPayload({ prpEvent: { + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", ...correlation, itemId, + priority: 0, emittedAt: "2026-11-03T12:00:00.000Z", + sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + eventType, payload, + } }) }; +} + +// Read the real immutable-file producer, not a hand-written success result. No model or DB is used. +async function observedRead(text = fixture.markdown, rawInput?: Row) { + const directory = await mkdtemp(path.join(tmpdir(), "saved-skill-native-read-")); + try { + const files = [{ path: "SKILL.md", sha256: hash(text), mode: 0o444, size: Buffer.byteLength(text) }]; + const digest = hash(JSON.stringify(files)), rootPath = path.join(directory, "bundles", digest); + const manifest = `${JSON.stringify({ schema: "paperclip.runtime-asset-manifest.v1", digest, + fileCount: 1, totalBytes: Buffer.byteLength(text), files })}\n`; + await mkdir(rootPath, { recursive: true }); + await mkdir(path.join(directory, "manifests")); + await writeFile(path.join(rootPath, "SKILL.md"), text); + await writeFile(path.join(directory, "manifests", `${digest}.json`), manifest); + const bundle = { rootPath, digest, manifestDigest: hash(manifest) }; + const input = rawInput ?? { offset: 0, skill: key, path: "SKILL.md" }; + const result = await readAssignedSkill({ skills: [{ key, runtimeName, versionId: "version", bundle }] } as any, input); + return { input, result }; + } finally { await rm(directory, { recursive: true, force: true }); } +} + +async function observations(text = fixture.markdown, rawInput?: Row) { + const { input, result } = await observedRead(text, rawInput); + const run: Row = { id: binding.runId, companyId: binding.companyId, agentId: binding.agentId, runtimeMode: "native", + runnerInstanceId: "runner-instance", nativeSessionId: correlation.normalizedSessionId, + contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex" }, + runtimeContext: { skills: [{ key, versionId: "version", runtimeName }] } } } }; + const assignment = savedSkillReadAssignment({ key, currentVersionId: "version" }, + { entries: [{ key, versionId: "version", runtimeName }] }, run)!; + // Core091 provider_backend.rs uses these protocol envelopes, adds safe input, + // and names the successful fallback receipt operation_. + const semanticInput = { ...createPrpSemanticToolInputEnvelope({ operationId: "read_assigned_skill", callId, + correlation, content: input }), input }; + const semanticResult = createPrpSemanticToolResultEnvelope({ operationId: "read_assigned_skill", callId, + correlation, content: result, outcome: "succeeded", code: "semantic_tool_succeeded", retryable: false, + authorizationBoundary: "active_task", operationReceiptId: `operation_${callId}` }); + // Genuine converter output has dynamic transport and no invented first-party namespace/readOnly hint. + const dynamic = ["item/started", "item/completed"].map(method => canonicalProviderEventsFromCodex(method, + { item: { id: callId, type: "dynamicToolCall", tool: "read_assigned_skill", status: "completed" } })[0]!); + const events = [event(1, dynamic[0]!.eventType, dynamic[0]!.payload, dynamic[0]!.itemId), + event(2, "semantic_tool.input", { semantic_tool: semanticInput }), + event(3, "semantic_tool.result", { semantic_tool: semanticResult }), + event(4, dynamic[1]!.eventType, dynamic[1]!.payload, dynamic[1]!.itemId)]; + return { events, run, assignment, result, input }; +} + +const read = (value: Awaited>, f = fixture) => + savedSkillReadReceipts(value.events, binding, f, undefined, value.assignment, value.run); +const semantic = (value: Awaited>, index: number) => + value.events[index]!.payload.prpEvent.payload.semantic_tool as Row; + +describe("saved-skill Codex native authority", () => { + it("accepts the complete real read's authoritative input/result commitment after API redaction", async () => { + const value = await observations(); + expect(RUNNER_BRIDGE_READ_OPERATIONS.has("read_assigned_skill")).toBe(true); + expect(Object.keys(value.result).sort()).toEqual(["byteSize", "nextOffset", "path", "sha256", "skill", "text", "versionId"]); + for (const row of value.events) expect(validatePrpEvent(row.payload.prpEvent).ok).toBe(true); + expect(value.events[0]!.payload.prpEvent.payload).toMatchObject({ transport: "dynamic", readOnly: null, namespace: null }); + expect(read(value)).toEqual([{ runId: "run", executionId: callId, seq: 3 }]); + }); + + it("checks UTF-8 source bytes and canonical key order using the real Unicode producer", async () => { + const text = `${fixture.markdown}\nUnicode: café 🌍\n`; + const value = await observations(text); + expect(value.result.byteSize).toBe(Buffer.byteLength(text)); + expect(value.result.byteSize).toBeGreaterThan(text.length); + expect(read(value, { ...fixture, markdown: text })).toHaveLength(1); + }); + + it("accepts Core's omitted defaults and only its exact admitted runtime alias", async () => { + const value = await observations(fixture.markdown, { skill: runtimeName }); + expect(read(value)).toHaveLength(1); + }); + + it("rejects raw dynamic calls even when they copy the complete source and invent readOnly/namespace", async () => { + const value = await observations(); + value.events = [value.events[0]!, value.events[3]!]; + for (const row of value.events) Object.assign(row.payload.prpEvent.payload, + { name: "paperclip_read_assigned_skill", readOnly: true, namespace: "paperclip", + output: JSON.stringify({ type: "inputText", text: JSON.stringify(value.result) }), outputTruncated: false }); + const p = value.events[1]!.payload.prpEvent.payload; p.outputBytes = Buffer.byteLength(p.output); + expect(read(value)).toEqual([]); + }); + + it.each([ + ["missing input", (v: any) => { v.events.splice(1, 1); }], + ["missing result", (v: any) => { v.events.splice(2, 1); }], + ["duplicate input", (v: any) => { v.events.push(structuredClone(v.events[1])); }], + ["duplicate result", (v: any) => { v.events.push(structuredClone(v.events[2])); }], + ["result before input", (v: any) => { v.events[2].seq = 0; }], + ["foreign provider", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.provider.kind = "unknown"; }], + ["foreign runner", (v: any) => { v.run.runnerInstanceId = "foreign"; }], + ["foreign session", (v: any) => { v.run.nativeSessionId = "foreign"; }], + ["missing runner identity", (v: any) => { delete v.run.runnerInstanceId; }], + ["missing session identity", (v: any) => { delete v.run.nativeSessionId; }], + ["foreign run owner", (v: any) => { v.run.companyId = "foreign"; }], + ["missing native assignment", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["duplicate native assignment", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]); }], + ["foreign native version", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "foreign"; }], + ["missing selection snapshot", (v: any) => { v.run.contextSnapshot.skillSelectionSnapshot = false; }], + ["foreign admitted version", (v: any) => { v.run.contextSnapshot.skillVersions[key] = "foreign"; }], + ["foreign call", (v: any) => { semantic(v, 2).callId = "foreign"; }], + ["foreign turn", (v: any) => { semantic(v, 2).correlation.turnId = "foreign"; }], + ["foreign item", (v: any) => { semantic(v, 2).correlation.itemId = "foreign"; }], + ["foreign source", (v: any) => { v.events[2].sourceInstanceId = v.events[2].payload.prpEvent.sourceInstanceId = "foreign"; }], + ["foreign operation", (v: any) => { for (const i of [1, 2]) semantic(v, i).operationId = "paperclip_read_assigned_skill"; }], + ["MCP carriers", (v: any) => { for (const i of [1, 2]) v.events[i].eventType = v.events[i].payload.prpEvent.eventType = i === 1 ? "mcp_app.tool_input" : "mcp_app.tool_result"; }], + ["failed result", (v: any) => { semantic(v, 2).outcome = "failed"; }], + ["wrong boundary", (v: any) => { semantic(v, 2).authorizationBoundary = "company"; }], + ["wrong receipt", (v: any) => { semantic(v, 2).operationReceiptId = "foreign"; }], + ["wrong input digest", (v: any) => { semantic(v, 1).content.digest = `sha256:${"0".repeat(64)}`; }], + ["foreign input key", (v: any) => { semantic(v, 1).input.skill = "foreign"; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["wrong source path", (v: any) => { semantic(v, 1).input.path = "other.md"; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["nonzero offset", (v: any) => { semantic(v, 1).input.offset = 1; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["wrong result digest", (v: any) => { semantic(v, 2).content.digest = `sha256:${"0".repeat(64)}`; }], + ["truncated disclosure", (v: any) => { semantic(v, 2).content.redactionDisposition = "redacted"; }], + ["truncated marker", (v: any) => { semantic(v, 2).outputTruncated = true; }], + ["partial page", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, nextOffset: 1 }); }], + ["missing page completion", (v: any) => { const result = { ...v.result }; delete result.nextOffset; semantic(v, 2).content.digest = digestPrpSemanticContent(result); }], + ["wrong source bytes", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, byteSize: v.result.byteSize + 1 }); }], + ["wrong source checksum", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, sha256: "0".repeat(64) }); }], + ["wrong version", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, versionId: "foreign" }); }], + ["copied nonce only", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, text: fixture.expected.reference }); }], + ] as const)("rejects authoritative-looking evidence with %s", async (_label, mutate) => { + const value = await observations(); mutate(value); expect(read(value)).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts new file mode 100644 index 00000000000..3841766ef97 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts @@ -0,0 +1,311 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { savedSkillDocumentAttributed, savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts, savedSkillReadRuntimeName, savedSkillResultMatches, savedSkillSourceMatches } from "./saved-skill-use-evidence.js"; + +type Row = Record; +const fixture = savedSkillFixture("campaign-id", "62ff548dfec748da8fef6d5fbf6cc23d"); +const binding = { companyId: "company", agentId: "agent", issueId: "issue", runId: "run" }; +const assignment = { key: `company/${binding.companyId}/${fixture.slug}`, versionId: "version" }; +function native(seq: number, eventType: string, payload: Row): Row { + return { ...binding, seq, eventType, protocolSchemaVersion: 1, sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: binding.runId, + sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, turnId: "turn", eventType, payload } } }; +} +function readEvents(): Row[] { + const payload = { schema: "paperclip.tool.execution.v1", executionId: "read-skill", name: "read", operation: "read", + transport: "builtin", readOnly: true, target: `/workspace/.agents/skills/${fixture.slug}/SKILL.md` }; + return [native(1, "tool.execution.started", { ...payload, status: "running" }), + native(2, "tool.execution.completed", { ...payload, status: "completed", output: fixture.markdown, outputBytes: Buffer.byteLength(fixture.markdown), outputTruncated: false })]; +} +const runtimeName = `${fixture.slug}--0123456789`; +function wrappedReadEvents(): Row[] { + const output = `/workspace/.claude/skills/${runtimeName}/SKILL.md\nfile\n\n` + + fixture.markdown.trimEnd().split("\n").map((line, index) => `${index + 1}: ${line}`).join("\n") + + "\n(End of file - total 12 lines)\n"; + const events = readEvents(); + for (const row of events) Object.assign(row.payload.prpEvent.payload, { operation: "unknown", target: null }); + Object.assign(events[1]!.payload.prpEvent.payload, { output, outputBytes: Buffer.byteLength(output) }); + return events; +} +function assignedReadEvents(): Row[] { + const result = { skill: `company/${binding.companyId}/${fixture.slug}`, versionId: "version", path: "SKILL.md", + text: fixture.markdown, sha256: createHash("sha256").update(fixture.markdown).digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const output = JSON.stringify({ type: "inputText", text: JSON.stringify(result) }); + const events = readEvents(); + for (const row of events) Object.assign(row.payload.prpEvent.payload, + { name: "paperclip_read_assigned_skill", operation: "unknown", target: null }); + Object.assign(events[1]!.payload.prpEvent.payload, { output, outputBytes: Buffer.byteLength(output) }); + return events; +} +function changeAssignedResult(events: Row[], change: (value: Row) => void) { + const payload = events[1]!.payload.prpEvent.payload, wrapper = JSON.parse(payload.output), result = JSON.parse(wrapper.text); + change(result); wrapper.text = JSON.stringify(result); payload.output = JSON.stringify(wrapper); +} +function source() { + const skill: Row = { id: "skill", companyId: binding.companyId, key: `company/${binding.companyId}/${fixture.slug}`, sourceType: "managed", name: fixture.name, slug: fixture.slug, + markdown: fixture.markdown, currentVersionId: "version" }; + const version: Row = { id: "version", companyId: binding.companyId, companySkillId: skill.id, + fileInventory: [{ path: "SKILL.md", kind: "skill", content: fixture.markdown }] }; + const assignments: Row = { desiredSkillEntries: [{ key: skill.key, versionId: null }] }; + return { skill, version, assignments }; +} +function output() { + const document: Row = { id: "document", companyId: binding.companyId, issueId: binding.issueId, key: fixture.key, format: "markdown", + latestRevisionId: "revision", latestRevisionNumber: 1, createdByAgentId: binding.agentId, createdByUserId: null, + updatedByAgentId: binding.agentId, updatedByUserId: null, body: JSON.stringify(fixture.expected) }; + const revision: Row = { id: "revision", documentId: document.id, companyId: binding.companyId, issueId: binding.issueId, + key: fixture.key, format: "markdown", revisionNumber: 1, createdByAgentId: binding.agentId, createdByUserId: null, createdByRunId: binding.runId, + body: document.body }; + return { document, revision }; +} + +describe("saved skill use evidence", () => { + it("keeps the random output reference in the skill, independently of its discoverable name", () => { + expect(fixture.markdown).toContain(fixture.expected.reference); + expect(fixture.name + fixture.slug).not.toContain(fixture.expected.reference); + const owned = savedSkillFixture("journey-1234-5678", fixture.expected.reference); + expect(owned.name).toBe(owned.slug); + expect(owned.name).toContain("journey-1234-5678"); + expect(owned.markdown).toContain(`\nname: ${owned.name}\n`); + expect(fixture.markdown).not.toContain("\\n"); + expect(() => savedSkillFixture("", fixture.expected.reference)).toThrow(); + expect(() => savedSkillFixture(fixture.expected.reference, fixture.expected.reference)).toThrow(); + expect(() => savedSkillFixture("campaign", "short")).toThrow(); + }); + it("requires an ordinary Latest assignment, its exact managed current version and original source bytes", () => { + const good = source(); + expect(savedSkillSourceMatches(good.skill, good.version, good.assignments, fixture, binding.companyId)).toBe(true); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.skill.companyId = "foreign"; }, v => { v.version.companyId = "foreign"; }, + v => { v.version.companySkillId = "foreign"; }, v => { v.skill.slug = "foreign"; }, + v => { v.skill.name = "different saved name"; }, + v => { v.skill.sourceType = "local_path"; }, + v => { v.skill.key = "unknown"; v.assignments.desiredSkillEntries[0].key = "unknown"; }, + v => { v.skill.currentVersionId = "different"; }, + v => { v.skill.markdown += "changed"; }, v => { v.version.id = "stale"; }, + v => { v.version.fileInventory = []; }, v => { v.version.fileInventory[0].content += "changed"; }, + v => { v.version.fileInventory[0].encoding = "base64"; }, v => { v.assignments.desiredSkillEntries = []; }, + v => { v.assignments.desiredSkillEntries[0].versionId = "version"; }, + v => { v.assignments.desiredSkillEntries[0].versionId = "stale"; }, + v => { delete v.assignments.desiredSkillEntries[0].versionId; }, + v => { v.assignments.desiredSkillEntries[0].key = "foreign"; }, + v => { v.assignments.desiredSkillEntries.push(v.assignments.desiredSkillEntries[0]); }, + ]; + for (const change of changes) { + const wrong = source(); change(wrong); + expect(savedSkillSourceMatches(wrong.skill, wrong.version, wrong.assignments, fixture, binding.companyId)).toBe(false); + } + expect(savedSkillSourceMatches(undefined, good.version, good.assignments, fixture, binding.companyId)).toBe(false); + }); + it("accepts the bound successful source read and rejects missing, foreign, claimed and failed reads", () => { + expect(savedSkillReadReceipts(readEvents(), binding, fixture)).toEqual([{ runId: "run", executionId: "read-skill", seq: 2 }]); + const changes: Array<(events: Row[]) => void> = [ + e => { e.splice(0, 1); }, e => { e.pop(); }, e => { e[1].runId = "foreign"; }, + e => { e[1].companyId = "foreign"; }, e => { e[1].agentId = "foreign"; }, + e => { e[1].payload.prpEvent.runId = "foreign"; }, e => { e[1].payload.prpEvent.sourceKind = "provider"; }, + e => { e[1].sourceInstanceId = "foreign"; }, e => { e[1].sourceSeq += 1; }, + e => { e[1].payload.prpEvent.turnId = "foreign"; }, e => { e[1].payload.prpEvent.payload.status = "failed"; }, + e => { for (const row of e) delete row.payload.prpEvent.turnId; }, + e => { e[1].payload.prpEvent.payload.outputBytes = 0; }, e => { e[1].payload.prpEvent.payload.outputTruncated = true; }, + e => { delete e[1].payload.prpEvent.payload.outputTruncated; }, + e => { e[1].payload.prpEvent.payload.output = "I read the skill and followed it."; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = `/workspace/other/${fixture.slug}/../SKILL.md`; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = "/workspace/other-skill/SKILL.md"; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = null; }, + e => { for (const row of e) Object.assign(row.payload.prpEvent.payload, { name: "bash", operation: "execute", transport: "process" }); }, + e => { e[1].payload.prpEvent.payload.executionId = "other-read"; }, + e => { e[1].payload.prpEvent.eventType = e[1].eventType = "item.completed"; }, + ]; + for (const change of changes) { + const wrong = readEvents(); change(wrong); + expect(savedSkillReadReceipts(wrong, binding, fixture)).toEqual([]); + } + expect(savedSkillReadReceipts([], binding, fixture)).toEqual([]); + }); + it("accepts the deployed OpenCode read wrapper for its exact version-bound runtime skill directory", () => { + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, runtimeName)).toHaveLength(1); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture)).toEqual([]); + }); + it("accepts the complete first-party assigned-skill receipt with exact immutable source identity and bytes", () => { + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, assignment)) + .toEqual([{ runId: "run", executionId: "read-skill", seq: 2 }]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, assignment)).toHaveLength(1); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName)).toEqual([]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, { ...assignment, key: "foreign" })).toEqual([]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, { ...assignment, versionId: "foreign" })).toEqual([]); + }); + it.each([{ kind: "codex" }, { kind: "acpx", agent: "cursor" }, { kind: "cursor_cloud" }])( + "binds the semantic source to common native skill entries for %j while preserving the OpenCode file guard", provider => { + const { skill } = source(); + const good = () => ({ assignments: { entries: [{ key: skill.key, versionId: "version", runtimeName }] }, + run: { contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider, + runtimeContext: { skills: [{ key: skill.key, versionId: "version", runtimeName }] } } } } }); + const valid = good(), selected = savedSkillReadAssignment(skill, valid.assignments, valid.run); + expect(selected).toEqual(assignment); + expect(savedSkillReadRuntimeName(skill, valid.assignments, valid.run)).toBeUndefined(); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, selected)).toHaveLength(1); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, undefined, selected)).toEqual([]); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.assignments.entries = []; }, + v => { v.assignments.entries[0]!.key = "foreign"; }, + v => { v.assignments.entries[0]!.versionId = "foreign"; }, + v => { v.assignments.entries.push(v.assignments.entries[0]!); }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.key = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.versionId = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!); }, + v => { v.run.contextSnapshot.skillSelectionSnapshot = false; }, + v => { v.run.contextSnapshot.skillVersions = {}; }, + v => { v.run.contextSnapshot.skillVersions[skill.key] = "foreign"; }, + ]; + for (const mutate of changes) { + const wrong = good(); mutate(wrong); + const selected = savedSkillReadAssignment(skill, wrong.assignments, wrong.run); + expect(selected).toBeUndefined(); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, selected)).toEqual([]); + } + }); + it.each([ + ["foreign canonical key", (e: Row[]) => changeAssignedResult(e, r => { r.skill = "company/foreign/skill"; })], + ["foreign immutable version", (e: Row[]) => changeAssignedResult(e, r => { r.versionId = "foreign"; })], + ["wrong file", (e: Row[]) => changeAssignedResult(e, r => { r.path = "other/SKILL.md"; })], + ["incomplete page", (e: Row[]) => changeAssignedResult(e, r => { r.nextOffset = r.text.length; })], + ["missing pagination receipt", (e: Row[]) => changeAssignedResult(e, r => { delete r.nextOffset; })], + ["wrong checksum", (e: Row[]) => changeAssignedResult(e, r => { r.sha256 = "0".repeat(64); })], + ["wrong source byte count", (e: Row[]) => changeAssignedResult(e, r => { r.byteSize++; })], + ["partial procedure", (e: Row[]) => changeAssignedResult(e, r => { r.text = fixture.expected.reference; })], + ["changed full source", (e: Row[]) => changeAssignedResult(e, r => { r.text += "extra source"; })], + ["semantic failure with copied source", (e: Row[]) => changeAssignedResult(e, r => { r.outcome = "failed"; r.code = "runner_skill_not_assigned"; })], + ["shell copied receipt", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "bash"; }], + ["another first-party tool", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "paperclip_read_document"; }], + ["non-builtin transport", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.transport = "process"; }], + ["mutating operation", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.operation = "edit"; }], + ["not read only", (e: Row[]) => { e[1]!.payload.prpEvent.payload.readOnly = false; }], + ["start not read only", (e: Row[]) => { e[0]!.payload.prpEvent.payload.readOnly = false; }], + ["failed completion", (e: Row[]) => { e[1]!.payload.prpEvent.payload.status = "failed"; }], + ["truncated completion", (e: Row[]) => { e[1]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing start", (e: Row[]) => { e.splice(0, 1); }], + ["call mismatch", (e: Row[]) => { e[0]!.payload.prpEvent.payload.executionId = "another-call"; }], + ["foreign runner", (e: Row[]) => { e[0]!.sourceInstanceId = e[0]!.payload.prpEvent.sourceInstanceId = "another-runner"; }], + ["foreign run", (e: Row[]) => { e[1]!.runId = e[1]!.payload.prpEvent.runId = "another-run"; }], + ["foreign company", (e: Row[]) => { e[1]!.companyId = "another-company"; }], + ["foreign turn", (e: Row[]) => { e[1]!.payload.prpEvent.turnId = "another-turn"; }], + ["invalid wrapper", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = fixture.markdown; }], + ["wrong wrapper type", (e: Row[]) => { const p = e[1]!.payload.prpEvent.payload; p.output = p.output.replace("inputText", "outputText"); }], + ] as const)("rejects a first-party assigned-skill receipt with %s", (_label, mutate) => { + const events = assignedReadEvents(); mutate(events); + const completed = events.find(row => row.eventType === "tool.execution.completed")!; + completed.payload.prpEvent.payload.outputBytes = Buffer.byteLength(completed.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture, runtimeName, assignment)).toEqual([]); + }); + it.each([ + ["foreign path", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace(runtimeName, "foreign"); }], + ["relative path", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("/workspace/", "workspace/"); }], + ["path traversal", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("/workspace/", "/workspace/../"); }], + ["wrong file type", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("file", "directory"); }], + ["missing wrapper", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = fixture.procedure; }], + ["partial procedure", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("Do not create other tasks or deliverables.", ""); }], + ["copied shell output", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "bash"; }], + ["execute operation", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.operation = "execute"; }], + ["not read only", (e: Row[]) => { e[1]!.payload.prpEvent.payload.readOnly = false; }], + ["start not read only", (e: Row[]) => { e[0]!.payload.prpEvent.payload.readOnly = false; }], + ["failed read", (e: Row[]) => { e[1]!.payload.prpEvent.payload.status = "failed"; }], + ["truncated read", (e: Row[]) => { e[1]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing start", (e: Row[]) => { e.shift(); }], + ["call mismatch", (e: Row[]) => { e[0]!.payload.prpEvent.payload.executionId = "another-call"; }], + ["foreign runner", (e: Row[]) => { e[0]!.sourceInstanceId = e[0]!.payload.prpEvent.sourceInstanceId = "another-runner"; }], + ["foreign run", (e: Row[]) => { e[1]!.runId = e[1]!.payload.prpEvent.runId = "another-run"; }], + ["foreign turn", (e: Row[]) => { e[1]!.payload.prpEvent.turnId = "another-turn"; }], + ] as const)("rejects a wrapped native read with %s", (_label, mutate) => { + const events = wrappedReadEvents(); mutate(events); + const completed = events.find(row => row.eventType === "tool.execution.completed")!; + completed.payload.prpEvent.payload.outputBytes = Buffer.byteLength(completed.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture, runtimeName)).toEqual([]); + }); + it("requires both API and admitted native input to bind the same immutable skill runtime name", () => { + const { skill } = source(); + const good = () => ({ assignments: { entries: [{ key: skill.key, runtimeName, versionId: "version" }] }, + run: { contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode" }, + runtimeContext: { skills: [{ key: skill.key, runtimeName, versionId: "version" }] } } } } }); + const valid = good(); + expect(savedSkillReadRuntimeName(skill, valid.assignments, valid.run)).toBe(runtimeName); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.assignments.entries = []; }, + v => { v.assignments.entries[0]!.key = "foreign"; }, + v => { v.assignments.entries[0]!.versionId = "stale"; }, + v => { v.assignments.entries[0]!.runtimeName = "foreign"; }, + v => { v.assignments.entries.push(v.assignments.entries[0]!); }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.key = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.versionId = "stale"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.runtimeName = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!); }, + v => { v.run.contextSnapshot.skillSelectionSnapshot = false; }, + v => { v.run.contextSnapshot.skillVersions[skill.key] = "stale"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }, + v => { v.assignments.entries[0]!.runtimeName = v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.runtimeName = "../foreign"; }, + ]; + for (const mutate of changes) { + const wrong = good(); mutate(wrong); + const name = savedSkillReadRuntimeName(skill, wrong.assignments, wrong.run); + expect(name).toBeUndefined(); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, name)).toEqual([]); + } + }); + it("accepts the real OpenCode native skill-loader mapping, including its completed provider output", () => { + const part = (status: string) => ({ id: "load-skill", type: "tool", tool: "skill", state: { + status, output: `\n${fixture.procedure}\n`, + } }); + const events = ["pending", "completed"].map((status, index) => { + const event = canonicalProviderEventsFromOpenCodePart(part(status))[0]!; + return native(index + 1, event.eventType, event.payload); + }); + expect(savedSkillReadReceipts(events, binding, fixture)).toHaveLength(1); + for (const output of [fixture.procedure, `\n${fixture.procedure}\n`]) { + const wrong = structuredClone(events); wrong[1]!.payload.prpEvent.payload.output = output; + wrong[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(output); + expect(savedSkillReadReceipts(wrong, binding, fixture)).toEqual([]); + } + }); + it("allows known file-reader line prefixes without treating partial output as a complete read", () => { + const events = readEvents(); + events[1]!.payload.prpEvent.payload.output = fixture.markdown.split("\n").map((line, i) => `${i + 1}: ${line}`).join("\n"); + events[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(events[1]!.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture)).toHaveLength(1); + events[1]!.payload.prpEvent.payload.output = fixture.expected.reference; + events[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(fixture.expected.reference); + expect(savedSkillReadReceipts(events, binding, fixture)).toEqual([]); + }); + it("requires the exact derived result and a saved revision authored by this run", () => { + const good = output(); + expect(savedSkillResultMatches(`\u0060\u0060\u0060json\n${good.document.body}\n\u0060\u0060\u0060`, fixture.expected)).toBe(true); + expect(savedSkillDocumentAttributed(good.document, good.revision, fixture, binding)).toBe(true); + for (const body of ["I finished", "{}", JSON.stringify({ ...fixture.expected, extra: true }), + JSON.stringify({ ...fixture.expected, total: fixture.expected.total + 1 }), JSON.stringify({ ...fixture.expected, reference: "foreign" })]) { + expect(savedSkillResultMatches(body, fixture.expected)).toBe(false); + } + const changes: Array<(value: ReturnType) => void> = [ + v => { v.document.companyId = "foreign"; }, v => { v.document.issueId = "foreign"; }, + v => { v.document.createdByAgentId = "foreign"; }, v => { v.document.updatedByAgentId = "foreign"; }, + v => { v.document.createdByUserId = "owner"; }, v => { v.document.updatedByUserId = "owner"; }, + v => { v.document.key = "other"; }, v => { v.document.body = "{}"; }, + v => { v.document.latestRevisionId = null; }, v => { v.revision.id = "old"; }, + v => { delete v.document.latestRevisionNumber; delete v.revision.revisionNumber; }, + v => { v.revision.documentId = "foreign"; }, v => { v.revision.companyId = "foreign"; }, + v => { v.revision.issueId = "foreign"; }, v => { v.revision.key = "other"; }, + v => { v.revision.format = "unknown"; }, + v => { v.revision.createdByRunId = "foreign"; }, v => { v.revision.createdByAgentId = "foreign"; }, + v => { v.revision.createdByUserId = "owner"; }, v => { v.revision.revisionNumber = 2; }, + v => { v.revision.body = "{}"; }, + ]; + for (const change of changes) { + const wrong = output(); change(wrong); + expect(savedSkillDocumentAttributed(wrong.document, wrong.revision, fixture, binding)).toBe(false); + } + expect(savedSkillDocumentAttributed(undefined, good.revision, fixture, binding)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts new file mode 100644 index 00000000000..0f4ede10092 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts @@ -0,0 +1,238 @@ +import { createHash } from "node:crypto"; +import { isBoundNativePrpEventRow, type NativePrpRunBinding } from "../native-event-envelope.js"; + +type Row = Record; +export interface SavedSkillFixture { + name: string; slug: string; key: "saved-skill-result"; markdown: string; procedure: string; + expected: { reference: string; total: number }; +} + +/** The public request names the skill; only its saved version contains these inputs. */ +export function savedSkillFixture(identity: string, reference: string): SavedSkillFixture { + if (!/^[a-f0-9]{32}$/.test(reference)) throw new Error("invalid_saved_skill_reference"); + if (!/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/.test(identity) || identity.length > 48 || identity.includes(reference)) + throw new Error("invalid_saved_skill_identity"); + const slug = `qa-saved-skill-${identity}`, key = "saved-skill-result"; + const seed = createHash("sha256").update(reference).digest(); + const first = seed[0]! + 1, second = seed[1]! + 1; + const procedure = `# Saved receipt procedure\n\nThe receipt reference is ${reference}.\n` + + `Add the two amounts ${first} and ${second}.\n` + + `Save exactly one Paperclip task document with key ${key}. Its body must be JSON with exactly ` + + `two fields: reference (the receipt reference) and total (the sum of the two amounts).\n` + + "Check the saved document and finish the task. Do not create other tasks or deliverables.\n"; + return { name: slug, slug, key, procedure, + markdown: `---\nname: ${slug}\ndescription: Make a receipt using this saved procedure.\n---\n\n${procedure}`, + expected: { reference, total: first + second } }; +} + +export function savedSkillSourceMatches(skill: Row | undefined, version: Row | undefined, assignments: Row | undefined, + fixture: SavedSkillFixture, companyId: string): boolean { + if (!skill || !version || !assignments) return false; + const files = version.fileInventory; + const entries = assignments.desiredSkillEntries; + return Boolean(skill.id && skill.companyId === companyId && skill.sourceType === "managed" + && skill.name === fixture.name && skill.slug === fixture.slug && skill.key === `company/${companyId}/${fixture.slug}` + && skill.markdown === fixture.markdown && skill.currentVersionId + && version.id === skill.currentVersionId && version.companyId === companyId && version.companySkillId === skill.id + && Array.isArray(files) && files.length === 1 && files[0].path === "SKILL.md" && files[0].content === fixture.markdown + && (files[0].encoding === undefined || files[0].encoding === "utf8") + && Array.isArray(entries) && entries.filter(row => row.key === skill.key).length === 1 + && entries.find(row => row.key === skill.key)?.versionId === null); +} + +export function savedSkillResultMatches(body: unknown, expected: SavedSkillFixture["expected"]): boolean { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const value = JSON.parse(json); + return Boolean(value && typeof value === "object" && !Array.isArray(value) + && Object.keys(value).sort().join(",") === "reference,total" + && value.reference === expected.reference && value.total === expected.total); + } catch { return false; } +} + +export interface SavedSkillReadAssignment { key: string; versionId: string } + +/** All native providers share the canonical skill/version selection contract. */ +export function savedSkillReadAssignment(skill: Row, assignments: Row, run: Row): SavedSkillReadAssignment | undefined { + const input = run.runnerProfileJson?.nativeExecutionInput; + const assigned = Array.isArray(assignments.entries) ? assignments.entries.filter((row: Row) => row.key === skill.key) : []; + const native = Array.isArray(input?.runtimeContext?.skills) + ? input.runtimeContext.skills.filter((row: Row) => row.key === skill.key) : []; + const versions = run.contextSnapshot?.skillVersions; + if (typeof skill.key !== "string" || !skill.key.trim() || assigned.length !== 1 || native.length !== 1 + || typeof skill.currentVersionId !== "string" || !skill.currentVersionId + || run.contextSnapshot?.skillSelectionSnapshot !== true + || !versions || typeof versions !== "object" || Array.isArray(versions) || !Object.hasOwn(versions, skill.key) + || versions[skill.key] !== skill.currentVersionId + || assigned[0].versionId !== skill.currentVersionId || native[0].versionId !== skill.currentVersionId) return undefined; + return { key: skill.key, versionId: skill.currentVersionId }; +} + +/** OpenCode's file wrapper also needs the exact materialized directory identity. */ +export function savedSkillReadRuntimeName(skill: Row, assignments: Row, run: Row): string | undefined { + if (run.runnerProfileJson?.nativeExecutionInput?.provider?.kind !== "opencode" + || !savedSkillReadAssignment(skill, assignments, run)) return undefined; + const assigned = assignments.entries.find((row: Row) => row.key === skill.key); + const native = run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.find((row: Row) => row.key === skill.key); + if (assigned.runtimeName !== native.runtimeName + || typeof native.runtimeName !== "string" || !/^[a-z0-9][a-z0-9_-]*$/i.test(native.runtimeName)) return undefined; + return native.runtimeName; +} + +/** Core's assigned-skill read verifies the pinned manifest and returns one UTF-8 page. */ +function assignedSkillReadText(output: string, binding: NativePrpRunBinding, fixture: SavedSkillFixture, versionId: string): string | undefined { + try { + const wrapper = JSON.parse(output); + if (wrapper?.type !== "inputText" || typeof wrapper.text !== "string" + || Object.keys(wrapper).sort().join(",") !== "text,type") return undefined; + const result = JSON.parse(wrapper.text); + if (Object.keys(result).sort().join(",") !== "byteSize,nextOffset,path,sha256,skill,text,versionId" + || result.skill !== `company/${binding.companyId}/${fixture.slug}` || result.versionId !== versionId + || result.path !== "SKILL.md" || result.nextOffset !== null || result.text !== fixture.markdown + || result.byteSize !== Buffer.byteLength(fixture.markdown) + || result.sha256 !== createHash("sha256").update(fixture.markdown).digest("hex")) return undefined; + return result.text; + } catch { return undefined; } +} + +// Core091 provider_bridge.rs hashes UTF-8 JSON with object keys ordered by UTF-16. +// This oracle reconstructs the complete pinned read result, not provider display text. +function semanticDigest(value: unknown): string { + const canonical = (value: unknown): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => + `${JSON.stringify(key)}:${canonical((value as Row)[key])}`).join(",")}}` : JSON.stringify(value) ?? "null"; + return `sha256:${createHash("sha256").update(canonical(value), "utf8").digest("hex")}`; +} + +/** Codex's raw dynamic name, namespace and optional readOnly hint grant no authority. + * Its admitted first-party read instead has a controller-bound semantic input/result. + * Core's read_assigned_skill operation is read-only and verifies the immutable file. + */ +function codexAssignedSkillReceipts(bound: Row[], binding: NativePrpRunBinding, fixture: SavedSkillFixture, + assignment: SavedSkillReadAssignment | undefined, run: Row | undefined) { + const input = run?.runnerProfileJson?.nativeExecutionInput; + const skills = Array.isArray(input?.runtimeContext?.skills) + ? input.runtimeContext.skills.filter((skill: Row) => skill.key === assignment?.key) : []; + if (!run || run.runtimeMode !== "native" || input?.provider?.kind !== "codex" + || run.id !== binding.runId || run.companyId !== binding.companyId || run.agentId !== binding.agentId + || typeof run.runnerInstanceId !== "string" || !run.runnerInstanceId.trim() + || typeof run.nativeSessionId !== "string" || !run.nativeSessionId.trim() + || !assignment || assignment.key !== `company/${binding.companyId}/${fixture.slug}` || !assignment.versionId + || skills.length !== 1 || skills[0].versionId !== assignment.versionId + || run.contextSnapshot?.skillSelectionSnapshot !== true + || run.contextSnapshot?.skillVersions?.[assignment.key] !== assignment.versionId + || !fixture.markdown.includes(fixture.procedure.trim()) || !fixture.markdown.includes(fixture.expected.reference)) return []; + const expected = { skill: assignment.key, versionId: assignment.versionId, path: "SKILL.md", text: fixture.markdown, + sha256: createHash("sha256").update(fixture.markdown, "utf8").digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const semantic = (row: Row): Row | undefined => row.payload.prpEvent.payload?.semantic_tool; + const content = (value: Row, digest: string) => value.content?.digest === digest + && value.content.redactionDisposition === "digest_only" && Array.isArray(value.content.references) + && value.content.references.length === 0 + && Object.keys(value.content).sort().join(",") === "digest,redactionDisposition,references"; + const correlated = (row: Row, phase: "input" | "result") => { + const event = row.payload.prpEvent, value = semantic(row), correlation = value?.correlation; + const keys = ["schema", "schemaVersion", "phase", "operationId", "callId", "correlation", "idempotencyKey", "content", + ...(phase === "input" ? ["input"] : ["outcome", "code", "retryable", "authorizationBoundary", "operationReceiptId"])]; + return Boolean(value && event.eventType === `semantic_tool.${phase}` && value.phase === phase + && value.schema === "paperclip.prp.semantic_tool.v1" && value.schemaVersion === 1 + && Object.keys(value).sort().join(",") === keys.sort().join(",") + && value.operationId === "read_assigned_skill" && typeof value.callId === "string" && value.callId.trim() + && value.idempotencyKey === null && Number.isSafeInteger(row.seq) + && event.sourceInstanceId === run.runnerInstanceId && event.normalizedSessionId === run.nativeSessionId + && typeof event.turnId === "string" && event.turnId.trim() && typeof event.itemId === "string" && event.itemId.trim() + && correlation && Object.keys(correlation).sort().join(",") === "itemId,normalizedSessionId,runId,turnId" + && correlation.runId === binding.runId && correlation.normalizedSessionId === run.nativeSessionId + && correlation.turnId === event.turnId && correlation.itemId === event.itemId); + }; + return bound.flatMap(start => { + if (!correlated(start, "input")) return []; + const initial = semantic(start)!, args = initial.input; + if (!args || typeof args !== "object" || Array.isArray(args) + || Object.keys(args).some(key => !["skill", "path", "offset"].includes(key)) + || !(args.skill === assignment.key || typeof skills[0].runtimeName === "string" + && skills[0].runtimeName.trim() && args.skill === skills[0].runtimeName) + || Object.hasOwn(args, "path") && args.path !== "SKILL.md" + || Object.hasOwn(args, "offset") && args.offset !== 0 || !content(initial, semanticDigest(args))) return []; + // Duplicates, reconciled alternatives and foreign same-call carriers cannot supply a second interpretation. + const pair = bound.filter(row => semantic(row)?.callId === initial.callId); + const end = pair.find(row => correlated(row, "result")); + if (pair.length !== 2 || !end || start.seq >= end.seq + || start.payload.prpEvent.sourceSeq >= end.payload.prpEvent.sourceSeq) return []; + const result = semantic(end)!; + if (semanticDigest(initial.correlation) !== semanticDigest(result.correlation) + || result.outcome !== "succeeded" || result.code !== "semantic_tool_succeeded" || result.retryable !== false + || result.authorizationBoundary !== "active_task" || result.operationReceiptId !== `operation_${initial.callId}` + || !content(result, semanticDigest(expected))) return []; + return [{ runId: binding.runId, executionId: String(initial.callId), seq: Number(end.seq) }]; + }); +} + +/** Successful built-in reads and native skill loads expose the actual skill text. + * A model claim, discovered name, shell echo, failed read or foreign run cannot qualify. */ +export function savedSkillReadReceipts(events: Row[], binding: NativePrpRunBinding, fixture: SavedSkillFixture, + runtimeName?: string, assignment?: SavedSkillReadAssignment, run?: Row) { + const bound = events.filter(row => isBoundNativePrpEventRow(row, binding)); + const builtin = bound.filter(row => { + const event = row.payload.prpEvent, payload = event.payload; + if (event.eventType !== "tool.execution.completed" || payload?.schema !== "paperclip.tool.execution.v1" + || payload.transport !== "builtin" || payload.status !== "completed" + || typeof event.turnId !== "string" || !event.turnId.trim() + || typeof payload.executionId !== "string" || !payload.executionId.trim() + || payload.outputTruncated !== false || typeof payload.output !== "string" || payload.outputBytes <= 0 + || payload.outputBytes !== Buffer.byteLength(payload.output)) return false; + const name = String(payload.name ?? "").toLowerCase(); + const target = typeof payload.target === "string" ? payload.target.replaceAll("\\", "/") : ""; + const reading = payload.operation === "read" && payload.readOnly === true && /^(read|read_file|file_read)$/.test(name) + && !target.split("/").includes("..") && target.endsWith(`/${fixture.slug}/SKILL.md`); + // OpenCode's skill tool names the loaded skill in its provider-visible result. + const loading = payload.operation === "execute" && name === "skill" + && payload.output.includes(``); + // Deployed OpenCode reports this read with operation unknown and target null. + // Its successful file-result wrapper supplies the exact materialized source path. + const wrapped = /^([^<>\r\n]+)<\/path>\r?\nfile<\/type>\r?\n\r?\n([\s\S]*)\r?\n<\/content>\s*$/.exec(payload.output); + const wrappedPath = wrapped?.[1]; + const wrappedRead = name === "read" && payload.readOnly === true && payload.target === null + && (payload.operation === "unknown" || payload.operation === "read") + && typeof runtimeName === "string" && /^[a-z0-9][a-z0-9_-]*$/i.test(runtimeName) + && wrappedPath?.startsWith("/") && !wrappedPath.includes("\\") && !wrappedPath.includes("\0") + && !wrappedPath.split("/").some(part => part === "." || part === "..") + && wrappedPath.endsWith(`/.claude/skills/${runtimeName}/SKILL.md`); + // The ordinary semantic tool is another native read path, not a filesystem wrapper. + // Its canonical key/version binding does not depend on a provider's directory name. + const assignedText = name === "paperclip_read_assigned_skill" && payload.readOnly === true && payload.target === null + && (payload.operation === "unknown" || payload.operation === "read") + && assignment?.key === `company/${binding.companyId}/${fixture.slug}` + && typeof assignment.versionId === "string" && assignment.versionId.trim() + ? assignedSkillReadText(payload.output, binding, fixture, assignment.versionId) : undefined; + if (!reading && !loading && !wrappedRead && assignedText === undefined) return false; + // File readers may prefix each source line with its line number. + const output = (assignedText ?? (wrappedRead ? wrapped![2]! : payload.output)).split("\n").map((line: string) => line.replace(/^\s*\d+(?:[:|]|→)\s?/, "")).join("\n"); + if (!output.includes(fixture.procedure.trim())) return false; + return bound.some(start => { + const initial = start.payload.prpEvent, p = initial.payload; + return initial.eventType === "tool.execution.started" && start.seq < row.seq + && initial.sourceInstanceId === event.sourceInstanceId && initial.turnId === event.turnId + && p?.schema === payload.schema && p.executionId === payload.executionId && p.name === payload.name + && p.transport === payload.transport && p.operation === payload.operation && p.status === "running" + && (!(wrappedRead || assignedText !== undefined) || p.readOnly === true); + }); + }).map(row => ({ runId: binding.runId, executionId: String(row.payload.prpEvent.payload.executionId), seq: Number(row.seq) })); + return [...builtin, ...codexAssignedSkillReceipts(bound, binding, fixture, assignment, run)]; +} + +export function savedSkillDocumentAttributed(document: Row | undefined, revision: Row | undefined, fixture: SavedSkillFixture, + binding: NativePrpRunBinding & { issueId: string }): boolean { + if (!document || !revision) return false; + return Boolean(document.id && document.companyId === binding.companyId && document.issueId === binding.issueId + && document.key === fixture.key && document.format === "markdown" && document.latestRevisionId + && Number.isSafeInteger(document.latestRevisionNumber) && document.latestRevisionNumber > 0 + && document.createdByAgentId === binding.agentId && document.createdByUserId === null + && document.updatedByAgentId === binding.agentId && document.updatedByUserId === null + && savedSkillResultMatches(document.body, fixture.expected) + && revision.id === document.latestRevisionId && revision.documentId === document.id + && revision.companyId === binding.companyId && revision.issueId === binding.issueId && revision.key === fixture.key && revision.format === document.format + && revision.createdByRunId === binding.runId && revision.createdByAgentId === binding.agentId && revision.createdByUserId === null + && revision.revisionNumber === document.latestRevisionNumber && savedSkillResultMatches(revision.body, fixture.expected)); +} diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts new file mode 100644 index 00000000000..266d07514f5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts @@ -0,0 +1,370 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { createPrpSemanticToolInputEnvelope, createPrpSemanticToolResultEnvelope } from "../../../packages/paperclip-runner/src/protocol/semantic-tool-receipts.js"; +import { parseFrontmatterMarkdown } from "../../../packages/shared/src/frontmatter.js"; +import { agentSkillSyncSchema } from "../../../packages/shared/src/validators/adapter-skills.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { runSavedSkillUse } from "./saved-skill-use-flow.js"; +import { savedSkillFixture, type SavedSkillFixture } from "./saved-skill-use-evidence.js"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; + +type Row = Record; +vi.mock("../user-actions.js", async original => ({ ...await original(), createTaskThroughUi: vi.fn() })); +vi.mock("./task-document-flow.js", async original => ({ ...await original(), openDocumentAuthoring: vi.fn() })); +vi.mock("@playwright/test", () => ({ expect: (locator: { textContent(): Promise }) => ({ + async toContainText(text: string) { expect(await locator.textContent()).toContain(text); }, +}) })); +const directories: string[] = []; +afterEach(async () => { vi.clearAllMocks(); await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); + +async function setup(change: (kind: string, row: any) => any = (_kind, row) => row, campaignId = "campaign", betaSkillsEnabled = false, + wrappedRead: boolean | "assigned" | "codex" = false) { + const directory = await mkdtemp(path.join(os.tmpdir(), "saved-skill-flow-")); directories.push(directory); + let fixture: SavedSkillFixture; + let skill: Row, assigned = false, issueCreated = false, reloaded = false; + const savedSkills: Row[] = []; + const own = vi.fn(async (_resource: Row) => {}); + const evidence = { checks: [] } as unknown as JourneyEvidence; + const api = { + origin: "https://qa.example.test", + post: vi.fn(async (route: string, body: Row) => { + if (route === "/api/companies/company/skills") { + const reference = /receipt reference is ([a-f0-9]{32})/.exec(body.markdown)![1]!; + fixture = savedSkillFixture(campaignId, reference); + // createLocalSkill gives the parsed frontmatter name precedence over the API input name. + const savedName = parseFrontmatterMarkdown(body.markdown).frontmatter.name; + skill = { id: "skill", companyId: "company", key: `company/company/${fixture.slug}`, sourceType: "managed", slug: fixture.slug, + name: typeof savedName === "string" && savedName.trim() ? savedName.trim() : body.name, + markdown: body.markdown, currentVersionId: "version" }; + savedSkills.push(structuredClone(skill)); + return change("create-skill", structuredClone(skill)); + } + if (route === "/api/agents/agent/skills/sync?companyId=company") { + const parsed = agentSkillSyncSchema.parse(body); + // Core normalizes a string selection to Latest; only a non-null pin requires Beta skills. + const selections = parsed.desiredSkills.map(value => typeof value === "string" ? { key: value, versionId: null } : value); + if (!betaSkillsEnabled && selections.some(entry => entry.versionId !== null)) throw new Error("Beta skill version pins require Beta skills"); + expect(parsed).toEqual({ desiredSkills: [skill.key], mode: "add" }); + assigned = true; return {}; + } + throw new Error(`Unexpected mutation ${route}`); + }), + get: vi.fn(async (route: string) => { + if (route === "/api/instance/settings/experimental") return { enableBetaSkills: betaSkillsEnabled }; + if (route === "/api/companies/company/skills/skill") return change(reloaded ? "retained-skill" : "skill", structuredClone(skill)); + if (route === "/api/companies/company/skills/skill/versions/version") return change(reloaded ? "retained-version" : "version", { + id: "version", companyId: "company", companySkillId: "skill", fileInventory: [{ path: "SKILL.md", content: fixture.markdown }], + }); + if (route === "/api/agents/agent/skills?companyId=company") return change("assignment", { + desiredSkillEntries: assigned ? [{ key: skill.key, versionId: null }] : [], + entries: assigned ? [{ key: skill.key, versionId: "version", runtimeName: `${fixture.slug}--0123456789` }] : [], + }); + if (!issueCreated) throw new Error("Measured result was read before browser dispatch"); + if (route === "/api/issues/issue") return { id: "issue", identifier: "QA-1", companyId: "company", assigneeAgentId: "agent", status: "done", + description: change("description", vi.mocked(createTaskThroughUi).mock.calls.at(-1)![0].prompt) }; + if (route === "/api/issues/issue/runs") return change("runs", [{ runId: "run" }]); + if (route === "/api/issues/issue/comments?order=asc" || route === "/api/issues/issue/interactions") return []; + if (route === "/api/heartbeat-runs/run") return change("run", { id: "run", companyId: "company", agentId: "agent", issueId: "issue", + status: "succeeded", runtimeMode: "native", nativePhase: "committed", resultJson: null, + runnerInstanceId: "instance", nativeSessionId: "session", + contextSnapshot: { paperclipEnvironment: { id: "environment" }, skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode", model: "model" }, + runtimeContext: { skills: [{ key: skill.key, versionId: "version", runtimeName: `${fixture.slug}--0123456789` }] } } } }); + if (route.startsWith("/api/heartbeat-runs/run/events?")) { + if (wrappedRead === "codex") { + const correlation = { runId: "run", normalizedSessionId: "session", turnId: "turn", itemId: "item" }; + const input = { skill: skill.key }; + const result = change("assigned-read", { skill: skill.key, versionId: skill.currentVersionId, + path: "SKILL.md", text: fixture.markdown, sha256: createHash("sha256").update(fixture.markdown).digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }); + const semantic = [{ ...createPrpSemanticToolInputEnvelope({ operationId: "read_assigned_skill", callId: "load", + correlation, content: input }), input }, createPrpSemanticToolResultEnvelope({ operationId: "read_assigned_skill", + callId: "load", correlation, content: result, outcome: "succeeded", code: "semantic_tool_succeeded", + retryable: false, authorizationBoundary: "active_task", operationReceiptId: "operation_load" })]; + return semantic.map((value, i) => change("event", { runId: "run", companyId: "company", agentId: "agent", + seq: i + 1, sourceSeq: i + 1, sourceEventId: `event-${i}`, sourceInstanceId: "instance", protocolSchemaVersion: 1, + eventType: `semantic_tool.${value.phase}`, payload: { prpEvent: { ...correlation, + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceSeq: i + 1, + sourceEventId: `event-${i}`, sourceInstanceId: "instance", eventType: `semantic_tool.${value.phase}`, + payload: { semantic_tool: value } } } })); + } + return ["pending", "completed"].map((status, i) => { + const p = canonicalProviderEventsFromOpenCodePart({ id: "load", type: "tool", tool: "skill", state: { + status, output: `\n${fixture.procedure}\n`, + } })[0]!; + if (wrappedRead === true) { + const output = `/workspace/.claude/skills/${fixture.slug}--0123456789/SKILL.md\nfile\n\n` + + fixture.markdown.trimEnd().split("\n").map((line, index) => `${index + 1}: ${line}`).join("\n") + + "\n(End of file - total 12 lines)\n"; + Object.assign(p.payload, { name: "read", operation: "unknown", readOnly: true, target: null, + output: status === "completed" ? output : "", outputBytes: status === "completed" ? Buffer.byteLength(output) : 0 }); + } + if (wrappedRead === "assigned") { + const result = { skill: skill.key, versionId: skill.currentVersionId, path: "SKILL.md", text: fixture.markdown, + sha256: createHash("sha256").update(fixture.markdown).digest("hex"), byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const output = JSON.stringify({ type: "inputText", text: JSON.stringify(change("assigned-read", result)) }); + Object.assign(p.payload, { name: "paperclip_read_assigned_skill", operation: "unknown", readOnly: true, target: null, + output: status === "completed" ? output : "", outputBytes: status === "completed" ? Buffer.byteLength(output) : 0 }); + } + return change("event", { runId: "run", companyId: "company", agentId: "agent", seq: i + 1, eventType: p.eventType, + sourceEventId: `event-${i}`, sourceInstanceId: "instance", sourceSeq: i + 1, protocolSchemaVersion: 1, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", + eventType: p.eventType, sourceEventId: `event-${i}`, sourceInstanceId: "instance", sourceSeq: i + 1, payload: p.payload } } }); + }); + } + if (route === "/api/issues/issue/documents") return change("documents", [{ key: fixture.key }]); + if (route === `/api/issues/issue/documents/${fixture.key}`) return { id: "document", companyId: "company", issueId: "issue", + key: fixture.key, format: "markdown", body: JSON.stringify(fixture.expected), latestRevisionId: "revision", latestRevisionNumber: 1, + createdByAgentId: "agent", createdByUserId: null, updatedByAgentId: "agent", updatedByUserId: null }; + if (route === `/api/issues/issue/documents/${fixture.key}/revisions/revision`) return change("revision", { + id: "revision", companyId: "company", issueId: "issue", documentId: "document", key: fixture.key, format: "markdown", revisionNumber: 1, + createdByRunId: "run", createdByAgentId: "agent", createdByUserId: null, body: JSON.stringify(fixture.expected), + }); + throw new Error(`Unexpected observation ${route}`); + }), + }; + const page = { goto: vi.fn(async () => {}), reload: vi.fn(async () => { reloaded = true; }), + locator: vi.fn(() => ({ async textContent() { return JSON.stringify(fixture.expected); } })) }; + vi.mocked(createTaskThroughUi).mockImplementation(async input => { + expect(assigned).toBe(true); expect(input.prompt).not.toContain(fixture.expected.reference); + await input.beforeDispatch?.(); issueCreated = true; return { issueId: "issue", submittedAtMs: Date.now() }; + }); + const context = { api, page, company: { id: "company", issuePrefix: "QA" }, user: { id: "owner", name: "Owner" }, + environmentId: "environment", capabilities: ["company", "board-user", "task-write", "agent", "skill-write"], + agent: { id: "agent", name: "Owned agent", provider: "opencode", model: "model", runtimeMode: "native" } } as unknown as JourneyContext; + return { context, api, page, own, evidence, directory, savedSkills, + run: (timeoutMs = 5000) => runSavedSkillUse(context, { own } as unknown as TargetSession, campaignId, evidence, + { deadlineAt: Date.now() + timeoutMs, directory }) }; +} + +describe("saved skill journey orchestration", () => { + it.each([{ kind: "codex" }, { kind: "acpx", agent: "cursor" }, { kind: "cursor_cloud" }])( + "binds a common semantic read for provider %j without an OpenCode directory", async provider => { + const f = await setup((kind, row) => { + if (kind === "run") Object.assign(row.runnerProfileJson.nativeExecutionInput.provider, provider); + return row; + }, "campaign", false, provider.kind === "codex" ? "codex" : "assigned"); + f.context.agent!.provider = provider.kind; + await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + }); + it.each(["codex", "acpx", "cursor_cloud"])("rejects missing or foreign semantic assignment evidence for %s", async provider => { + const changes: Array<{ target: string; mutate(row: Row): void }> = [ + { target: "assignment", mutate: row => { row.entries = []; } }, + { target: "run", mutate: row => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; } }, + { target: "run", mutate: row => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "foreign"; } }, + { target: "assigned-read", mutate: row => { row.versionId = "foreign"; } }, + ]; + for (const { target, mutate } of changes) { + const f = await setup((kind, row) => { + if (kind === "run") Object.assign(row.runnerProfileJson.nativeExecutionInput.provider, { kind: provider, + ...(provider === "acpx" ? { agent: "cursor" } : {}) }); + if (kind === target) mutate(row); + return row; + }, "campaign", false, provider === "codex" ? "codex" : "assigned"); + f.context.agent!.provider = provider; + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + } + }); + it("verifies the ordinary semantic assigned-skill read through all seven checks with Beta skills off", async () => { + const f = await setup(undefined, "campaign", false, "assigned"); await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + expect(f.api.post).toHaveBeenCalledTimes(2); + expect(createTaskThroughUi).toHaveBeenCalledOnce(); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.receipts).toEqual([{ runId: "run", executionId: "load", seq: 2 }]); + expect(receipt.reopened).toEqual(receipt.document); + expect(receipt.retainedSkill).toEqual(receipt.skill); + }); + it.each([ + ["wrong receipt key", "assigned-read", (r: Row) => { r.skill = "foreign"; }], + ["wrong receipt version", "assigned-read", (r: Row) => { r.versionId = "stale"; }], + ["partial source page", "assigned-read", (r: Row) => { r.nextOffset = r.text.length; }], + ["semantic failure", "assigned-read", (r: Row) => { r.outcome = "failed"; }], + ["wrong assigned version", "assignment", (r: Row) => { r.entries[0].versionId = "stale"; }], + ["missing admitted skill", "run", (r: Row) => { r.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["wrong admitted version", "run", (r: Row) => { r.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "stale"; }], + ] as const)("rejects a semantic assigned-skill read with %s before reading its document", async (_label, target, mutate) => { + const f = await setup((kind, row) => { if (kind === target) mutate(row); return row; }, "campaign", false, "assigned"); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it("assigns the input before browser dispatch and verifies native use, saved result and reload without setup supplying output", async () => { + const f = await setup(); await f.run(); + expect(f.own.mock.calls.map(call => call[0])).toEqual([ + expect.objectContaining({ kind: "skill", journeyId: "saved-skill-use" }), + expect.objectContaining({ kind: "skill", id: "skill" }), + expect.objectContaining({ kind: "issue", journeyId: "saved-skill-use" }), + expect.objectContaining({ kind: "issue", id: "issue" }), + ]); + expect(f.api.post).toHaveBeenCalledTimes(2); + expect(createTaskThroughUi).toHaveBeenCalledTimes(1); + expect(f.page.reload).toHaveBeenCalledTimes(1); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.receipts).toEqual([{ runId: "run", executionId: "load", seq: 2 }]); + expect(receipt.reopened).toEqual(receipt.document); + expect(receipt.retainedSkill).toEqual(receipt.skill); + expect(receipt.retainedVersion).toEqual(receipt.version); + expect(receipt.retainedAssignments).toEqual(receipt.assignments); + }); + it("verifies the deployed wrapped native read and then independently checks the saved result and reload", async () => { + const f = await setup(undefined, "campaign", false, true); await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents/saved-skill-result/revisions/"))).toBe(true); + }); + it.each([ + ["missing native skill", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["wrong native version", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "stale"; }], + ["wrong native name", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].runtimeName = "foreign"; }], + ["wrong assigned version", "assignment", (row: Row) => { row.entries[0].versionId = "stale"; }], + ["wrong assigned name", "assignment", (row: Row) => { row.entries[0].runtimeName = "foreign"; }], + ] as const)("rejects a complete wrapped read with %s before inspecting its document", async (_label, target, mutate) => { + const f = await setup((kind, row) => { if (kind === target) mutate(row); return row; }, "campaign", false, true); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it.each([false, true])("uses ordinary Latest assignment with Beta skills %s and no setting read or write", async betaSkillsEnabled => { + const f = await setup(undefined, "campaign", betaSkillsEnabled); await f.run(); + expect(f.api.post.mock.calls[1]).toEqual(["/api/agents/agent/skills/sync?companyId=company", + { desiredSkills: [f.savedSkills[0]!.key], mode: "add" }]); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + expect(f.api.post.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + ["missing snapshot", (row: Row) => { delete row.contextSnapshot.skillSelectionSnapshot; }], + ["false snapshot", (row: Row) => { row.contextSnapshot.skillSelectionSnapshot = false; }], + ["missing versions", (row: Row) => { delete row.contextSnapshot.skillVersions; }], + ["null versions", (row: Row) => { row.contextSnapshot.skillVersions = null; }], + ["array versions", (row: Row) => { row.contextSnapshot.skillVersions = []; }], + ["foreign key", (row: Row) => { row.contextSnapshot.skillVersions = { foreign: "version" }; }], + ["wrong version", (row: Row) => { Object.keys(row.contextSnapshot.skillVersions).forEach(key => { row.contextSnapshot.skillVersions[key] = "stale"; }); }], + ] as const)("rejects an admitted run with %s before trusting its read or result", async (_label, mutate) => { + const f = await setup((kind, row) => { if (kind === "run") mutate(row); return row; }); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_run_version_mismatch" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/events?") || route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it.each([undefined, "stale"])("rejects a created-version receipt %s that cannot bind the immutable input", async currentVersionId => { + const f = await setup((kind, row) => kind === "create-skill" ? { ...row, currentVersionId } : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_created_version_mismatch" }); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + expect(f.own).toHaveBeenCalledTimes(2); + }); + it("rejects Latest source drift between creation and dispatch", async () => { + let reads = 0; + const f = await setup((kind, row) => kind === "skill" && ++reads === 2 ? { ...row, currentVersionId: "changed" } : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_assignment_bound" }); + expect(reads).toBe(2); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + }); + it("rejects another skill's complete native output even when the receipt document would be correct", async () => { + const f = await setup((kind, row) => { + if (kind === "event" && row.eventType === "tool.execution.completed") { + const payload = row.payload.prpEvent.payload; + payload.output = payload.output.replace(//, ''); + payload.outputBytes = Buffer.byteLength(payload.output); + } + return row; + }); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + }); + it.each([null, { presentationDecision: { commentAction: "none" } }])( + "qualifies authoritative committed finalization with optional resultJson %j", async resultJson => { + const f = await setup((kind, row) => kind === "run" ? { ...row, resultJson } : row); + await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + [null, null], + [undefined, { presentationDecision: { commentAction: "none" } }], + ["workspace_finalizing", null], + ["retryable_failure", { finalizationPhase: "committed" }], + ])("does not qualify Done and succeeded before native phase %s is committed", async (nativePhase, resultJson) => { + const f = await setup((kind, row) => kind === "run" ? { ...row, nativePhase, resultJson } : row); + await expect(f.run(250)).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_task_not_completed" }); + expect(f.api.get.mock.calls.some(([route]) => route === "/api/heartbeat-runs/run")).toBe(true); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/events?") || route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.state.taskRuns[0].nativePhase).toBe(nativePhase); + }); + it("waits for native commit before qualifying the already saved current-run result", async () => { + let runReads = 0; + const f = await setup((kind, row) => { + if (kind === "run") return { ...row, nativePhase: ++runReads === 1 ? "workspace_finalizing" : "committed", + resultJson: { finalizationPhase: "committed" } }; + if (kind === "documents") expect(runReads).toBe(2); + return row; + }); + await f.run(); + expect(runReads).toBe(2); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: [{ key: "unknown", versionId: null }] })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: row.desiredSkillEntries.map((entry: Row) => ({ ...entry, versionId: "stale" })) })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: row.desiredSkillEntries.map(({ key }: Row) => ({ key })) })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: [] })], + ["skill", "saved_skill_assignment_bound", (row: Row) => ({ ...row, sourceType: "local_path" })], + ["version", "saved_skill_assignment_bound", (row: Row) => ({ ...row, fileInventory: [{ path: "SKILL.md", content: "changed source" }] })], + ["runs", "saved_skill_run_limit_exceeded", (row: Row[]) => [...row, { runId: "extra" }]], + ["event", "saved_skill_native_read_attributed", (row: Row) => ({ ...row, companyId: "foreign" })], + ["documents", "saved_skill_unexpected_documents", (row: Row[]) => [...row, { key: "unrelated" }]], + ["revision", "saved_skill_result_authored_by_run", (row: Row) => ({ ...row, createdByRunId: "foreign" })], + ["retained-skill", "saved_skill_source_preserved", (row: Row) => ({ ...row, currentVersionId: "changed" })], + ["retained-version", "saved_skill_source_preserved", (row: Row) => ({ ...row, fileInventory: [] })], + ] as const)("fails closed on changed %s evidence and retains its failure checkpoint", async (kind, code, change) => { + const f = await setup((which, row) => which === kind ? change(row as never) : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code }); + expect(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")).toContain("fixture"); + if (kind === "assignment" || kind === "version" || kind === "skill") expect(createTaskThroughUi).not.toHaveBeenCalled(); + else expect(createTaskThroughUi).toHaveBeenCalledTimes(1); + }); + it("journals uncertain creation once without replaying it or dispatching a task", async () => { + const f = await setup(); f.api.post.mockRejectedValueOnce(new Error("Response lost after POST")); + await expect(f.run()).rejects.toThrow("Response lost after POST"); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(f.own).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + }); + it("recovers the exact persisted frontmatter name after a successful creation response is lost", async () => { + const campaignId = "journey-1793701020304-a1b2c3d4"; + const f = await setup((kind, row) => { + if (kind === "create-skill") throw new Error("Response lost after successful creation"); + return row; + }, campaignId); + await expect(f.run()).rejects.toThrow("Response lost after successful creation"); + expect(f.savedSkills).toHaveLength(1); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(f.own).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + const intent = f.own.mock.calls[0]![0]; + const recovered = f.savedSkills.filter(row => row.companyId === f.context.company.id && row.name === intent.name); + expect(recovered).toEqual([f.savedSkills[0]]); + expect(intent.name).toContain(campaignId); + expect(f.savedSkills[0]!.name).toContain(campaignId); + expect(f.api.post.mock.calls[0]![1].name).toBe(f.savedSkills[0]!.name); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts b/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts new file mode 100644 index 00000000000..750de80f293 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts @@ -0,0 +1,129 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { collectRunEvents } from "../run-observations.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { savedSkillDocumentAttributed, savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts, savedSkillReadRuntimeName, savedSkillSourceMatches } from "./saved-skill-use-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "saved-skill-use"; + +export async function runSavedSkillUse(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + if (!context.capabilities.includes("skill-write")) throw new JourneyError("unsupported", "missing_skill_write"); + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const read = createObservedReader(context.api, options.deadlineAt); + const fixture = savedSkillFixture(campaignId, randomBytes(16).toString("hex")); + const title = `Journey QA saved skill ${campaignId}`; + const prompt = `Use /${fixture.slug} for this task. Follow that saved skill's procedure, verify its saved result, and finish. ` + + "Do not create child tasks or unrelated deliverables."; + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + let checkpoint: Row = { fixture }; + let primaryFailure: unknown; + try { + const companyRoute = `/api/companies/${context.company.id}`; + await session.own({ kind: "skill", name: fixture.name, journeyId: id }); + const createdSkill = await context.api.post(`${companyRoute}/skills`, { + name: fixture.name, slug: fixture.slug, description: "Make a receipt using this saved procedure.", + markdown: fixture.markdown, idempotencyKey: `saved-skill-${campaignId}`, + }); + if (!createdSkill?.id) throw new JourneyError("failed", "saved_skill_creation_receipt_missing"); + await session.own({ kind: "skill", name: fixture.name, id: createdSkill.id, journeyId: id }); + checkpoint = { ...checkpoint, createdSkill }; + const skillRoute = `${companyRoute}/skills/${createdSkill.id}`; + const skill = await read.get(skillRoute); + if (!skill?.key || !skill.currentVersionId) throw new JourneyError("failed", "saved_skill_version_missing"); + if (skill.id !== createdSkill.id || !createdSkill.currentVersionId || skill.currentVersionId !== createdSkill.currentVersionId) + throw new JourneyError("failed", "saved_skill_created_version_mismatch"); + await context.api.post(`/api/agents/${agent.id}/skills/sync?companyId=${context.company.id}`, { + desiredSkills: [skill.key], mode: "add", + }); + const assignmentRoute = `/api/agents/${agent.id}/skills?companyId=${context.company.id}`; + const versionRoute = `${skillRoute}/versions/${skill.currentVersionId}`; + const [version, assignments, beforeDispatchSkill] = await Promise.all([ + read.get(versionRoute), read.get(assignmentRoute), read.get(skillRoute), + ]); + checkpoint = { ...checkpoint, skill, version, assignments, beforeDispatchSkill }; + check("saved-skill-assignment-bound", beforeDispatchSkill.id === createdSkill.id + && beforeDispatchSkill.currentVersionId === createdSkill.currentVersionId + && savedSkillSourceMatches(beforeDispatchSkill, version, assignments, fixture, context.company.id)); + + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, + issuePrefix: context.company.issuePrefix, agentName: agent.name, title, prompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const state = await waitFor(options.deadlineAt, async () => { + const [currentIssue, summaries, comments, interactions] = await Promise.all([ + read.get(route), read.get(`${route}/runs`), + read.get(`${route}/comments?order=asc`), read.get>(`${route}/interactions`), + ]); + if (!Array.isArray(summaries) || summaries.length > 1) throw new JourneyError("failed", "saved_skill_run_limit_exceeded"); + const taskRuns = await Promise.all(summaries.map(row => read.get(`/api/heartbeat-runs/${row.runId}`))); + const value = { currentIssue, taskRuns, comments, interactions, attachments: [] }; + validateAgentTaskState(context, created.issueId, value, 1); + checkpoint = { ...checkpoint, state: value }; + return value; + }, value => value.currentIssue.status === "done" && value.taskRuns.length === 1 && value.taskRuns[0]!.status === "succeeded" + && value.taskRuns[0]!.nativePhase === "committed" + && value.interactions.every(row => row.status !== "pending"), "saved_skill_task_not_completed"); + const admittedVersions = state.taskRuns[0]!.contextSnapshot?.skillVersions; + if (state.taskRuns[0]!.contextSnapshot?.skillSelectionSnapshot !== true || !admittedVersions + || typeof admittedVersions !== "object" || Array.isArray(admittedVersions) + || !Object.hasOwn(admittedVersions, skill.key) || admittedVersions[skill.key] !== version.id) + throw new JourneyError("failed", "saved_skill_run_version_mismatch"); + const binding = { companyId: context.company.id, agentId: agent.id, runId: state.taskRuns[0]!.id, issueId: created.issueId }; + const request = String(state.currentIssue.description ?? ""); + check("saved-skill-input-only-in-source", request.includes(`/${fixture.slug}`) && !request.includes(fixture.expected.reference) + && !state.comments.some(row => row.authorUserId && String(row.body ?? "").includes(fixture.expected.reference))); + const events = await collectRunEvents( + (afterSeq, limit) => read.get(`/api/heartbeat-runs/${binding.runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + const runtimeName = savedSkillReadRuntimeName(skill, assignments, state.taskRuns[0]!); + const assignment = savedSkillReadAssignment(skill, assignments, state.taskRuns[0]!); + const receipts = savedSkillReadReceipts(events, binding, fixture, runtimeName, assignment, state.taskRuns[0]!); + checkpoint = { ...checkpoint, events, receipts }; + check("saved-skill-native-read-attributed", receipts.length > 0); + const documents = await read.get(`${route}/documents`); + if (documents.length !== 1 || documents[0]?.key !== fixture.key) throw new JourneyError("failed", "saved_skill_unexpected_documents"); + const documentRoute = `${route}/documents/${fixture.key}`; + const document = await read.get(documentRoute); + const revision = await read.get(`${documentRoute}/revisions/${document.latestRevisionId}`); + checkpoint = { ...checkpoint, document, revision }; + check("saved-skill-result-authored-by-run", savedSkillDocumentAttributed(document, revision, fixture, binding)); + const url = `${context.api.origin}/${context.company.issuePrefix}/issues/${state.currentIssue.identifier ?? created.issueId}`; + await context.page.goto(url, { waitUntil: "domcontentloaded" }); + await context.page.reload(); + await openDocumentAuthoring(context); + await expect(context.page.locator(`#document-${fixture.key}`)).toContainText(fixture.expected.reference); + const [reopened, retainedSkill, retainedVersion, retainedAssignments] = await Promise.all([ + read.get(documentRoute), read.get(skillRoute), read.get(versionRoute), read.get(assignmentRoute), + ]); + checkpoint = { ...checkpoint, reopened, retainedSkill, retainedVersion, retainedAssignments }; + check("saved-skill-result-reopened", reopened.latestRevisionId === document.latestRevisionId + && savedSkillDocumentAttributed(reopened, revision, fixture, binding)); + check("saved-skill-source-preserved", savedSkillSourceMatches(retainedSkill, retainedVersion, retainedAssignments, fixture, context.company.id) + && retainedSkill.id === skill.id && retainedSkill.currentVersionId === skill.currentVersionId); + check("configured-model-and-environment", true); // All observed runs were independently validated above. + } catch (error) { + primaryFailure = error; + throw error; + } finally { + try { + await writeFile(path.join(options.directory, "saved-skill-use-checks.json"), JSON.stringify(checkpoint, null, 2), { mode: 0o600 }); + await read.preserve(options.directory); + } catch (error) { + if (!primaryFailure) throw error; + } + } +} diff --git a/tests/runner-e2e/product-journeys/saved-skill-use.md b/tests/runner-e2e/product-journeys/saved-skill-use.md new file mode 100644 index 00000000000..1ea01e75855 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use.md @@ -0,0 +1,97 @@ +# Use a saved assigned skill + +`saved-skill-use` is a reusable Product E2E journey for an existing target. It +uses one native work run on the target provider's campaign-owned agent. It does +not enroll itself in a daily plan. Local calibration and catalog discovery are +separate from a real staging pass. + +The target supplies ordinary company skill-write access. Assignment uses +`desiredSkills: [canonicalSkillKey]`, which selects Latest and returns a stored +`versionId: null`. This path works with Beta skills disabled. No experimental +setting is read or changed. Experimental version pins need separate qualification. + +The fixture creates one synthetic company skill through the public skill API. +The journal name, API name and Markdown frontmatter name equal its slug. Each +contains the exact campaign ID, including its hyphens. Recovery can therefore +find the persisted skill by name if a successful creation response is lost. +It assigns the skill on the owned agent through the normal skill sync API. +The created skill must be managed, with the exact canonical company key. Its +creation receipt and separate reads must agree on the immutable current version. +A version read must return the exact `SKILL.md` bytes. The skill's +random receipt reference and two numeric inputs occur only in those saved bytes. +The slug and task prompt do not contain the reference. The task is created +through the production browser composer and explicitly names that skill. + +A pass requires all seven closed checks: + +- `saved-skill-assignment-bound`: exact company, managed skill, creation version, + source bytes and one ordinary Latest assignment before dispatch. The successful + run must have `contextSnapshot.skillSelectionSnapshot: true` and an admitted + `skillVersions[canonicalSkillKey]` equal to that immutable created version. +- `saved-skill-input-only-in-source`: the persisted request names the skill but + does not expose its receipt reference; no human comment supplies that reference. +- `saved-skill-native-read-attributed`: a complete successful native skill load or + source-file read, with start/completion identity bound to this company, agent, + run, turn and runner. Its native receipt must contain or cryptographically + commit to the complete saved procedure. +- `saved-skill-result-authored-by-run`: exactly one saved task document contains + the reference and independently calculated sum. Its revision belongs to the + successful native run and agent, without human authorship. The run's + authoritative `nativePhase` must be `committed` before the result qualifies. +- `saved-skill-result-reopened`: the browser reload shows that document, and its + independently read content and revision are unchanged. +- `saved-skill-source-preserved`: the original skill version, source bytes and + assignment remain unchanged after execution. +- `configured-model-and-environment`: the one successful run matches the owned + company, task, agent, configured native provider/model and execution environment. + +The read oracle accepts a completed built-in file read with the exact skill +directory target, or OpenCode's native `skill` load with its exact named +`skill_content` wrapper. Deployed OpenCode can instead report a native `read` +with an unknown operation and null target. That path requires its complete file +result wrapper: the file path must name the exact materialized skill directory. +Both the saved assignment entry and the admitted native execution input must +bind that runtime name to the same canonical key and immutable created version. +The native read must remain read-only, successful, paired by call/turn/runner, +untruncated and byte-complete, and its content must contain the full procedure. +A targetless filesystem read without this bound file wrapper cannot qualify. Discovery, +shell output, model prose, failed or truncated reads, foreign run receipts and +marker-only output cannot pass. + +The first-party `paperclip_read_assigned_skill` tool can also supply the read. +Its complete `inputText` result must contain the exact canonical skill key, +immutable created version, `SKILL.md` path, original Markdown, byte size and +SHA-256 checksum. `nextOffset` must be null. The same dual assignment and +admitted canonical key/version binding is required for every native provider. +The OpenCode materialized directory name is required only for its filesystem +wrapper; it is not a condition for the common first-party semantic read. +A semantic read failure can have a +completed tool status; it does not qualify. A partial page, copied tool result, +foreign version, missing start or unbound native event also fails closed. + +Codex's dynamic tool display is not first-party authority. Its raw name, +optional namespace or read-only hint cannot qualify a read. For Codex, the +grader also accepts Core's retained `semantic_tool.input` and `semantic_tool.result` +for the closed read-only operation `read_assigned_skill`. Both must name the +actual run's runner, native session, call, turn and item. The input must match +its own digest and select the canonical skill or its exact admitted alias, +`SKILL.md` and offset zero (including Core's defaults). The successful +`active_task` result must commit to the exact complete immutable read object: +canonical key, version, file, all UTF-8 source text, SHA-256, byte size and a +null next-page offset. The digest uses Core's canonical JSON key ordering. +Missing, duplicate, reordered, failed, foreign or differently hashed carriers +fail. This path does not relax the existing OpenCode built-in or filesystem guards. + +The existing campaign deadline bounds observations. Only failed read transport +operations can be retried; task input, skill creation and execution are not +replayed. Done status and a successful run alone cannot qualify incomplete native +finalization. The optional `resultJson.finalizationPhase` presentation field is +not required or accepted in place of `nativePhase`; compact or null result data +can qualify when the authoritative phase is committed. +The provider journals skill and task ownership before creation, then +collects evidence and stops/cleans only campaign-owned resources. The fixed skill +source, native events and saved document receipts stay in the private bundle. +The public report uses the existing closed checks and authenticated task capture. + +No instance-wide experimental setting, shared source agent, global model, +company budget or production workspace is changed by this journey. diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index fd9b6886ce8..d730a85f21b 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,5 +1,6 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { + "saved-skill-use": 1, "repository-continuation": 2, "routine-scheduled-run": 1, "file-input-answer": 1, @@ -24,6 +25,7 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { + "saved-skill-use": ["saved-skill-assignment-bound", "saved-skill-input-only-in-source", "saved-skill-native-read-attributed", "saved-skill-result-authored-by-run", "saved-skill-result-reopened", "saved-skill-source-preserved", "configured-model-and-environment"], "task-edit-lifecycle": ["task-lifecycle-browser-created", "task-lifecycle-visible-edits-persisted", "task-lifecycle-edits-reloaded", "task-lifecycle-completed-and-reopened", "task-lifecycle-unedited-context-preserved", "no-agent-work"], "company-access-isolation": ["isolation-owner-resource-controls", "cloud-workspace-switch-and-return", "workspace-data-scoped", "restricted-agent-positive-controls", "cross-workspace-agent-read-write-denied", "isolation-credential-revoked", "no-agent-work"], "repository-continuation": ["repository-configured-source-bound", "repository-first-commit-verified", "repository-original-files-saved", "repository-followup-stored-once", "repository-same-workspace-retained", "repository-second-commit-verified", "repository-original-and-result-reopened", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 2a776353237..51cacf91541 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -26,7 +26,8 @@ export type JourneyId = | "inbox-triage" | "routine-scheduled-run" | "repository-continuation" - | "company-access-isolation"; + | "company-access-isolation" + | "saved-skill-use"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; From cb75c9510c06900411b52573696b3d03c03ac63c Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:28:49 -0600 Subject: [PATCH 25/28] test(evals): restore chat setup and pin secure-entry guidance (#18914) ## Thinking Path > - Paperclip manages agent work and keeps credentials under human control. > - Agents discover supported operations through existing API and connection help. > - The measured secret-entry case chose external provider setup instead of native secure entry. > - The reviewed help distinguishes these actions and keeps the same permissions. > - Two related help changes explain recorded progress and partial team installation. > - The first remote setup found stale reporting-lock metadata on trusted master. > - This PR pins the candidate and renews that existing lock correction for narrow remote checks. ## Linked Issues or Issue Description Refs #18909. Refs #18780. The previous measurement did not complete the requested native secret-entry handoff. The next representative is case 580 on the existing TestOps plan 102 and job 100. Case 610 can use the same source for a separate representative. The progress-help change does not claim that case 466 passes. ## What Changed - Pin immutable candidate `b1ea76b286b870bb62a80eebf3fe00feed6f12c1`, which retains the approval-link fix from #18909. - The candidate adds only reviewed secret-entry guidance, three API help changes, and the matching SDK source registry entry. Its delta from the approval-link candidate covers three files. - Update the launcher snapshot target and expected target test. - Record the actual new SDK compilation and its one changed runtime JavaScript file. Preserve the previous qualification as history. - Refresh only the existing reporting-lock policy and patch from the prior reviewed lock to current master. The helper, actual lockfile, credential rules, and candidate source stay unchanged. - Keep all 244 case rows, inputs, clocks, model, private evidence controls, and held runtime cases unchanged. ## Verification - The owning secret help checks passed: 47 server/shared tests and 31 Secrets UI tests. The real discovery search failed before the help change and passed after it. - The owning progress and team-install search regressions each passed with their full 2057-test runner API suite. - In the candidate, the existing `scripts/ensure-plugin-build-deps.mjs` freshly compiled shared and plugin SDK outputs and exited zero. The emitted guidance consumer passed. Measured output changes contain only the guidance JavaScript, its declarations and maps, and two build receipts. - The old registry and default source proof rejected the changed input. After the existing registry generator updated that one entry, the 506-input check and default source proof passed. Runtime, compiler, parser, Zod, scanner, and canary certificate pins did not change. - All 51 affected SDK tests and all 20 input/build custody controls passed. One initial suite failed to load a missing local dependency; that setup failure is retained. The run with the existing package dependency links passed. - The three existing TestOps contract, adapter, and manifest test files passed: 47 tests, zero failures. - The first case-564 attempt stopped before credentials or model execution: the trusted controller lock had moved through normal dependency updates while its reporting-lock metadata remained stale. That attempt has no original native grade and is not a product failure or a PASS. - The existing lock-refresh command generated the exact reviewed correction from lock `54a81b2e` to current master lock `7540075e`. All seven correction, tamper, and refresh tests pass. The actual current-controller frozen install with lifecycle scripts disabled and both reporting/private-transfer imports pass. The frozen candidate still accepts its unchanged own lock. An initial offline install found an uncached package; the regular frozen install passed. - Fresh CI and review for the renewed head are pending. Local help checks do not establish a native agent outcome. No new TestOps PASS is claimed. ## Risks Help can improve discovery without changing the agent's final choice. The representative runs must test that remaining gap. The candidate retains the prior catalog application base plus the reviewed fixes. It does not refresh or qualify the full catalog. Automatic model retries remain zero and the authorized concurrency cap remains eight. ## Model Used OpenAI `gpt-6.1-sol` through Codex. Capabilities used: source analysis, code integration, compilation, and local validation. The host does not expose the context-window limit. The model under evaluation remains OpenCode/OpenRouter `deepseek/deepseek-v4.1-flash-1017`. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../chat-full-catalog-testops-manifest.json | 47 +++++-- ...hat-full-catalog-testops-manifest.test.mjs | 4 +- tests/runner-e2e/report-lock.json | 6 +- tests/runner-e2e/report-lock.patch | 121 ++++++++++++++++-- 4 files changed, 151 insertions(+), 27 deletions(-) diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index 255f181243b..01595aab574 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1,6 +1,6 @@ { "schema": "paperclip.chat-full-catalog-testops-manifest/v1", - "targetSha": "abbde061c7ec9181021858f7b1af21a48637f97f", + "targetSha": "b1ea76b286b870bb62a80eebf3fe00feed6f12c1", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "profile": "runner-opencode", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017", @@ -1527,8 +1527,8 @@ "condition": "Reviewed synthetic staging input only; per-worker singleton550 enables it; production feed and global index unchanged." }, "sdkPublicProof": { - "previousRegistrySha256": "32c7971e1d9c9d31cd6791045b0265f0475c8f10932ff6dd34681681b6a89342", - "registrySha256": "9169583e54d165dc8d08b9a9010175d11ca87f3590477a7b23032508020ea039", + "previousRegistrySha256": "9169583e54d165dc8d08b9a9010175d11ca87f3590477a7b23032508020ea039", + "registrySha256": "8ecea303821644fa8b7c21d042086096d43dda6c1615b1ca00fbcc4ebb9389c4", "inputCount": 506, "scannerAndCanaryCertificatesUnchanged": true, "plugin558StillHeld": true, @@ -1539,23 +1539,42 @@ "newCandidate115Unqualified": true, "sourceInputsChanged": true, "changedInputPaths": [ - "packages/shared/src/types/work-timeline.ts" + "packages/shared/src/connection-intent-guidance.ts" ], "compileQualification": { - "baselineSourceSha": "124232037eebe73a513907b08f0e96bfdd454c82", - "compiledSourceSha": "6abe5739a713d0a8bb0308b1495846881b82d253", - "acceptedSourceSha": "b734393eff0d7ef85b7ad8abd556d2a66ca3a4e6", + "baselineSourceSha": "abbde061c7ec9181021858f7b1af21a48637f97f", + "compiledSourceSha": "8d19f83b9467bd21d9a1fcd52e10ed372cfa2fbf", + "acceptedSourceSha": "b1ea76b286b870bb62a80eebf3fe00feed6f12c1", "all506InputGitBlobsMatched": true, "owningSharedAndPluginSdkTypeScriptBuildsPassed": true, - "changedRuntimeJavaScriptFiles": 0, - "changedDeclarationFiles": [ - "packages/shared/dist/types/work-timeline.d.ts" + "changedRuntimeJavaScriptFiles": 1, + "changedRuntimeJavaScriptPaths": [ + "packages/shared/dist/connection-intent-guidance.js" ], - "existingAndUnassignedConsumerCompiled": true, - "unknownEventKindRejected": true, + "changedDeclarationFiles": [ + "packages/shared/dist/connection-intent-guidance.d.ts" + ], + "emittedSecureEntryGuidanceConsumerPassed": true, + "existingConnectionRequestAuthorityRetained": true, "sourceSdkControlsPassed": 51, "inputAndBuildCustodyControlsPassed": 20, - "runtimeHeldCasesRemainHeld": true + "runtimeHeldCasesRemainHeld": true, + "previousQualification": { + "baselineSourceSha": "124232037eebe73a513907b08f0e96bfdd454c82", + "compiledSourceSha": "6abe5739a713d0a8bb0308b1495846881b82d253", + "acceptedSourceSha": "b734393eff0d7ef85b7ad8abd556d2a66ca3a4e6", + "all506InputGitBlobsMatched": true, + "owningSharedAndPluginSdkTypeScriptBuildsPassed": true, + "changedRuntimeJavaScriptFiles": 0, + "changedDeclarationFiles": [ + "packages/shared/dist/types/work-timeline.d.ts" + ], + "existingAndUnassignedConsumerCompiled": true, + "unknownEventKindRejected": true, + "sourceSdkControlsPassed": 51, + "inputAndBuildCustodyControlsPassed": 20, + "runtimeHeldCasesRemainHeld": true + } } }, "toolProposal487": { @@ -1578,7 +1597,7 @@ }, "candidateLauncherSnapshot": { "path": "fixtures/chat-repair-candidate-launcher.txt", - "targetSha": "abbde061c7ec9181021858f7b1af21a48637f97f", + "targetSha": "b1ea76b286b870bb62a80eebf3fe00feed6f12c1", "sourcePath": "tests/runner-e2e/launch.ts", "gitBlobOid": "dba1467e211611f37937e9e1fc4c0c5be4329cbc" } diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index 015a41207e6..e2411b41388 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -103,7 +103,7 @@ test('repair source metadata binds the corrected candidate and explicit changed const repair = manifest.source.repairMetadata; assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); assert.equal(repair.previousTargetSha, 'ab2019e976076ec0eb60b026844f3d9211cb0208'); - assert.equal(manifest.targetSha, 'abbde061c7ec9181021858f7b1af21a48637f97f'); + assert.equal(manifest.targetSha, 'b1ea76b286b870bb62a80eebf3fe00feed6f12c1'); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); assert.equal(repair.candidateDescriptorSnapshot, 'fixtures/chat-repair-candidate-descriptors.json'); const descriptorBytes = readFileSync(new URL(repair.candidateDescriptorSnapshot, import.meta.url)); @@ -317,7 +317,7 @@ test('repair source metadata binds the corrected candidate and explicit changed assert.equal(repair.runtimeConditions.toolGuidance106.corePRUnmerged, false); assert.equal(repair.runtimeConditions.sdkPublicProof.scannerAndCanaryCertificatesUnchanged, true); assert.equal(repair.runtimeConditions.sdkPublicProof.sourceInputsChanged, true); - assert.equal(repair.runtimeConditions.sdkPublicProof.registrySha256, "9169583e54d165dc8d08b9a9010175d11ca87f3590477a7b23032508020ea039"); + assert.equal(repair.runtimeConditions.sdkPublicProof.registrySha256, "8ecea303821644fa8b7c21d042086096d43dda6c1615b1ca00fbcc4ebb9389c4"); assert.equal(repair.runtimeConditions.sdkPublicProof.plugin558StillHeld, true); assert.equal(repair.runtimeConditions.sdkPublicProof.expectedLockSha256, "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8"); assert.equal(repair.runtimeConditions.sdkPublicProof.runtimeCompilerParserZodIdentitiesUnchanged, true); diff --git a/tests/runner-e2e/report-lock.json b/tests/runner-e2e/report-lock.json index 125e3af0bc9..e65019c8c6c 100644 --- a/tests/runner-e2e/report-lock.json +++ b/tests/runner-e2e/report-lock.json @@ -1,6 +1,6 @@ { - "inputSha256": "e106b3eab2caa8c282bded699a1c217096039641f885c0a4497ab65eaceedf58", - "outputSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8", - "patchSha256": "0132e268086b9a1e27981a4de76e61a328fe87d9809c7af8ae97f081ea24a0c8", + "inputSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8", + "outputSha256": "7540075e96f9a93924a1e54775c17346c5760dd5d02902d83fb5228f918996ba", + "patchSha256": "dd4a3451167d3306f941d3e82e6892a6dee57133f0d913b1acf43913df2bcf66", "basis": "Generated with the trusted master lockfile refresh. Review this offline correction and its hashes with the dependency changes." } diff --git a/tests/runner-e2e/report-lock.patch b/tests/runner-e2e/report-lock.patch index de0610deb7e..a342ffa458f 100644 --- a/tests/runner-e2e/report-lock.patch +++ b/tests/runner-e2e/report-lock.patch @@ -1,12 +1,117 @@ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml -index c55bccc..2953e8d 100644 +index 2953e8daef..f586e6f8f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml -@@ -5039,6 +5039,7 @@ packages: - '@smithy/core@3.35.1': - resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==} - engines: {node: '>=18.0.0'} -+ deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. +@@ -426,8 +426,8 @@ importers: + specifier: ^24.0.0 + version: 24.13.3 + drizzle-kit: +- specifier: ^0.31.10 +- version: 0.31.10 ++ specifier: ^0.31.11 ++ version: 0.31.11 + tsx: + specifier: ^4.23.15 + version: 4.23.15 +@@ -1015,7 +1015,7 @@ importers: + version: 2.1.0(@grpc/grpc-js@1.14.5)(nice-grpc-common@2.0.4)(nice-grpc@2.1.17) + '@vercel/connect': + specifier: 2.3.3 +- version: 2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)))) ++ version: 2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)))) + acorn: + specifier: 8.18.0 + version: 8.18.0 +@@ -1033,7 +1033,7 @@ importers: + version: 3.0.1(ajv@8.20.0) + better-auth: + specifier: 1.7.2 +- version: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) ++ version: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) + chat: + specifier: 4.39.0 + version: 4.39.0(ai@7.0.130(zod@4.4.3))(zod@4.4.3) +@@ -6667,8 +6667,8 @@ packages: + peerDependencies: + react: ^19.2.8 + +- drizzle-kit@0.31.10: +- resolution: {integrity: sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw==} ++ drizzle-kit@0.31.11: ++ resolution: {integrity: sha512-YCYqxTLIB2OCqf6w9/Vef13baBVbwQIPcbT4Y56AfO6B9ajZhDhD8Jkveg/hJvT/iuMloKD/IYYkyMMNhr7Kqg==} + hasBin: true + + drizzle-orm@0.45.2: +@@ -7066,8 +7066,8 @@ packages: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + +- get-tsconfig@4.14.0: +- resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} ++ get-tsconfig@4.14.3: ++ resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==} + + glob@13.0.6: + resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} +@@ -10641,7 +10641,7 @@ snapshots: + '@esbuild-kit/esm-loader@2.6.5': + dependencies: + '@esbuild-kit/core-utils': 3.3.2 +- get-tsconfig: 4.14.0 ++ get-tsconfig: 4.14.3 + + '@esbuild/aix-ppc64@0.25.12': + optional: true +@@ -13859,14 +13859,14 @@ snapshots: + dependencies: + execa: 5.1.1 + +- '@vercel/connect@2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))))': ++ '@vercel/connect@2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))))': + dependencies: + '@vercel/oidc': 3.8.9 + optionalDependencies: + '@chat-adapter/slack': 4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3) + '@modelcontextprotocol/sdk': 1.31.0(zod@4.4.3) + ai: 7.0.130(zod@4.4.3) +- better-auth: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) ++ better-auth: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) + + '@vercel/oidc@3.2.0': {} + +@@ -14143,7 +14143,7 @@ snapshots: + + before-after-hook@4.0.0: {} + +- better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))): ++ better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))): + dependencies: + '@better-auth/core': 1.7.2(@better-auth/utils@0.4.2)(@better-fetch/fetch@1.3.1)(@opentelemetry/api@1.9.1)(better-call@1.4.0(zod@4.4.3))(jose@6.2.10)(kysely@0.29.5)(nanostores@1.5.3) + '@better-auth/drizzle-adapter': 1.7.2(@better-auth/core@1.7.2(@better-auth/utils@0.4.2)(@better-fetch/fetch@1.3.1)(@opentelemetry/api@1.9.1)(better-call@1.4.0(zod@4.4.3))(jose@6.2.10)(kysely@0.29.5)(nanostores@1.5.3))(@better-auth/utils@0.4.2)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda))) +@@ -14163,7 +14163,7 @@ snapshots: + nanostores: 1.5.3 + zod: 4.4.3 + optionalDependencies: +- drizzle-kit: 0.31.10 ++ drizzle-kit: 0.31.11 + drizzle-orm: 0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)) + pg: 8.18.0 + react: 19.2.8 +@@ -14748,7 +14748,7 @@ snapshots: + react-is: 17.0.2 + tslib: 2.8.1 + +- drizzle-kit@0.31.10: ++ drizzle-kit@0.31.11: + dependencies: + '@drizzle-team/brocli': 0.10.2 + '@esbuild-kit/esm-loader': 2.6.5 +@@ -15156,7 +15156,7 @@ snapshots: + + get-stream@6.0.1: {} + +- get-tsconfig@4.14.0: ++ get-tsconfig@4.14.3: + dependencies: + resolve-pkg-maps: 1.0.0 - '@smithy/credential-provider-imds@4.5.2': - resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} From e7549ab6131555f655736e6b544892a3b1e323cb Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:47:56 -0600 Subject: [PATCH 26/28] fix(runtime): defer native routine recovery collisions (#18916) Treat the exact routine execution unique conflict as a pending recovery obligation after the complete losing transaction rolls back. Preserve the current routine owner, provider attempt and checkpoint, and continue the startup batch. Cover real constraint failures, row preservation, one reclaim after release and unrelated database failures. Co-Authored-By: Paperclip --- doc/execution-semantics.md | 5 + .../native-runtime/native-restart-recovery.ts | 7 ++ ...unner-restart-recovery.integration.test.ts | 96 +++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index ec5752baa18..a5b8a5fe76f 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1264,6 +1264,11 @@ those locks in place and clears them only when the task is done or cancelled. If a scheduled resume finds its execution lock free, restart recovery reclaims it, as the local resume claim does, unless another run for the task is queued, running, or scheduled. A lock held by another run still blocks the resume. +Another task can hold the same routine execution slot even when this task's +lock is free. If that unique claim fails, restart recovery rolls back the whole +candidate claim and waits through the existing recovery deadline. It preserves +the current routine owner, checkpoint, and provider attempt, and continues the +startup batch. Other database failures still fail the recovery pass. Automatic productivity reviews are retired. Run counts, missing comments, and elapsed task time do not create review tasks or impose continuation holds. Bounded continuation, provider recovery, budget limits, explicit blockers, and normal review/approval stages remain in force. Existing productivity-review tasks, comments, assignments, and dependencies remain unchanged and readable; their historical origins still identify them as recovery work for recursion suppression. diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 24afea8b80f..25613267f68 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -34,6 +34,7 @@ import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js import { readProcessStartedAt } from "../hot-restart.js"; import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; +import { isUniqueViolation } from "../../db-errors.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; @@ -1236,6 +1237,12 @@ export async function claimNativeRestartRecoveries(input: { kind: claimKind, ...common, } satisfies NativeRestartRecoveryClaim; + }).catch((error: unknown) => { + if (!isUniqueViolation(error, "issues_open_routine_execution_uq")) throw error; + terminalRunToReport = null; + claimScope = undefined; + return { kind: "awaiting_evidence" as const, runId: candidate.runId, + reason: "routine_execution_busy", retryAt: new Date(now.getTime() + 30_000) }; }); if (terminalRunToReport) void reportRunFailure(input.db, terminalRunToReport); diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index c90619e12c7..4083815d215 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -152,6 +152,102 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( expect(next.kind).toBe("bootstrap_incomplete"); }); + it("preserves a retry when another task owns its routine slot and continues startup recovery", async () => { + const candidates = [await seedRun("ROUTINE-COLLISION"), await seedRun("AFTER-ROUTINE-COLLISION")] + .sort((a, b) => a.runId.localeCompare(b.runId)); + const [waiting, unrelated] = candidates; + const holder = await seedRun("ROUTINE-SLOT-OWNER", waiting.db); + const routineId = randomUUID(); + const now = new Date(); + await waiting.db.update(issues).set({ executionRunId: null, originKind: "routine_execution", + originId: routineId, originFingerprint: "scheduled" }).where(eq(issues.id, waiting.issueId)); + await waiting.db.update(issues).set({ originKind: "routine_execution", + originId: routineId, originFingerprint: "scheduled" }).where(eq(issues.id, holder.issueId)); + await waiting.db.update(heartbeatRuns).set({ status: "failed", finishedAt: now, + nativePhase: "retryable_failure", runnerProfileJson: { ...waiting.preparedProfile, + nativeWorkspaceSync: remotePreparedWorkspace(), + sessionCheckpoint: { identity: { companyId, agentId, issueId: waiting.issueId, runId: waiting.runId, + sessionId: waiting.native.normalizedSessionId }, providerSessionId: "retained-routine-session" }, + } }).where(eq(heartbeatRuns.id, waiting.runId)); + await waiting.db.update(nativeRunFinalizations).set({ controllerBootId: successor.bootId, + leaseOwner: "predecessor-routine-controller", leaseExpiresAt: new Date(now.getTime() + 60_000) }) + .where(eq(nativeRunFinalizations.runId, waiting.runId)); + const authority = new DurablePrpControlPlane({ stateDirectory: resolve(runtimeRoot, waiting.runId), + identity: transportOptions(waiting, runtimeRoot).prpIdentity, + expectedRunnerVersion: "routine-recovery-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + try { + await registerNativeRunnerCheckpoint({ db: waiting.db, + binding: { companyId, issueId: waiting.issueId, runId: waiting.runId }, bootId: successor.bootId, authority }); + } finally { await authority.stop(); } + await waiting.db.update(nativeRunFinalizations).set({ phase: "retryable_failure", attempt: 1, + leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: new Date(0), failureCode: "provider_transport_interrupted" }) + .where(eq(nativeRunFinalizations.runId, waiting.runId)); + const state = async () => ({ + run: await waiting.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, waiting.runId)), + coordinator: await waiting.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, waiting.runId)), + checkpoint: await waiting.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, waiting.runId)), + task: await waiting.db.select().from(issues).where(eq(issues.id, waiting.issueId)), + holder: await waiting.db.select().from(issues).where(eq(issues.id, holder.issueId)), + }); + const before = await state(); + expect(before.checkpoint.length).toBeGreaterThan(0); + const onClaim = vi.fn(); + const input = { db: waiting.db, controller: successor, restartKind: "hard" as const, now }; + const dispositions = await claimNativeRestartRecoveries({ ...input, onClaim, + runIds: candidates.map(candidate => candidate.runId) }); + expect(dispositions).toEqual([ + { kind: "awaiting_evidence", runId: waiting.runId, reason: "routine_execution_busy", + retryAt: new Date(now.getTime() + 30_000) }, + expect.objectContaining({ kind: "bootstrap_incomplete", runId: unrelated.runId }), + ]); + expect(onClaim).toHaveBeenCalledOnce(); + expect(onClaim.mock.calls[0][0].runId).toBe(unrelated.runId); + expect(await state()).toEqual(before); + expect(await waiting.db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, waiting.runId), + eq(heartbeatRunEvents.eventType, "native.recovery.execution_reclaimed")))).toHaveLength(0); + expect(await waiting.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, waiting.runId))).toHaveLength(0); + + const heartbeat = heartbeatService(waiting.db); + try { + await expect(heartbeat.recoverNativeRunsAfterRestart(now, [waiting.runId])) + .resolves.toMatchObject({ claims: [], awaitingEvidenceRunIds: [waiting.runId] }); + } finally { await heartbeat.drainActiveRunExecutions(); } + expect(await state()).toEqual({ ...before, run: [{ ...before.run[0], + nextEventSeq: before.run[0].nextEventSeq + 1, updatedAt: expect.any(Date) }] }); + + await waiting.db.update(issues).set({ executionRunId: null }).where(eq(issues.id, holder.issueId)); + const recovered = (await Promise.all([1, 2].map(() => claimNativeRestartRecoveries({ ...input, + runIds: [waiting.runId] })))).flat(); + expect(recovered.filter(row => row.kind === "reattach_remote_runner")).toHaveLength(1); + const after = await state(); + expect(after.run[0].runnerProfileJson).toEqual(before.run[0].runnerProfileJson); + expect(after.checkpoint).toEqual(before.checkpoint); + expect(after.coordinator[0]).toMatchObject({ attempt: 1, controllerGeneration: 1 }); + expect(after.task[0].executionRunId).toBe(waiting.runId); + expect(await waiting.db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, waiting.runId), + eq(heartbeatRunEvents.eventType, "native.recovery.execution_reclaimed")))).toHaveLength(1); + }); + + it("does not turn another database constraint failure into a routine wait", async () => { + const fixture = await seedRun("UNRELATED-DATABASE-FAILURE"); + const db = new Proxy(fixture.db, { get(target, key) { + if (key === "transaction") return () => target.transaction(async tx => { + await tx.execute(sql`create temp table recovery_constraint_control + (id integer constraint recovery_unrelated_unique unique) on commit drop`); + await tx.execute(sql`insert into recovery_constraint_control values (1), (1)`); + }); + const value = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + } }); + const onClaim = vi.fn(); + await expect(claimNativeRestartRecoveries({ db, controller: successor, + restartKind: "hard", runIds: [fixture.runId], onClaim })).rejects.toThrow(/recovery_constraint_control/); + expect(onClaim).not.toHaveBeenCalled(); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "observed", controllerGeneration: 0, leaseOwner: null }); + }); + it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "finalizer-wait", "warm-suspended", "stale-projection", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry", "diskless-warm-suspended", "diskless-warm-suspended-read-retry", "diskless-live-inspection"])("preserves remote warm continuity across completed turns: %s", async (scenario) => { const hardRestart = !["warm", "same-server", "warm-suspended"].includes(scenario); const diskless = scenario.startsWith("diskless"); From f211d44c00fbc31d22e7ff687e22dfa05643da2a Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:09:58 -0600 Subject: [PATCH 27/28] fix(heartbeat): preserve queued coalesced comments at claim (#18913) ## Thinking Path > - Paperclip manages agents and their work. > - The wake queue records task comments before an agent starts. > - A queued run can accept another comment through coalescing. > - Claim rebuilds that run's input from its primary wake. > - Coalescing updated the run and its receipt but left the primary batch unchanged. > - This change saves the accepted batch atomically in the owning writer. > - The benefit is complete input at claim without repeating old work. ## Linked Issues or Issue Description Related: #18887 preserves offline queued turns. #13948 addresses equivalent automatic wake deduplication. This change repairs the accepted comment batch at the existing claim boundary. **What happened?** A queued run started with comment A. Admission accepted comment B, saved `[A, B]` in the run context, and finished B's coalesced receipt. The primary wake still held `[A]`. Actual claim rebuilt input from that wake and dropped B. **Expected behavior** Claim must retain every accepted live comment in order. Stop, changed source authority, deleted comments, and company boundaries must still apply. Already claimed wakes must retain their existing recovery source. **Steps to reproduce** 1. Queue a task run whose primary wake has an authoritative comment batch `[A]`. 2. Admit comment B through `createHeartbeatQueue.enqueueWakeup` and the real Postgres admission writer. 3. Confirm that admission accepts `[A, B]` and records B's coalesced receipt. 4. Claim the original queued run. Before this fix, the claimed input contains only A. **Paperclip version or commit** The untouched producer regression fails on `d7283e875772ca8c684dc86e471e6b48d543b7b1` with both process and native Runner/Codex adapter selection. Provider execution is disabled in the fixture. ## What Changed - Save the accepted comment batch in the unclaimed primary wake, run context, and coalesced receipt in one transaction. - Lock the primary wake before the run. Recheck the expected run status and same-company, same-agent and run bindings; apply the existing actor equality fence only to durable receipts before mutation. - Preserve incoming input as normal deferred work when the locked source changed. Preserve receipt identity and actor fields. - Preserve the original primary actor and trigger comment while updating its accepted batch. Each incoming receipt retains its actual actor. Keep claimed source payloads and legacy envelopes on their existing recovery paths. - Safely defer a same-named different agent without rewriting the original agent's run, wake or receipt. Missing and foreign-company targets remain rejected. Do not rebuild input from historical receipts. - Add actual process/native producer, real database rollback, Stop, deletion, foreign-source, and fallback controls. Update three direct writer fixtures for the required expected-status contract. ## Verification - The untouched process and native producer red on `d7283e875772ca8c684dc86e471e6b48d543b7b1` loses accepted comment B at actual claim. The initial incomplete mock is retained separately and excluded from behavioral proof. - Current process/native matrix: 22 passed, including mixed authors, system primary, same-named different agents, changed status, claimed sources, Stop, deletion and foreign-source boundaries. Queue, Postgres admission and use-case suites passed 234 cases before the source-preserving serializer follow-up; its 22 actual producer cases passed again afterward. - CI on the previous head found a real ordinary mixed-author workspace-copyback regression. The earlier 13-case selection omitted this test; that coverage gap is preserved. The first correction exposed a second real source-provenance/copyback failure. After preserving the primary trigger and actor while updating only its batch, the complete workspace recovery file passed all 188 cases, including ordinary mixed-author input and native copyback. - Real HTTP queued-comment consumption, discard/reorder and Stop boundaries: 101 passed before these review corrections. Existing chat and Slack consumers each passed their selected real writer case. - Previous-head CI also timed out in the CLI child-server startup fixture and a ZIP transfer case. The full untouched transfer file passed 17 cases locally. Startup remains unclassified; its intentionally reserved port is part of the fixture and is not diagnosed as the cause. No CI retry was issued. - An earlier queue run passed its 231 assertions but timed out during teardown; its preserved replay passed all 60 queue cases. This is distinct from the behavioral reds. - Exact `ab59d944895774d89f950248e356e93af00f4832` server type layers and production server/Runner build passed. The unchanged-default serializer file passed all 11 cases. Current-head CI and both substantive reviews remain required; previous grades and checks do not qualify this head. - No hosted task was retried or changed. Local tests do not establish any hosted deferred event's cause, provider execution, native acknowledgement, physical cleanup, or serving revision. ## Risks - Admission now performs bounded row reads and locks for each coalesced wake. It adds no idle scan or remote request. The existing task transaction owns the operation; lock order follows queued claim. - Claimed-wake recovery uses source identity. Its payload remains unchanged, and the existing actual native copyback tests cover that boundary. - A source that changes before the lock can defer the new input. This preserves work instead of consuming it into a stale run. No migration or historical backfill is required. ## Model Used OpenAI `gpt-6.1-sol`, with reasoning, repository inspection, shell execution, and test tools. The runtime does not report the context-window size. A separate OpenAI coordinator reviewed the source. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/execution-semantics.md | 17 +++ .../__tests__/chat-run-milestone-work.test.ts | 2 +- .../slack-conversation-lifecycle.test.ts | 2 +- .../wake-queue/adapters/postgres.test.ts | 34 ++++- .../modules/wake-queue/adapters/postgres.ts | 26 ++++ .../modules/wake-queue/application/ports.ts | 3 +- .../wake-queue/application/use-cases.test.ts | 16 +++ .../wake-queue/application/use-cases.ts | 6 +- server/src/services/heartbeat/queue.test.ts | 125 +++++++++++++++++- .../services/issue-queued-comment-queue.ts | 9 +- .../src/services/workspace-recovery.test.ts | 2 +- 11 files changed, 228 insertions(+), 14 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index a5b8a5fe76f..2f22aac559f 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2766,6 +2766,23 @@ replay or access: the saved input and current principal, repository, conversatio runtime fence and execution containment still pass normal admission. Explicit conflicting App bindings remain denials. +When comments join a queued run with an unclaimed queued wake, admission saves +the accepted batch in both the run context and its authoritative primary wake in +the same transaction as the coalesced receipt. Claim rebuilds input from that primary wake and filters deleted +comments. It does not reconstruct input from historical coalesced receipts. +Admission locks and rechecks the run and its same-company, same-agent primary +wake. A changed status, mismatched binding, or mismatched durable-receipt actor preserves the incoming +input as deferred work instead of consuming it into the old source. Legacy wakes +without an authoritative batch retain their existing context-based claim path. +An already claimed wake keeps its source unchanged for its existing recovery +and replay path; this change does not grant it a new claim. +Ordinary mixed-author comments keep their existing coalescing behavior. The +primary wake retains its original actor, and each new receipt records its own +actual actor. Coalescing updates the queued batch while preserving the original +trigger comment fields, so recovery still verifies the original source actor. +Same-named agents with different IDs retain separate authority; +their new input is deferred without changing the other agent's queued work. + ### Queued input after a native Stop A run-only Stop ends the current response. It does not discard queued user diff --git a/server/src/__tests__/chat-run-milestone-work.test.ts b/server/src/__tests__/chat-run-milestone-work.test.ts index 9f532ec8fc3..710ba505c68 100644 --- a/server/src/__tests__/chat-run-milestone-work.test.ts +++ b/server/src/__tests__/chat-run-milestone-work.test.ts @@ -330,7 +330,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.transaction(async tx => { await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { - companyId: f.companyId, activeExecutionRunId: seeded.id, mergedContextSnapshot: chatContextSnapshot(), + companyId: f.companyId, activeExecutionRunId: seeded.id, activeExecutionRunStatus: seeded.status, mergedContextSnapshot: chatContextSnapshot(), agentId: f.agentId, source: "on_demand", triggerDetail: null, payload: { issueId: f.issueId }, requestedByActorType: "user", requestedByActorId: f.userId, idempotencyKey: null, }); diff --git a/server/src/__tests__/slack-conversation-lifecycle.test.ts b/server/src/__tests__/slack-conversation-lifecycle.test.ts index 8644c13282b..6e2f5bf7031 100644 --- a/server/src/__tests__/slack-conversation-lifecycle.test.ts +++ b/server/src/__tests__/slack-conversation-lifecycle.test.ts @@ -206,7 +206,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, f.runId)); await db.transaction(async tx => { await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { - companyId: f.companyId, agentId: f.agentId, activeExecutionRunId: f.runId, + companyId: f.companyId, agentId: f.agentId, activeExecutionRunId: f.runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { issueId: f.issueId }, source: "automation", triggerDetail: "system", payload: { issueId: f.issueId }, requestedByActorType: "system", requestedByActorId: null, idempotencyKey: key, diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index cfb8919cc80..3a8eae708c0 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -32,6 +32,7 @@ import { import type { WakeQueuePostgresAdapterDeps } from "./postgres.js"; import { createReleaseIssueExecution } from "../application/use-cases.js"; import type { TransactionScope } from "../application/ports.js"; +import { withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../services/issue-queued-comment-queue.js"; // Proves the atomicity and company-scope properties the security review // requires: every mutation names `companyId` in its own SQL `WHERE` clause, @@ -1139,6 +1140,36 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { // owns it. These tests drive the admission writer directly against a // transaction they open themselves, the same way `heartbeat.ts` will. describe("wake admission", () => { + it.each(["running", "cancelled", "receipt-collision", "actor-mismatch"])("preserves both queued input sources after %s", async scenario => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const [oldComment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "source-user", body: "Original input" }).returning(); + const [newComment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "source-user", body: "Incoming input" }).returning(); + const runId = await seedRun({ companyId, agentId, status: ["receipt-collision", "actor-mismatch"].includes(scenario) ? "queued" : scenario, + contextSnapshot: withQueuedCommentIdsInRunContext({ issueId }, [oldComment.id]) }); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, runId, source: "assignment", status: "queued", + requestedByActorType: "user", requestedByActorId: "source-user", + payload: withQueuedCommentIdsInWakePayload({ issueId }, [oldComment.id]), + }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, runId)); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const operation = db.transaction(async tx => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( + createAdmissionTransactionScope(companyId, tx as unknown as Db), { + companyId, agentId, activeExecutionRunId: runId, activeExecutionRunStatus: "queued", + mergedContextSnapshot: withQueuedCommentIdsInRunContext({ issueId }, [oldComment.id, newComment.id]), + source: "on_demand", triggerDetail: "manual", payload: { issueId, commentId: newComment.id }, + requestedByActorType: "user", requestedByActorId: scenario === "actor-mismatch" ? "another-user" : "source-user", idempotencyKey: null, + ...(["receipt-collision", "actor-mismatch"].includes(scenario) + ? { durableReceipt: { id: scenario === "receipt-collision" ? wake.id : randomUUID(), requestedAt: new Date() } } : {}), + }, + )); + if (scenario === "receipt-collision") await expect(operation).rejects.toMatchObject({ cause: { code: "23505" } }); + else await expect(operation).resolves.toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]).toEqual(before); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).toEqual([wake]); + }); + it("rolls back a deferred merge when its own durable receipt insert fails", async () => { const companyId = await seedCompany(); const agentId = await seedAgent({ companyId }); @@ -1241,7 +1272,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { try { await db.transaction(async tx => { await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(companyId, tx as unknown as Db), { - companyId, activeExecutionRunId: runId, mergedContextSnapshot: { commentId: "fresh-input" }, + companyId, activeExecutionRunId: runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { commentId: "fresh-input" }, agentId, source: "on_demand", triggerDetail: null, payload: null, requestedByActorType: "user", requestedByActorId: null, idempotencyKey: null, }); @@ -1269,6 +1300,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { await writer.coalesceIntoActiveExecutionRun(scope, { companyId: otherCompanyId, activeExecutionRunId: runId, + activeExecutionRunStatus: "running", mergedContextSnapshot: { taskKey: "issue-1", commentId: "c1" }, agentId, source: "on_demand", diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 5c328e25bce..8cfe44af184 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -49,6 +49,7 @@ import { withRecoveryContext } from "../../../services/recovery/status-only-cont import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; import { queuedCommentIdsFromWakePayload, + queuedCommentIdsFromRunContext, withQueuedCommentIdsInWakePayload, } from "../../../services/issue-queued-comment-queue.js"; import { CHAT_CONVERSATION_ID_KEY, extractWakeCommentIds } from "../../run-dispatch/index.js"; @@ -971,7 +972,32 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { async coalesceIntoActiveExecutionRun(scope, input) { const tx = requireAdmissionTx(scope, input.companyId); const now = new Date(); + const [candidate] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, input.activeExecutionRunId), eq(heartbeatRuns.companyId, input.companyId), + )).limit(1); + if (!candidate) throw new Error("wake-queue: the coalesce target run was not found for this company"); + if (candidate.agentId !== input.agentId) return null; + const [primary] = candidate.wakeupRequestId ? await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, candidate.wakeupRequestId), eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.agentId, input.agentId), + )).for("update").limit(1) : []; + const [current] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.agentId, input.agentId), + )).for("update").limit(1); + if (!current || current.status !== input.activeExecutionRunStatus || current.wakeupRequestId !== candidate.wakeupRequestId) return null; + const commentIds = queuedCommentIdsFromRunContext(input.mergedContextSnapshot); + const synchronizeBatch = current.status === "queued" && commentIds.length > 0 + && primary?.status === "queued" && queuedCommentIdsFromWakePayload(primary.payload).length > 0; + if (current.status === "queued" && commentIds.length && current.wakeupRequestId && + (!primary || !["queued", "claimed"].includes(primary.status) || primary.runId !== current.id + || (input.durableReceipt && (primary.requestedByActorType !== input.requestedByActorType || primary.requestedByActorId !== input.requestedByActorId)) + || primary.payload?.issueId !== input.mergedContextSnapshot.issueId)) return null; await notifyChatRunMilestones(tx, { companyId: input.companyId, runId: input.activeExecutionRunId }); + if (synchronizeBatch && primary) await tx.update(agentWakeupRequests).set({ + payload: withQueuedCommentIdsInWakePayload(primary.payload, commentIds, { preserveSource: true }), updatedAt: now, + }).where(and(eq(agentWakeupRequests.id, primary.id), eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.agentId, input.agentId), eq(agentWakeupRequests.runId, current.id), eq(agentWakeupRequests.status, primary.status))); const mergedRun = await tx .update(heartbeatRuns) .set({ contextSnapshot: input.mergedContextSnapshot, updatedAt: now }) diff --git a/server/src/modules/wake-queue/application/ports.ts b/server/src/modules/wake-queue/application/ports.ts index 66c8a631c15..500d1675bc2 100644 --- a/server/src/modules/wake-queue/application/ports.ts +++ b/server/src/modules/wake-queue/application/ports.ts @@ -425,6 +425,7 @@ export interface WakeAdmissionWriter { input: { companyId: string; activeExecutionRunId: string; + activeExecutionRunStatus: WakeAdmissionActiveExecutionRun["status"]; mergedContextSnapshot: Record; durableReceipt?: DurableWakeAdmissionReceipt; agentId: string; @@ -435,7 +436,7 @@ export interface WakeAdmissionWriter { requestedByActorId: string | null; idempotencyKey: string | null; }, - ): Promise>; + ): Promise | null>; /** Merges the wake's context into an already-queued deferred wake, guarded by its current status. */ mergeIntoExistingDeferredWake( scope: TransactionScope, diff --git a/server/src/modules/wake-queue/application/use-cases.test.ts b/server/src/modules/wake-queue/application/use-cases.test.ts index 7f605e70f30..0677818c9c2 100644 --- a/server/src/modules/wake-queue/application/use-cases.test.ts +++ b/server/src/modules/wake-queue/application/use-cases.test.ts @@ -1228,6 +1228,22 @@ describe("admitWakeBehindIssueExecution", () => { expect(reader.findExistingDeferredWake).not.toHaveBeenCalled(); }); + it("retains incoming receipt authority when the locked coalesce target has changed", async () => { + const writer = createFakeAdmissionWriter({ coalesceIntoActiveExecutionRun: vi.fn(async () => null) }); + const reader = createFakeAdmissionReader(); + const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() }); + const durableReceipt = { id: "incoming-source", requestedAt: new Date("2026-11-04T00:00:00Z") }; + expect(await admit(SCOPE, admissionInput({ durableReceipt, wakeCommentId: "new-comment", + contextSnapshot: { issueId: "issue-1", wakeCommentId: "new-comment" }, payload: { issueId: "issue-1", commentId: "new-comment" }, + }))).toEqual({ kind: "deferred" }); + expect(writer.insertNewDeferredWake).toHaveBeenCalledWith(SCOPE, expect.objectContaining({ + companyId: "company-1", agentId: "wake-agent", durableReceipt, + requestedByActorType: "user", requestedByActorId: "user-1", + payload: expect.objectContaining({ issueId: "issue-1", commentId: "new-comment" }), + })); + expect(writer.mergeIntoExistingDeferredWake).not.toHaveBeenCalled(); + }); + it("merges into the existing deferred wake when the policy returns a merge target", async () => { const mergeIntoExistingDeferredWake = vi.fn( async (_scope: TransactionScope, _input: Parameters[1]) => {}, diff --git a/server/src/modules/wake-queue/application/use-cases.ts b/server/src/modules/wake-queue/application/use-cases.ts index 79aada9b05f..6484ee18b9a 100644 --- a/server/src/modules/wake-queue/application/use-cases.ts +++ b/server/src/modules/wake-queue/application/use-cases.ts @@ -870,6 +870,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { const run = await deps.writer.coalesceIntoActiveExecutionRun(scope, { companyId: input.companyId, activeExecutionRunId: target.id, + activeExecutionRunStatus: target.status, mergedContextSnapshot, ...(input.durableReceipt ? { durableReceipt: input.durableReceipt } @@ -882,12 +883,9 @@ export function createAdmitWakeBehindIssueExecution(deps: { requestedByActorId: input.requestedByActorId, idempotencyKey: input.idempotencyKey, }); - return { kind: "coalesced", run }; + if (run) return { kind: "coalesced", run }; } - // decision.kind === "defer": only now does the module read for an - // existing deferred wake, so the coalesce path (the common path) never - // pays for this query. const existingDeferred = (interactionReceipt || input.allowRunCoalescing === false) ? null diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index ef92bf94c65..1dd39dd5067 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -1,13 +1,16 @@ import { claimNativeSessionResumptions } from "../native-runtime/native-finalization-reconciler.js"; import type { Db } from "@paperclipai/db"; -import { authUsers, agents, agentWakeupRequests, caseHandoffs, cases, chatCompletionDeliveries, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { authUsers, agents, agentWakeupRequests, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; -import { createHeartbeatQueue, type HeartbeatQueueDependencies } from "./queue.js"; +import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; +import { createWakeQueue } from "../../modules/wake-queue/index.js"; +import { mergeCoalescedContextSnapshot } from "./run-preparation.js"; +import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js"; import { settleConversationTurn } from "../agent-conversations.js"; import { chatCompletionDeliveryService, recordChatCompletion, recordChatHandoff } from "../chat-completion-delivery.js"; import { instanceSettingsService } from "../instance-settings.js"; @@ -256,6 +259,124 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { return { company, agent, issue, wake, run }; } + it.each((["process", "paperclip_runner"] as const).flatMap(adapterType => (["coalesced", "claimed-primary", "legacy-envelope", "delete-old", "delete-new", "stopped", "running", "system-primary", "foreign-primary", "same-named-agent", "mixed-author"] as const).map(scenario => ({ adapterType, scenario }))))("protects a queued comment batch across $adapterType/$scenario", async ({ adapterType, scenario }) => { + const f = await fixture("queued"); + let admissionAgent: typeof agents.$inferSelect = { ...f.agent, adapterType, adapterConfig: adapterType === "paperclip_runner" ? { provider: "codex" } : {} }; + await db.update(agents).set({ adapterType, adapterConfig: admissionAgent.adapterConfig }).where(eq(agents.id, f.agent.id)); + if (scenario === "same-named-agent") [admissionAgent] = await db.insert(agents).values({ companyId: f.company.id, + name: f.agent.name.toUpperCase(), role: "engineer", status: "idle", adapterType, adapterConfig: admissionAgent.adapterConfig, + }).returning(); + const userId = `source-user-${f.company.id}`; + await db.insert(authUsers).values({ id: userId, name: "Source owner", email: `${f.company.id}@queue.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId: f.company.id, principalType: "user", principalId: userId, status: "active", membershipRole: "operator" }); + await db.update(companies).set({ defaultResponsibleUserId: userId }).where(eq(companies.id, f.company.id)); + const incomingActorId = scenario === "mixed-author" ? `second-user-${f.company.id}` : userId; + if (scenario === "mixed-author") { + await db.insert(authUsers).values({ id: incomingActorId, name: "Incoming source owner", email: `${incomingActorId}@queue.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId: f.company.id, principalType: "user", principalId: incomingActorId, status: "active", membershipRole: "operator" }); + } + const [oldComment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: f.issue.id, + authorType: "user", authorUserId: userId, body: "Review source A" }).returning(); + const [newComment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: f.issue.id, + authorType: "user", authorUserId: incomingActorId, body: "Review source B" }).returning(); + await db.update(agentWakeupRequests).set({ status: scenario === "claimed-primary" ? "claimed" : "queued", runId: f.run.id, claimedAt: null, + requestedByActorType: scenario === "system-primary" ? "system" : "user", requestedByActorId: scenario === "system-primary" ? null : userId, + payload: scenario === "legacy-envelope" ? { issueId: f.issue.id, commentId: oldComment.id } + : withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, retainedSource: "original-source" }, [oldComment.id]), + }).where(eq(agentWakeupRequests.id, f.wake.id)); + const [scanned] = await db.update(heartbeatRuns).set({ responsibleUserId: userId, + contextSnapshot: withQueuedCommentIdsInRunContext({ issueId: f.issue.id, wakeReason: "issue_commented" }, [oldComment.id]), + }).where(eq(heartbeatRuns.id, f.run.id)).returning(); + let foreignPrimary: typeof agentWakeupRequests.$inferSelect | undefined; + if (scenario === "foreign-primary") { + const [company] = await db.insert(companies).values({ name: "Foreign source", issuePrefix: "FOREIGN" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Foreign source agent", adapterType: "process" }).returning(); + [foreignPrimary] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", payload: { source: "foreign" } }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: foreignPrimary.id }).where(eq(heartbeatRuns.id, f.run.id)); + } + if (scenario === "delete-old") await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, oldComment.id)); + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(admissionAgent); + deps.canStartNewExternalBotWork.mockResolvedValue(false); + deps.instanceSettings.getExperimental.mockResolvedValue({ enableIsolatedWorkspaces: false, enableNativeRunner: true } as Awaited>); + const admission = createWakeQueue(db, { + resolveResponsibleUserId: async () => userId, + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + wakeAdmissionHelpers: { filterZombieCoalesceTarget: deps.filterZombieCoalesceTarget, + mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); + deps.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope); + deps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution); + const queue = createHeartbeatQueue(db, deps); + if (scenario === "running") { + deps.canStartNewExternalBotWork.mockResolvedValue(true); + expect(await queue.claimQueuedRun(scanned)).toMatchObject({ status: "running" }); + deps.canStartNewExternalBotWork.mockResolvedValue(false); + } + const [originalPrimary] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)); + const coalesced = await queue.enqueueWakeup(admissionAgent.id, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: { issueId: f.issue.id, commentId: newComment.id }, contextSnapshot: { issueId: f.issue.id, wakeCommentId: newComment.id }, + requestedByActorType: "user", requestedByActorId: incomingActorId }); + if (["running", "foreign-primary", "same-named-agent"].includes(scenario)) { + expect(coalesced).toBeNull(); + const [preserved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(preserved.contextSnapshot).toMatchObject({ wakeCommentId: oldComment.id, wakeCommentIds: [oldComment.id] }); + const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.company.id)); + expect(receipts.find(row => row.id !== f.wake.id)).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(receipts.find(row => row.id === f.wake.id)).toEqual(originalPrimary); + if (scenario === "same-named-agent") { + expect(preserved).toEqual(scanned); + expect(receipts.find(row => row.id !== f.wake.id)).toMatchObject({ agentId: admissionAgent.id, + requestedByActorType: "user", requestedByActorId: incomingActorId, payload: { issueId: f.issue.id, commentId: newComment.id } }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.company.id))).toEqual([scanned]); + } + if (foreignPrimary) expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, foreignPrimary.id)))[0]).toEqual(foreignPrimary); + expect(deps.executeRun).not.toHaveBeenCalled(); + return; + } + expect(coalesced).toMatchObject({ id: f.run.id, status: "queued", contextSnapshot: { wakeCommentIds: [oldComment.id, newComment.id] } }); + const accepted = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.company.id)); + expect(accepted.find(row => row.id !== f.wake.id)).toMatchObject({ status: "coalesced", runId: f.run.id, finishedAt: expect.any(Date), + requestedByActorType: "user", requestedByActorId: incomingActorId }); + const [updatedPrimary] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)); + expect(updatedPrimary).toMatchObject({ id: originalPrimary.id, companyId: originalPrimary.companyId, + agentId: originalPrimary.agentId, source: originalPrimary.source, requestedByActorType: originalPrimary.requestedByActorType, + requestedByActorId: originalPrimary.requestedByActorId, idempotencyKey: originalPrimary.idempotencyKey, requestedAt: originalPrimary.requestedAt }); + if (scenario === "legacy-envelope" || scenario === "claimed-primary") expect(updatedPrimary.payload).toEqual(originalPrimary.payload); + else expect(updatedPrimary.payload).toMatchObject({ retainedSource: "original-source", commentId: oldComment.id, + _paperclipWakeContext: { commentId: oldComment.id, wakeCommentId: oldComment.id, wakeCommentIds: [oldComment.id, newComment.id] } }); + if (scenario === "claimed-primary") { + deps.canStartNewExternalBotWork.mockResolvedValue(true); + expect(await queue.claimQueuedRun(scanned)).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]) + .toMatchObject({ status: "queued", contextSnapshot: { wakeCommentIds: [oldComment.id, newComment.id] } }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)))[0]).toEqual(originalPrimary); + expect(deps.executeRun).not.toHaveBeenCalled(); + return; + } + if (scenario === "stopped") { + await createHeartbeatRunControl(db, runControlCallbacks(db)).cancelRunInternal(f.run.id, "operator Stop"); + deps.canStartNewExternalBotWork.mockResolvedValue(true); + expect(await queue.claimQueuedRun(scanned)).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]) + .toMatchObject({ status: "cancelled", contextSnapshot: { wakeCommentIds: [oldComment.id, newComment.id] } }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)))[0].status).toBe("cancelled"); + expect(deps.executeRun).not.toHaveBeenCalled(); + return; + } + if (scenario === "delete-new") await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, newComment.id)); + deps.canStartNewExternalBotWork.mockResolvedValue(true); + const claimed = await queue.claimQueuedRun(scanned); + expect(claimed).toMatchObject({ id: f.run.id, status: "running", + contextSnapshot: { wakeCommentId: scenario === "delete-new" ? oldComment.id : newComment.id, + wakeCommentIds: scenario === "delete-old" ? [newComment.id] : scenario === "delete-new" ? [oldComment.id] : [oldComment.id, newComment.id] } }); + const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.company.id)); + expect(receipts.find(row => row.id !== f.wake.id)).toMatchObject({ status: "coalesced", runId: f.run.id }); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + async function strandedMention(runtimeMode = "legacy") { const seeded = await fixture("succeeded"); await db.update(heartbeatRuns).set({ runtimeMode, nativeIssueId: runtimeMode === "native" ? seeded.issue.id : null, finishedAt: new Date() }).where(eq(heartbeatRuns.id, seeded.run.id)); diff --git a/server/src/services/issue-queued-comment-queue.ts b/server/src/services/issue-queued-comment-queue.ts index 60ca8efea1e..e50809fe6c7 100644 --- a/server/src/services/issue-queued-comment-queue.ts +++ b/server/src/services/issue-queued-comment-queue.ts @@ -33,15 +33,18 @@ export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] export function withQueuedCommentIdsInWakePayload( payloadValue: unknown, ids: string[], + options: { preserveSource?: boolean } = {}, ): Record { const payload = { ...record(payloadValue) }; const context = { ...record(payload[QUEUE_CONTEXT_KEY]) }; if (ids.length > 0) { const latestId = ids[ids.length - 1]!; context[QUEUE_IDS_KEY] = ids; - context.wakeCommentId = latestId; - context.commentId = latestId; - payload.commentId = latestId; + if (!options.preserveSource) { + context.wakeCommentId = latestId; + context.commentId = latestId; + payload.commentId = latestId; + } } else { delete context[QUEUE_IDS_KEY]; delete context.wakeCommentId; diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index b2d4f1c9155..f6b2177ab78 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -2433,7 +2433,7 @@ describe("automatic retained workspace recovery", () => { expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, actor)).toBe(true); expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, { ...actor, requestedByActorId: "other-user" })).toBe(false); await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(scope, { - companyId: f.companyId, agentId: original.agentId, activeExecutionRunId: queued.id, + companyId: f.companyId, agentId: original.agentId, activeExecutionRunId: queued.id, activeExecutionRunStatus: queued.status, mergedContextSnapshot: mergeCoalescedContextSnapshot((await run(queued.id)).contextSnapshot, incoming, { preserveExistingInteractionContinuation: true }), source: "on_demand", triggerDetail: "manual", From 4a83ba4050f6e6d334fe4cb6e3ace050097530ce Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:30:48 -0600 Subject: [PATCH 28/28] fix(workspaces): isolate unpublished preparation directories (#18922) Bind new remote preparation directories to the writer epoch and environment lease. Preserve earlier unpublished files and the recorded path for published warm workspaces. Run the interrupted reused-allocation and saved-file lifecycle regressions with owned Postgres by default, and retain the warm-path recovery checks. Co-Authored-By: Paperclip --- doc/execution-semantics.md | 7 +++ .../remote-workspaces.integration.test.ts | 51 ++++++++++++++----- server/src/services/remote-workspaces.ts | 3 +- .../src/services/workspace-recovery.test.ts | 4 +- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 2f22aac559f..95d333ac571 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -937,6 +937,13 @@ identity, unknown failures, active leases, and incomplete cleanup still require their normal reconciliation and termination proof. Unknown historical setup errors remain outside this policy, and generic recovery cannot renew its exhausted budget. +When an admitted continuation reuses a sandbox after verified cleanup, an +unpublished preparation uses a directory bound to its writer epoch and lease. +Earlier unpublished directories and their receipts stay intact. Repeating the +same preparation retains its directory and receipt check. A published workspace +on the same allocation retains its recorded directory. Automatic bootstrap +timeout retries still require a fresh allocation under the policy above. + ### Runner authentication and ACP startup handshake bounds The native Runner waits up to the ten-second connection setup budget for the diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index d87caddcc77..0ef9e7ccdf3 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -38,19 +38,30 @@ import { issueService } from "./issues.js"; import { instanceSettingsService } from "./instance-settings.js"; import { resolveRemoteWorkspaceSource } from "./heartbeat/remote-workspace-source.js"; import { toExecutionWorkspace } from "./execution-workspaces.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "../__tests__/helpers/recovery-postgres.js"; const exec = promisify(execFile); -const databaseUrl = process.env.PAPERCLIP_REMOTE_WORKSPACE_TEST_DATABASE_URL; +let databaseUrl = process.env.PAPERCLIP_REMOTE_WORKSPACE_TEST_DATABASE_URL; if (databaseUrl && !/^remote_workspace_[a-z_]*test$/.test(new URL(databaseUrl).pathname.slice(1))) throw new Error("Disposable workspace database required"); -const suite = databaseUrl ? describe : describe.skip; +const suite = databaseUrl || recoveryPostgresSupport.supported ? describe : describe.skip; const s3Bucket = process.env.PAPERCLIP_REMOTE_WORKSPACE_TEST_S3_BUCKET; if (s3Bucket && !s3Bucket.startsWith("paperclip-workspace-")) throw new Error("Disposable workspace S3 bucket required"); suite("remote workspace lifecycle with real Postgres and independent filesystem execution", () => { let db: ReturnType, root: string, storage: StorageService; + let temporary: Awaited> | undefined; let ids: { company: string; agent: string; issue: string; run: string; environment: string; lease: string }; - beforeAll(async () => { await applyPendingMigrations(databaseUrl!); db = createDb(databaseUrl!); }, 120_000); - afterAll(async () => { await closeRegisteredClients(databaseUrl!); }); + beforeAll(async () => { + if (!databaseUrl) { + temporary = await startRecoveryPostgres("remote-workspace-lifecycle-"); + databaseUrl = temporary.connectionString; + } + await applyPendingMigrations(databaseUrl); db = createDb(databaseUrl); + }, 120_000); + afterAll(async () => { + try { if (databaseUrl) await closeRegisteredClients(databaseUrl); } + finally { await temporary?.cleanup(); } + }); beforeEach(async () => { vi.stubEnv("DATABASE_URL", databaseUrl!); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "62".repeat(32)); root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-remote-workspace-lifecycle-"))); @@ -173,40 +184,51 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem expect(Boolean((await lease()).metadata?.remoteExecutionTermination)).toBe(receipt); } - it.each(["receipt", "void"] as const)("recovers a lost preparation reply after %s cleanup and retains the resulting work", async cleanup => { + it.each([["receipt", "fresh"], ["void", "fresh"], ["receipt", "reused"]] as const)( + "continues unpublished preparation after %s cleanup on a %s allocation and retains the resulting work", async (cleanup, allocation) => { await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, issueId: ids.issue }) .where(eq(heartbeatRuns.id, ids.run)); const reserved = await reserve({ kind: "empty" }), first = await target("lost-bootstrap-reply"); + const interrupted = allocation === "reused"; const actual = first.runner!.execute.bind(first.runner!); const execute = vi.spyOn(first.runner!, "execute").mockImplementationOnce(async input => { const result = await actual(input); expect(result.exitCode).toBe(0); - return { ...result, timedOut: true }; + return { ...result, timedOut: !interrupted }; }); - await expect(prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first })) - .rejects.toMatchObject({ code: "remote_workspace_prepare_timeout" }); - const oldRoot = path.join(first.remoteCwd, ".paperclip-workspaces", reserved.id); + let ownershipChecks = 0; + const preparation = prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first, + assertOwned: async () => { if (interrupted && ++ownershipChecks === 2) throw new Error("lease_released_before_terminal"); } }); + if (interrupted) await expect(preparation).rejects.toThrow("lease_released_before_terminal"); + else await expect(preparation).rejects.toMatchObject({ code: "remote_workspace_prepare_timeout" }); + const oldRoot = execute.mock.calls[0][0].args?.[2]; + if (!oldRoot) throw new Error("Preparation did not record its directory"); const oldReceipt = await fs.readFile(path.join(oldRoot, ".paperclip-runtime", "workspace-ready.json"), "utf8"); const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, ids.run)); - await terminalizeLegacyExecution({ db, run: failed!, status: "failed", patch: { - errorCode: "remote_workspace_prepare_timeout", resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + await terminalizeLegacyExecution({ db, run: failed!, status: interrupted ? "interrupted" : "failed", patch: { + errorCode: interrupted ? "lease_released_before_terminal" : "remote_workspace_prepare_timeout", + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, } }); await releaseBootstrapLease(cleanup === "receipt"); const continueOn = async (name: string) => { const runId = randomUUID(), leaseId = randomUUID(); + const reused = allocation === "reused" && name === "fresh-bootstrap-retry"; + const providerLeaseId = reused ? `allocation-${ids.lease}` : `allocation-${leaseId}`; await db.insert(heartbeatRuns).values({ id: runId, companyId: ids.company, agentId: ids.agent, nativeIssueId: ids.issue, runtimeMode: "native", status: "running" }); const [nextLease] = await db.insert(environmentLeases).values({ id: leaseId, companyId: ids.company, environmentId: ids.environment, heartbeatRunId: runId, issueId: ids.issue, provider: "daytona", - providerLeaseId: `allocation-${leaseId}`, status: "active" }).returning(); + providerLeaseId, status: "active" }).returning(); const next = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, runId, agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, existingWorkspaceId: reserved.id, source: { kind: "empty" } }); - const remote = await target(name); + const remote = reused ? { ...first, leaseId, sandboxId: providerLeaseId, + runner: { ...first.runner!, execute: actual } } : await target(name); return prepareRemoteWorkspace({ db, storage, workspace: next, runId, lease: nextLease as EnvironmentLease, target: remote }).then(prepared => ({ runId, prepared, remote })); }; const { runId, prepared, remote } = await continueOn("fresh-bootstrap-retry"); + expect(prepared.remoteRoot).not.toBe(oldRoot); await fs.writeFile(path.join(prepared.remoteRoot, "report.md"), "useful recovered result\n"); const { snapshotId } = await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId, lease: prepared.lease, target: remote, assertOwned: async () => undefined }); @@ -597,7 +619,8 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem existingWorkspaceId: reserved.id, source: { kind: "empty" } }); const nextTarget = await target("restore-cleanup-destination"); const sandbox = nextTarget as Extract; - const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces", reserved.id); + const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces", + `${reserved.id}-${readRemoteWorkspaceAuthority(nextWorkspace.metadata)!.epoch}-${nextLease}`); const execute = sandbox.runner!.execute.bind(sandbox.runner!); let failCleanup = true, cleanupAttempts = 0; sandbox.runner!.execute = async input => { diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 18a31f71f5a..4712ae33568 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -344,7 +344,8 @@ export async function prepareRemoteWorkspace(input: { const sameAllocation = initial.location?.providerLeaseId === input.lease.providerLeaseId && initial.location?.environmentId === input.lease.environmentId; if (initial.location && !sameAllocation && initial.dirty) throw conflict("The prior workspace has unsaved work; recover it before replacing its allocation."); - const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces", input.workspace.id); + const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces", + `${input.workspace.id}-${initial.epoch}-${input.lease.id}`); const preparationIdentity = createHash("sha256").update(JSON.stringify([input.workspace.companyId, input.workspace.id, input.runId, initial.epoch, input.lease.id, input.lease.providerLeaseId, remoteRoot, initial.source])).digest("hex"); let restoredExport: Awaited> | null = null; diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index f6b2177ab78..c93da025b86 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -1815,7 +1815,9 @@ describe("automatic retained workspace recovery", () => { existingWorkspaceId: kind === "same" ? first.workspace.id : null }); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: nextWorkspace, runId: nextRun.id, lease: nextLease as EnvironmentLease, target: { ...target, remoteCwd: resumed!.metadata!.remoteCwd as string } }); - expect(prepared.remoteRoot).toBe(path.join(allocationRoot, ".paperclip-workspaces", nextWorkspace.id)); + const expectedRemoteRoot = kind === "same" ? first.remoteRoot : path.join(allocationRoot, ".paperclip-workspaces", + `${nextWorkspace.id}-${readRemoteWorkspaceAuthority(nextWorkspace.metadata)!.epoch}-${nextLease.id}`); + expect(prepared.remoteRoot).toBe(expectedRemoteRoot); if (kind === "same") expect(await readFile(path.join(prepared.remoteRoot, "saved.txt"), "utf8")).toBe("prior task work\n"); else expect(prepared.remoteRoot).not.toBe(first.remoteRoot); await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId: nextRun.id, lease: prepared.lease,