diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index fdfc680c06e..f8d408cdff9 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2627,6 +2627,11 @@ Environment overrides: negative, empty, or non-numeric value falls back to the default. `PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_ENABLED=false` turns expiry off, and `PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DRY_RUN=true` only reports candidates. +- `PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS=` sets how long native + session continuity-break archives and quarantined session state are kept (see + `doc/recoverable-disk-cleanup.md`). The default is `7` and the minimum is `1`. A + negative, empty, or non-numeric value falls back to the default. The backup + expiration enabled and dry-run switches apply to it as well. - `PAPERCLIP_NATIVE_WORKSPACE_SYNC_RETENTION_DAYS=` sets how long a settled run's native workspace sync folder (descriptor, manifest and seed archives under `/native-workspace-sync/`) is kept (see diff --git a/doc/recoverable-disk-cleanup.md b/doc/recoverable-disk-cleanup.md index b33a0cc74fd..93f9ccb6fef 100644 --- a/doc/recoverable-disk-cleanup.md +++ b/doc/recoverable-disk-cleanup.md @@ -89,10 +89,43 @@ The rest of the session directory, including its control-plane state, is kept. | `PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_ENABLED` | `true` | Set `false` to stop expiring copies | | `PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DRY_RUN` | `false` | Set `true` to report candidates without deleting copies or task sessions | | `PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DAYS` | `7` | Days an issue must be closed before its sessions' copies expire (minimum `1`) | +| `PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS` | `7` | Days continuity-break archives and quarantined session state are kept (minimum `1`) | Each batch that finds candidates logs them with candidate and reclaimed allocated bytes, cleared task sessions, and skip reasons. +### Continuity-break archives and quarantined session state + +The same sweep also removes two kinds of evidence that native sessions leave behind +under the same root: + +- `/continuity-breaks/-/`, written when a session starts a + fresh provider session and moves the retired session's state aside. Nothing reads + these archives again. +- `quarantine/..../`, written when a session's + state loses authority. Recovery and failed-run retry still consult these entries. + +An entry's age comes from the epoch in its name, or from its modification time when +that epoch is in the future. An entry is removed once it is older than +`PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS` and its session is not executing, +initializing, warm, or closing in this process. Quarantine entries created by +native cleanup (`.identity_indeterminate.cleanup.`) are referenced +from recovery history and kept. Symlinks are never followed or removed. Once a +quarantined entry is removed, recovery and failed-run retry no longer see it, so a +later run of that session may start a fresh provider session instead of refusing. + +The evidence check runs on the sweep only when something is due: once at startup, +then when the oldest retained entry, or an entry the executor has written since, +reaches the window, and on the next sweep after a busy session, a failed removal, +or a full batch. It only reads the filesystem and never queries the +database. A dry run reports each candidate once until newly written evidence comes due. + +An entry is renamed to `.expired-` in its folder before it is deleted, and a +later sweep finishes an interrupted deletion. An empty `continuity-breaks` folder +is removed. Archives, quarantine entries, and leftovers count toward the batch +limits of 200 entries and 50 removals shared with backup copies, and +`PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_ENABLED` and `_DRY_RUN` apply to both. + ## Native workspace sync folders Each native run that syncs a workspace with a remote sandbox writes a folder at diff --git a/server/src/__tests__/native-harness-backup-expiration-config.test.ts b/server/src/__tests__/native-harness-backup-expiration-config.test.ts index 81967a9bdad..0b79c92e542 100644 --- a/server/src/__tests__/native-harness-backup-expiration-config.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration-config.test.ts @@ -37,4 +37,15 @@ describe("native harness backup expiration config parsing", () => { vi.stubEnv("PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DAYS", "0"); expect(loadConfig()).toMatchObject({ nativeHarnessBackupExpirationDays: 1 }); }); + + it("keeps native runner evidence for seven days by default and at least one day", () => { + vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", undefined); + expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 7 }); + vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", "14"); + expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 14 }); + vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", "0"); + expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 1 }); + vi.stubEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", "many"); + expect(loadConfig()).toMatchObject({ nativeRunnerEvidenceRetentionDays: 7 }); + }); }); diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index b6330409e72..0123d816a86 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -301,4 +301,378 @@ describeEmbeddedPostgres("native harness backup expiration", () => { expect(await exists(leftover)).toBe(false); expect(await exists(session.current)).toBe(true); }); + async function seedArchive(root: string, input: { stampedDaysAgo?: number; mtimeDaysAgo?: number; name?: string } = {}) { + const stampedAt = now.getTime() - (input.stampedDaysAgo ?? 10) * DAY_MS; + const archive = path.join(root, "continuity-breaks", input.name ?? `${stampedAt}-${randomUUID()}`); + await fs.mkdir(path.join(archive, "codex-home"), { recursive: true }); + await fs.writeFile(path.join(archive, "codex-home", "history.jsonl"), "x".repeat(64 * 1024)); + const mtime = new Date(now.getTime() - (input.mtimeDaysAgo ?? input.stampedDaysAgo ?? 10) * DAY_MS); + await fs.utimes(archive, mtime, mtime); + return archive; + } + + async function seedQuarantine( + scope: string, + input: { stampedDaysAgo?: number; name?: string } = {}, + ) { + if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-durable-sessions-")); + const stampedAt = now.getTime() - (input.stampedDaysAgo ?? 10) * DAY_MS; + const entry = path.join( + stateBase, + "quarantine", + input.name ?? `${scope}.identity_indeterminate.${stampedAt}.4242.${randomUUID()}`, + ); + await fs.mkdir(path.join(entry, "control-plane"), { recursive: true }); + await fs.writeFile(path.join(entry, "history.jsonl"), "x".repeat(64 * 1024)); + return entry; + } + + function evidenceService(overrides: Parameters[1] = {}) { + return nativeHarnessBackupExpirationService(db, { + stateBase, + now, + evidenceRetentionDays: 7, + isSessionBusy: () => false, + ...overrides, + }); + } + + async function emptyStateBase() { + if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-durable-sessions-")); + return createHash("sha256").update(randomUUID()).digest("hex"); + } + + it("removes continuity archives older than the retention window and keeps recent ones", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const old = await seedArchive(session.root, { stampedDaysAgo: 9 }); + const recent = await seedArchive(session.root, { stampedDaysAgo: 2 }); + + const result = await evidenceService().tick(); + + expect(result).toMatchObject({ evidenceCandidates: 1, evidenceRemoved: 1, failed: 0 }); + expect(result.evidenceReclaimedBytes).toBeGreaterThan(0); + expect(result.skipped.recent_evidence).toBe(1); + expect(await exists(old)).toBe(false); + expect(await exists(recent)).toBe(true); + expect(await exists(session.current)).toBe(true); + }); + + it("removes the continuity-breaks folder once its archives are gone", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + await seedArchive(session.root); + + expect(await evidenceService().tick()).toMatchObject({ evidenceRemoved: 1 }); + expect(await exists(path.join(session.root, "continuity-breaks"))).toBe(false); + }); + + it("ages a continuity archive by its name and falls back to mtime for a future stamp", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const stampedRecently = await seedArchive(session.root, { stampedDaysAgo: 1, mtimeDaysAgo: 30 }); + const futureStamp = await seedArchive(session.root, { + name: `${now.getTime() + DAY_MS}-${randomUUID()}`, + mtimeDaysAgo: 30, + }); + const futureRecent = await seedArchive(session.root, { + name: `${now.getTime() + 2 * DAY_MS}-${randomUUID()}`, + mtimeDaysAgo: 1, + }); + + expect(await evidenceService().tick()).toMatchObject({ evidenceRemoved: 1 }); + expect(await exists(stampedRecently)).toBe(true); + expect(await exists(futureStamp)).toBe(false); + expect(await exists(futureRecent)).toBe(true); + }); + + it("ignores folders that are not continuity archives", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const other = await seedArchive(session.root, { name: "notes", stampedDaysAgo: 30 }); + + expect(await evidenceService().tick()).toMatchObject({ evidenceCandidates: 0 }); + expect(await exists(other)).toBe(true); + }); + + it("keeps the evidence of a session that is executing in this process", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const archive = await seedArchive(session.root); + const quarantine = await seedQuarantine(session.scope); + + const result = await evidenceService({ isSessionBusy: (scope) => scope === session.scope }).tick(); + + expect(result).toMatchObject({ evidenceCandidates: 0, evidenceRemoved: 0 }); + expect(result.skipped.session_active).toBe(3); + expect(await exists(archive)).toBe(true); + expect(await exists(quarantine)).toBe(true); + }); + + it("keeps evidence when its session starts while the removal is being prepared", async () => { + const scope = await emptyStateBase(); + const root = path.join(stateBase, scope); + const archive = await seedArchive(root); + let checks = 0; + + const result = await evidenceService({ isSessionBusy: () => ++checks > 1 }).tick(); + + expect(result).toMatchObject({ evidenceCandidates: 1, evidenceRemoved: 0 }); + expect(result.skipped.session_active).toBe(1); + expect(await exists(archive)).toBe(true); + }); + + it("reports evidence without deleting it in dry run", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const archive = await seedArchive(session.root); + const quarantine = await seedQuarantine(session.scope); + const leftover = path.join(session.root, "continuity-breaks", `.expired-${randomUUID()}`); + await fs.mkdir(leftover); + + const result = await evidenceService({ dryRun: true }).tick(); + + expect(result).toMatchObject({ evidenceCandidates: 2, evidenceRemoved: 0, evidenceReclaimedBytes: 0 }); + expect(result.evidenceCandidateBytes).toBeGreaterThan(0); + expect(await exists(archive)).toBe(true); + expect(await exists(quarantine)).toBe(true); + expect(await exists(leftover)).toBe(true); + }); + + it("skips symlinked evidence without following it", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-evidence-outside-")); + try { + await fs.writeFile(path.join(outside, "keep.txt"), "keep"); + const stampedAt = now.getTime() - 10 * DAY_MS; + await fs.mkdir(path.join(session.root, "continuity-breaks"), { recursive: true }); + const archiveLink = path.join(session.root, "continuity-breaks", `${stampedAt}-${randomUUID()}`); + await fs.symlink(outside, archiveLink); + await fs.mkdir(path.join(stateBase, "quarantine"), { recursive: true }); + const quarantineLink = path.join( + stateBase, + "quarantine", + `${session.scope}.identity_indeterminate.${stampedAt}.4242.${randomUUID()}`, + ); + await fs.symlink(outside, quarantineLink); + + const result = await evidenceService().tick(); + + expect(result).toMatchObject({ evidenceCandidates: 0, evidenceRemoved: 0 }); + expect(result.skipped.unsafe_path).toBe(2); + expect(await exists(archiveLink)).toBe(true); + expect(await exists(quarantineLink)).toBe(true); + expect(await exists(path.join(outside, "keep.txt"))).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it("skips symlinked evidence folders without following them", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-evidence-outside-")); + try { + const stampedAt = now.getTime() - 10 * DAY_MS; + const inside = path.join(outside, `${stampedAt}-${randomUUID()}`); + await fs.mkdir(inside); + await fs.utimes(inside, new Date(stampedAt), new Date(stampedAt)); + await fs.symlink(outside, path.join(session.root, "continuity-breaks")); + await fs.symlink(outside, path.join(stateBase, "quarantine")); + + const result = await evidenceService().tick(); + + expect(result).toMatchObject({ evidenceCandidates: 0, evidenceRemoved: 0 }); + expect(result.skipped.unsafe_path).toBe(2); + expect(await exists(inside)).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it("removes quarantine entries older than the retention window and keeps recent ones", async () => { + const scope = await emptyStateBase(); + const old = await seedQuarantine(scope, { stampedDaysAgo: 9 }); + const recent = await seedQuarantine(scope, { stampedDaysAgo: 2 }); + + const result = await evidenceService().tick(); + + expect(result).toMatchObject({ evidenceCandidates: 1, evidenceRemoved: 1, failed: 0 }); + expect(result.skipped.recent_evidence).toBe(1); + expect(await exists(old)).toBe(false); + expect(await exists(recent)).toBe(true); + expect(await fs.readdir(path.join(stateBase, "quarantine"))).toEqual([path.basename(recent)]); + }); + + it("keeps cleanup source archives in quarantine", async () => { + const scope = await emptyStateBase(); + const entry = await seedQuarantine(scope, { name: `${scope}.identity_indeterminate.cleanup.${randomUUID()}` }); + const old = new Date(now.getTime() - 30 * DAY_MS); + await fs.utimes(entry, old, old); + + const result = await evidenceService().tick(); + + expect(result.evidenceCandidates).toBe(0); + expect(result.skipped.cleanup_archive).toBe(1); + expect(await exists(entry)).toBe(true); + }); + + it("finishes removing evidence left over from an interrupted removal", async () => { + const session = await seedSession({ issueStatus: "in_progress" }); + const archive = await seedArchive(session.root, { stampedDaysAgo: 1 }); + const leftover = path.join(session.root, "continuity-breaks", `.expired-${randomUUID()}`); + await fs.mkdir(path.join(leftover, "codex-home"), { recursive: true }); + const quarantineLeftover = path.join(stateBase, "quarantine", `.expired-${randomUUID()}`); + await fs.mkdir(quarantineLeftover, { recursive: true }); + + await evidenceService().tick(); + + expect(await exists(leftover)).toBe(false); + expect(await exists(quarantineLeftover)).toBe(false); + expect(await exists(archive)).toBe(true); + }); + + it.each([false, true])("pages evidence removals across ticks (dry run %s)", async (dryRun) => { + const session = await seedSession({ issueStatus: "in_progress" }); + const archives = [await seedArchive(session.root), await seedArchive(session.root), await seedArchive(session.root)]; + const quarantine = await seedQuarantine(session.scope); + const retention = evidenceService({ dryRun, maxRemovals: 2 }); + + const results = []; + for (let i = 0; i < 4; i++) { + const result = await retention.tick(); + results.push(result); + if (!result.truncated) break; + } + + expect(results.map((result) => result.evidenceCandidates)).toEqual([2, 2]); + expect(results.map((result) => result.truncated)).toEqual([true, false]); + for (const archive of archives) expect(await exists(archive)).toBe(dryRun); + expect(await exists(quarantine)).toBe(dryRun); + }); + + it("does not recount a session's backup when its archives resume on the next tick", async () => { + const session = await seedSession(); + await seedArchive(session.root); + await seedArchive(session.root); + const retention = evidenceService({ dryRun: true, maxRemovals: 2 }); + + const first = await retention.tick(); + const second = await retention.tick(); + + expect(first).toMatchObject({ candidates: 1, evidenceCandidates: 1, truncated: true }); + expect(second).toMatchObject({ candidates: 0, evidenceCandidates: 1, truncated: false }); + }); + it("removes old quarantine entries even when their session's issue is still open", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + const entry = await seedQuarantine(session.scope); + await fs.cp(path.join(session.root, "control-plane"), path.join(entry, "control-plane"), { recursive: true }); + + expect(await evidenceService().tick()).toMatchObject({ evidenceRemoved: 1 }); + expect(await exists(entry)).toBe(false); + }); + + it("stays quiet until retained evidence is due and wakes when new evidence is written", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + const recent = await seedArchive(session.root, { stampedDaysAgo: 2 }); + const retention = evidenceService(); + const at = (days: number) => ({ now: new Date(now.getTime() + days * DAY_MS) }); + + expect((await retention.tick()).skipped.recent_evidence).toBe(1); + expect((await retention.tick(at(4))).skipped.recent_evidence).toBeUndefined(); + expect(await retention.tick(at(6))).toMatchObject({ evidenceRemoved: 1 }); + expect(await exists(recent)).toBe(false); + + const written = await seedArchive(session.root, { stampedDaysAgo: 0 }); + expect(await retention.tick(at(8))).toMatchObject({ evidenceCandidates: 0 }); + retention.noteEvidence(now.getTime()); + expect(await retention.tick(at(8))).toMatchObject({ evidenceRemoved: 1 }); + expect(await exists(written)).toBe(false); + }); + + it("retries evidence of a busy session on the next tick", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + const archive = await seedArchive(session.root); + let busy = true; + const retention = evidenceService({ isSessionBusy: () => busy }); + + expect(await retention.tick()).toMatchObject({ evidenceRemoved: 0 }); + busy = false; + expect(await retention.tick()).toMatchObject({ evidenceRemoved: 1 }); + expect(await exists(archive)).toBe(false); + }); + + it("reports dry-run evidence once until new evidence is written", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + await seedArchive(session.root); + const retention = evidenceService({ dryRun: true }); + + expect(await retention.tick()).toMatchObject({ evidenceCandidates: 1 }); + expect(await retention.tick()).toMatchObject({ evidenceCandidates: 0 }); + retention.noteEvidence(now.getTime()); + expect(await retention.tick({ now: new Date(now.getTime() + 7 * DAY_MS) })).toMatchObject({ evidenceCandidates: 1 }); + }); + + it("counts inspected archives and leftovers against the entry limit", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + const archives = [ + await seedArchive(session.root, { stampedDaysAgo: 1 }), + await seedArchive(session.root, { stampedDaysAgo: 2 }), + await seedArchive(session.root, { stampedDaysAgo: 3 }), + ]; + const leftovers = [randomUUID(), randomUUID()].map((id) => path.join(session.root, "continuity-breaks", `.expired-${id}`)); + for (const leftover of leftovers) await fs.mkdir(leftover); + const retention = evidenceService({ maxEntries: 2 }); + + const first = await retention.tick(); + expect(first).toMatchObject({ checked: 2, truncated: true }); + expect(first.skipped.recent_evidence).toBeUndefined(); + expect((await fs.readdir(path.join(session.root, "continuity-breaks"))).filter((name) => name.startsWith(".expired-"))).toHaveLength(1); + + const results = [first]; + for (let i = 0; i < 6 && results.at(-1)!.truncated; i++) results.push(await retention.tick()); + expect(results.at(-1)!.truncated).toBe(false); + expect(results.every((result) => result.checked <= 2)).toBe(true); + expect(results.every((result) => (result.skipped.recent_evidence ?? 0) <= 2)).toBe(true); + expect(results.reduce((sum, result) => sum + (result.skipped.recent_evidence ?? 0), 0)).toBe(3); + for (const leftover of leftovers) expect(await exists(leftover)).toBe(false); + for (const archive of archives) expect(await exists(archive)).toBe(true); + }); + + it("stops removing leftovers once the sweep must stop", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + await fs.mkdir(path.join(session.root, "continuity-breaks")); + const leftover = path.join(session.root, "continuity-breaks", `.expired-${randomUUID()}`); + await fs.mkdir(leftover); + let calls = 0; + + const retention = evidenceService({ shouldContinue: () => ++calls !== 2 }); + + await retention.tick(); + expect(await exists(leftover)).toBe(true); + await retention.tick(); + expect(await exists(leftover)).toBe(false); + }); + + it("keeps a due evidence pass pending when drain stops it before any session", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + const archive = await seedArchive(session.root); + let draining = true; + const retention = evidenceService({ shouldContinue: () => !draining }); + + expect(await retention.tick()).toMatchObject({ checked: 0, truncated: true }); + draining = false; + expect(await retention.tick()).toMatchObject({ evidenceRemoved: 1 }); + expect(await exists(archive)).toBe(false); + }); + + it("finishes one-entry batches without recounting a resumed session", async () => { + const session = await seedSession({ issueStatus: "in_progress", backupAgeDays: 1 }); + const archives = [await seedArchive(session.root), await seedArchive(session.root)]; + const quarantine = await seedQuarantine(session.scope); + const retention = evidenceService({ maxEntries: 1 }); + + const results = []; + for (let i = 0; i < 8; i++) { + results.push(await retention.tick()); + if (!results.at(-1)!.truncated) break; + } + + expect(results.at(-1)!.truncated).toBe(false); + for (const archive of archives) expect(await exists(archive)).toBe(false); + expect(await exists(quarantine)).toBe(false); + }); }); diff --git a/server/src/config.ts b/server/src/config.ts index 74e80c77ee1..dac9dba2142 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -95,6 +95,7 @@ export interface Config { nativeHarnessBackupExpirationEnabled: boolean; nativeHarnessBackupExpirationDryRun: boolean; nativeHarnessBackupExpirationDays: number; + nativeRunnerEvidenceRetentionDays: number; nativeWorkspaceSyncRetentionEnabled: boolean; nativeWorkspaceSyncRetentionDryRun: boolean; nativeWorkspaceSyncRetentionDays: number; @@ -351,6 +352,7 @@ export function loadConfig(): Config { const skillVersionRetentionKeepNewest = nonNegativeIntegerEnv("PAPERCLIP_SKILL_VERSION_RETENTION_KEEP", 0); const skillVersionRetentionGraceMinutes = nonNegativeIntegerEnv("PAPERCLIP_SKILL_VERSION_RETENTION_GRACE_MINUTES", 60); const nativeHarnessBackupExpirationDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DAYS", 7)); + const nativeRunnerEvidenceRetentionDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS", 7)); const nativeWorkspaceSyncRetentionDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_NATIVE_WORKSPACE_SYNC_RETENTION_DAYS", 7)); const gitRestorePointRetentionDays = Math.max(1, nonNegativeIntegerEnv("PAPERCLIP_GIT_RESTORE_POINT_RETENTION_DAYS", 90)); const gitRestorePointTerminalDelayDays = nonNegativeIntegerEnv("PAPERCLIP_GIT_RESTORE_POINT_TERMINAL_DELAY_DAYS", 3); @@ -417,6 +419,7 @@ export function loadConfig(): Config { nativeHarnessBackupExpirationEnabled: process.env.PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_ENABLED !== "false", nativeHarnessBackupExpirationDryRun: process.env.PAPERCLIP_NATIVE_HARNESS_BACKUP_EXPIRATION_DRY_RUN === "true", nativeHarnessBackupExpirationDays, + nativeRunnerEvidenceRetentionDays, nativeWorkspaceSyncRetentionEnabled: process.env.PAPERCLIP_NATIVE_WORKSPACE_SYNC_RETENTION_ENABLED !== "false", nativeWorkspaceSyncRetentionDryRun: process.env.PAPERCLIP_NATIVE_WORKSPACE_SYNC_RETENTION_DRY_RUN === "true", nativeWorkspaceSyncRetentionDays, diff --git a/server/src/index.ts b/server/src/index.ts index 207f342a2ae..4212387dd04 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -13,6 +13,7 @@ import { NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS, nativeHarnessBackupExpirationService, } from "./services/native-runtime/native-harness-backup-expiration.js"; +import { onNativeRunnerEvidence } from "./services/native-runtime/native-runner-evidence-signal.js"; import { gitRestorePointRetentionService, registerGitRestorePointWork } from "./services/git-restore-point-ledger.js"; import { nativeWorkspaceSyncRetentionService, @@ -1269,8 +1270,10 @@ async function startServerWithDatabaseTeardown( const nativeHarnessBackupExpiration = nativeHarnessBackupExpirationService(db as any, { dryRun: config.nativeHarnessBackupExpirationDryRun, closedForDays: config.nativeHarnessBackupExpirationDays, + evidenceRetentionDays: config.nativeRunnerEvidenceRetentionDays, shouldContinue: () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(), }); + const unsubscribeNativeRunnerEvidence = onNativeRunnerEvidence(nativeHarnessBackupExpiration.noteEvidence); let nativeHarnessBackupExpirationInFlight = false; let nativeHarnessBackupExpirationContinuation: ReturnType | undefined; const sweepNativeHarnessBackups = () => { @@ -1278,7 +1281,7 @@ async function startServerWithDatabaseTeardown( nativeHarnessBackupExpirationInFlight = true; trackHeartbeatSchedulerWork(nativeHarnessBackupExpiration.tick() .then((result) => { - if (result.candidates > 0 || result.failed > 0) { + if (result.candidates > 0 || result.evidenceCandidates > 0 || result.failed > 0) { logger.info({ ...result, dryRun: config.nativeHarnessBackupExpirationDryRun }, "native harness backup expiration"); } if (result.truncated && !heartbeatSchedulerStopped) { @@ -2095,6 +2098,7 @@ async function startServerWithDatabaseTeardown( clearInterval(nativeHarnessBackupExpirationInterval); clearTimeout(nativeHarnessBackupExpirationStartup); clearTimeout(nativeHarnessBackupExpirationContinuation); + unsubscribeNativeRunnerEvidence(); await skillVersionRetention.dispose(); await skillSourceSync.stop(); unsubscribeChatCompletions(); diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index 448997670e1..d4c0975f0b5 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -17,10 +17,15 @@ import { nativeSessionScopeDigestIsBusy } from "./native-session-executor.js"; export const NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS = 60 * 60 * 1000; export const NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS = 2 * 60 * 1000; export const NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS = 7; +export const NATIVE_RUNNER_EVIDENCE_RETENTION_DEFAULT_DAYS = 7; const DAY_MS = 24 * 60 * 60 * 1000; const SCOPE_PATTERN = /^[0-9a-f]{64}$/; const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const CONTINUITY_ARCHIVE_PATTERN = /^(\d{1,15})-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const QUARANTINE_PATTERN = + /^([0-9a-f]{64})\.(?:identity_indeterminate|identity_mismatch)\.(\d{1,15})\.\d{1,10}\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const QUARANTINE_CLEANUP_PATTERN = /^[0-9a-f]{64}\.identity_indeterminate\.cleanup\./; const EXPIRED_PREFIX = ".expired-"; const RETIRING_PREFIX = ".retiring-"; const LIVE_RUN_STATUSES = ["queued", "scheduled_retry", "running"]; @@ -31,6 +36,7 @@ const MAX_CONTROL_PLANE_STATE_BYTES = 4 * 1024 * 1024; export type NativeHarnessBackupExpirationOptions = { closedForDays?: number; + evidenceRetentionDays?: number; dryRun?: boolean; maxEntries?: number; maxRemovals?: number; @@ -47,11 +53,25 @@ export type NativeHarnessBackupExpirationResult = { expired: number; reclaimedBytes: number; clearedTaskSessions: number; + evidenceCandidates: number; + evidenceCandidateBytes: number; + evidenceRemoved: number; + evidenceReclaimedBytes: number; failed: number; truncated: boolean; skipped: Record; }; +type EvidenceContext = { + dryRun: boolean; + isSessionBusy: (scopeDigest: string) => boolean; + now: number; + retentionMs: number; + retainedBefore: number; + canInspect: () => boolean; + dueAt: (at: number) => void; +}; + type SessionIdentity = { runId: string; normalizedSessionId: string; @@ -66,6 +86,10 @@ function emptyResult(): NativeHarnessBackupExpirationResult { expired: 0, reclaimedBytes: 0, clearedTaskSessions: 0, + evidenceCandidates: 0, + evidenceCandidateBytes: 0, + evidenceRemoved: 0, + evidenceReclaimedBytes: 0, failed: 0, truncated: false, skipped: {}, @@ -223,6 +247,13 @@ export function nativeHarnessBackupExpirationService( defaults: NativeHarnessBackupExpirationOptions = {}, ) { let cursor: string | null = null; + let resume: { scope: string; archive: string } | null = null; + let quarantineCursor: string | null = null; + let phase: "sessions" | "quarantine" = "sessions"; + let passActive = false; + let evidencePass = false; + let evidenceDueAt = 0; + let passDueAt = Number.POSITIVE_INFINITY; async function settleLeftovers(backupRoot: string) { const current = path.join(backupRoot, "current"); @@ -310,46 +341,247 @@ export function nativeHarnessBackupExpirationService( result.reclaimedBytes += bytes; } + async function removeExpiredLeftovers(parent: string, context: EvidenceContext, result: NativeHarnessBackupExpirationResult) { + for (const name of await fs.readdir(parent).catch(() => [] as string[])) { + if (!name.startsWith(EXPIRED_PREFIX) || !UUID_PATTERN.test(name.slice(EXPIRED_PREFIX.length))) continue; + if (!context.canInspect()) { + context.dueAt(context.now); + return; + } + result.checked++; + await removeBackupTree(path.join(parent, name)); + } + } + + async function retireEvidence( + parent: string, + name: string, + scopeDigest: string, + stampedAt: number, + context: EvidenceContext, + result: NativeHarnessBackupExpirationResult, + ) { + const skip = (reason: string) => { + result.skipped[reason] = (result.skipped[reason] ?? 0) + 1; + }; + const target = path.join(parent, name); + const stat = await fs.lstat(target).catch(() => null); + if (path.dirname(target) !== parent || !stat?.isDirectory()) { + return skip("unsafe_path"); + } + const createdAt = stampedAt > 0 && stampedAt <= context.now ? stampedAt : stat.mtimeMs; + if (createdAt >= context.retainedBefore) { + context.dueAt(createdAt + context.retentionMs); + return skip("recent_evidence"); + } + if (context.isSessionBusy(scopeDigest)) { + context.dueAt(context.now); + return skip("session_active"); + } + const bytes = await allocatedBytes(target); + result.evidenceCandidates++; + result.evidenceCandidateBytes += bytes; + if (context.dryRun) return; + if (context.isSessionBusy(scopeDigest)) { + context.dueAt(context.now); + return skip("session_active"); + } + const trash = path.join(parent, `${EXPIRED_PREFIX}${randomUUID()}`); + await fs.rename(target, trash); + await removeBackupTree(trash); + result.evidenceRemoved++; + result.evidenceReclaimedBytes += bytes; + } + + async function retireContinuityBreaks( + root: string, + scopeDigest: string, + resumeAfter: string | null, + context: EvidenceContext, + result: NativeHarnessBackupExpirationResult, + ): Promise { + const parent = path.join(root, "continuity-breaks"); + const stat = await fs.lstat(parent).catch(() => null); + if (!stat) return null; + if (!stat.isDirectory()) { + result.skipped.unsafe_path = (result.skipped.unsafe_path ?? 0) + 1; + return null; + } + if (!context.dryRun) await removeExpiredLeftovers(parent, context, result); + let last = resumeAfter ?? ""; + const names = (await fs.readdir(parent)).filter((name) => name > last).sort(); + for (const name of names) { + const match = CONTINUITY_ARCHIVE_PATTERN.exec(name); + if (!match) continue; + if (!context.canInspect()) return last; + result.checked++; + last = name; + try { + await retireEvidence(parent, name, scopeDigest, Number(match[1]), context, result); + } catch (err) { + result.failed++; + context.dueAt(context.now); + logger.warn({ err, scope: scopeDigest }, "native runner evidence retention skipped a continuity archive"); + } + } + if (!context.dryRun) await fs.rmdir(parent).catch(() => undefined); + return null; + } + + async function retireQuarantine( + base: string, + context: EvidenceContext, + result: NativeHarnessBackupExpirationResult, + ): Promise { + const parent = path.join(base, "quarantine"); + const stat = await fs.lstat(parent).catch(() => null); + if (!stat) return true; + if (!stat.isDirectory()) { + result.skipped.unsafe_path = (result.skipped.unsafe_path ?? 0) + 1; + return true; + } + if (!context.dryRun && quarantineCursor === null) await removeExpiredLeftovers(parent, context, result); + const names = (await fs.readdir(parent)).sort(); + const pending = quarantineCursor ? names.filter((name) => name > quarantineCursor!) : names; + for (const name of pending) { + const match = QUARANTINE_PATTERN.exec(name); + const cleanupArchive = QUARANTINE_CLEANUP_PATTERN.test(name); + if (!match && !cleanupArchive) continue; + if (!context.canInspect()) return false; + result.checked++; + quarantineCursor = name; + if (!match) { + result.skipped.cleanup_archive = (result.skipped.cleanup_archive ?? 0) + 1; + continue; + } + try { + await retireEvidence(parent, name, match[1]!, Number(match[2]), context, result); + } catch (err) { + result.failed++; + context.dueAt(context.now); + logger.warn({ err, scope: match[1] }, "native runner evidence retention skipped a quarantine entry"); + } + } + quarantineCursor = null; + return true; + } + + function noteEvidence(writtenAt: number) { + evidenceDueAt = Math.min(evidenceDueAt, writtenAt + evidenceRetentionMs()); + } + + function evidenceRetentionMs(options: NativeHarnessBackupExpirationOptions = {}) { + return Math.max( + 1, + options.evidenceRetentionDays ?? defaults.evidenceRetentionDays ?? NATIVE_RUNNER_EVIDENCE_RETENTION_DEFAULT_DAYS, + ) * DAY_MS; + } + async function tick(options: NativeHarnessBackupExpirationOptions = {}) { const closedForDays = Math.max( 1, options.closedForDays ?? defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS, ); + const retentionMs = evidenceRetentionMs(options); const dryRun = options.dryRun ?? defaults.dryRun ?? false; const maxEntries = Math.max(1, options.maxEntries ?? defaults.maxEntries ?? 200); const maxRemovals = Math.max(1, options.maxRemovals ?? defaults.maxRemovals ?? 50); const shouldContinue = options.shouldContinue ?? defaults.shouldContinue ?? (() => true); const isSessionBusy = options.isSessionBusy ?? defaults.isSessionBusy ?? nativeSessionScopeDigestIsBusy; const base = path.resolve(options.stateBase ?? defaults.stateBase ?? nativeHarnessStateBase()); - const closedBefore = new Date((options.now ?? defaults.now ?? new Date()).getTime() - closedForDays * DAY_MS); + const now = (options.now ?? defaults.now ?? new Date()).getTime(); + const closedBefore = new Date(now - closedForDays * DAY_MS); const result = emptyResult(); - - const names = (await fs.readdir(base).catch(() => [] as string[])) - .filter((name) => SCOPE_PATTERN.test(name)) - .sort(); - const pending = cursor ? names.filter((name) => name > cursor!) : names; - for (const name of pending) { - if (!shouldContinue() || result.checked >= maxEntries || result.candidates >= maxRemovals) { - result.truncated = true; - break; - } - result.checked++; - cursor = name; - const root = path.join(base, name); - if (path.dirname(root) !== base || !(await isRealDirectory(root))) { - result.skipped.unsafe_path = (result.skipped.unsafe_path ?? 0) + 1; - continue; - } - try { - await expireSession(root, name, { dryRun, isSessionBusy, closedBefore }, result); - } catch (err) { - result.failed++; - logger.warn({ err, scope: name }, "native harness backup expiration skipped a session"); + const hasBudget = () => shouldContinue() && result.candidates + result.evidenceCandidates < maxRemovals; + if (!passActive) { + passActive = true; + evidencePass = evidenceDueAt <= now; + if (evidencePass) { + evidenceDueAt = Number.POSITIVE_INFINITY; + passDueAt = Number.POSITIVE_INFINITY; } } - if (!result.truncated) cursor = null; + const context: EvidenceContext = { + dryRun, + isSessionBusy, + now, + retentionMs, + retainedBefore: now - retentionMs, + canInspect: () => hasBudget() && result.checked < maxEntries, + dueAt: (at) => { + passDueAt = Math.min(passDueAt, at); + }, + }; + + if (phase === "sessions") { + const names = (await fs.readdir(base).catch(() => [] as string[])) + .filter((name) => SCOPE_PATTERN.test(name)) + .sort(); + const pending = cursor ? names.filter((name) => name > cursor!) : names; + for (const name of pending) { + if (!hasBudget() || result.checked >= maxEntries) { + result.truncated = true; + break; + } + const resumeAfter = resume?.scope === name ? resume.archive : null; + if (resume && name >= resume.scope) resume = null; + if (resumeAfter === null) result.checked++; + const previous = cursor; + cursor = name; + const root = path.join(base, name); + if (path.dirname(root) !== base || !(await isRealDirectory(root))) { + result.skipped.unsafe_path = (result.skipped.unsafe_path ?? 0) + 1; + continue; + } + if (resumeAfter === null) { + try { + await expireSession(root, name, { dryRun, isSessionBusy, closedBefore }, result); + } catch (err) { + result.failed++; + logger.warn({ err, scope: name }, "native harness backup expiration skipped a session"); + } + } + if (!evidencePass) continue; + let stoppedAt: string | null = null; + try { + stoppedAt = await retireContinuityBreaks(root, name, resumeAfter, context, result); + } catch (err) { + result.failed++; + context.dueAt(now); + logger.warn({ err, scope: name }, "native runner evidence retention skipped a session"); + } + if (stoppedAt !== null) { + resume = { scope: name, archive: stoppedAt }; + cursor = previous; + result.truncated = true; + break; + } + } + if (result.truncated) return result; + cursor = null; + if (!evidencePass) { + passActive = false; + return result; + } + phase = "quarantine"; + } + try { + if (!(await retireQuarantine(base, context, result))) { + result.truncated = true; + return result; + } + } catch (err) { + result.failed++; + context.dueAt(now); + quarantineCursor = null; + logger.warn({ err }, "native runner evidence retention skipped quarantine"); + } + phase = "sessions"; + passActive = false; + evidencePass = false; + evidenceDueAt = Math.min(evidenceDueAt, passDueAt); return result; } - return { tick }; + return { tick, noteEvidence }; } diff --git a/server/src/services/native-runtime/native-runner-evidence-signal.ts b/server/src/services/native-runtime/native-runner-evidence-signal.ts new file mode 100644 index 00000000000..39d51bee146 --- /dev/null +++ b/server/src/services/native-runtime/native-runner-evidence-signal.ts @@ -0,0 +1,12 @@ +const listeners = new Set<(writtenAt: number) => void>(); + +export function onNativeRunnerEvidence(listener: (writtenAt: number) => void): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +export function noteNativeRunnerEvidence(writtenAt: number): void { + for (const listener of listeners) listener(writtenAt); +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 5c8c3e211f7..39862333f8c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -394,6 +394,7 @@ import { verifyNativeHarnessBackup, shouldRestoreNativeHarnessBackupIntoSandbox, } from "./native-session-executor.js"; +import { onNativeRunnerEvidence } from "./native-runner-evidence-signal.js"; beforeEach(() => { usageRecorder.capture.mockReset().mockImplementation(async receipt => receipt); @@ -12442,6 +12443,8 @@ describe("runnerd provider runtime wiring", () => { await writeRunnerAuthority(legacyRoot, identity, "ready"); state.createBackend.mockClear(); state.createTransport.mockClear(); + const evidence: number[] = []; + const stopEvidence = onNativeRunnerEvidence((writtenAt) => evidence.push(writtenAt)); await expect( createRunnerdBackend({ @@ -12449,11 +12452,12 @@ describe("runnerd provider runtime wiring", () => { execution: currentExecution, runnerInstanceId: "runner-after-legacy-terminal-unsuspended", }), - ).rejects.toThrow("runner_state_identity_mismatch"); + ).rejects.toThrow("runner_state_identity_mismatch").finally(stopEvidence); await expect(access(legacyRoot)).rejects.toThrow(); const quarantineEntries = await readdir(join(stateBase, "quarantine")); expect(quarantineEntries).toHaveLength(1); expect(quarantineEntries[0]).toContain(".identity_indeterminate."); + expect(evidence).toEqual([Number(quarantineEntries[0]!.split(".")[2])]); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7f67a8aa2fc..234f673e664 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -209,6 +209,7 @@ import { type NativeRunTrace, } from "./native-run-trace.js"; import { createNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; +import { noteNativeRunnerEvidence } from "./native-runner-evidence-signal.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { registerLiveRunnerGoalController } from "../runner-goal-control-broker.js"; import { applyRunnerGoalPrpEvent } from "../runner-goals.js"; @@ -1952,11 +1953,13 @@ function quarantineRunnerdStateRoot( if (!isSafeNativeStateDirectory(quarantineRoot)) { throw new Error("runner_state_directory_unsafe"); } + const quarantinedAt = Date.now(); const destination = resolve( quarantineRoot, - `${stateKey}.${reason}.${Date.now()}.${process.pid}.${randomUUID()}`, + `${stateKey}.${reason}.${quarantinedAt}.${process.pid}.${randomUUID()}`, ); renameSync(resolvedRoot, destination); + noteNativeRunnerEvidence(quarantinedAt); return destination; } @@ -13276,9 +13279,11 @@ async function createRunnerdBackendWithinSessionClaim( if (hasRetainedWarmTransitionEvidence(root)) { throw new Error("native_runner_warm_transition_recovery_unproven"); } - const archiveToken = `${Date.now()}-${randomUUID()}`; + const archivedAt = Date.now(); + const archiveToken = `${archivedAt}-${randomUUID()}`; const archiveRoot = resolve(root, "continuity-breaks", archiveToken); mkdirSync(archiveRoot, { recursive: true, mode: 0o700 }); + noteNativeRunnerEvidence(archivedAt); for (const name of [ "control-plane", "runner",