diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index 9bbb5e95257..d4a9b2be91e 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -107,7 +107,14 @@ export async function validateNativeRunnerCheckpointOwner( gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`), notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : []; - if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) { + const [waitingSuccessor] = !successor && !preparingSuccessor && task?.executionRunId && task.executionRunId !== owner.runId + ? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId), + eq(heartbeatRuns.issueId, owner.issueId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]), + notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : []; + if (!current.resultId || newWork || !task + || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor && !waitingSuccessor)) { throw new Error("native_runner_checkpoint_owner_lost"); } } diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 336971ff09f..625ea711774 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -1328,7 +1328,7 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); } finally { await authority.stop(); } }); - it.each((["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller", "unaccepted"] as const).flatMap(scenario => + it.each((["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller", "unaccepted", "queued_successor", "scheduled_retry_successor", "queued_wrong_issue", "queued_new_work"] as const).flatMap(scenario => (["runner.suspend", "turn.stop"] as const).map(command => ({ scenario, command }))))( "settles a completed warm checkpoint during early retirement without granting work: $scenario/$command", async ({ scenario, command }) => { @@ -1365,17 +1365,19 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); let nextRunId: string | null = null; if (scenario !== "idle") { nextRunId = randomUUID(); - const preparing = scenario !== "claimed_successor" && scenario !== "unclaimed_successor"; - const otherIssue = scenario === "wrong_issue" ? (await seedRun(`WARM-OTHER-ISSUE-${command}`)).issueId : fixture.issueId; + const waiting = scenario.startsWith("queued_") || scenario === "scheduled_retry_successor"; + const preparing = !waiting && scenario !== "claimed_successor" && scenario !== "unclaimed_successor"; + const otherIssue = scenario === "wrong_issue" || scenario === "queued_wrong_issue" ? (await seedRun(`WARM-OTHER-ISSUE-${scenario}-${command}`)).issueId : fixture.issueId; await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId, issueId: otherIssue, scopeKind: "issue", contextSnapshot: { issueId: otherIssue }, - ...(preparing ? { runtimeMode: "legacy", executionStage: scenario === "dispatching_successor" ? "dispatching" : "preparing", + ...(waiting ? { status: scenario === "scheduled_retry_successor" ? "scheduled_retry" : "queued" } : {}), + ...(waiting ? {} : preparing ? { runtimeMode: "legacy", executionStage: scenario === "dispatching_successor" ? "dispatching" : "preparing", controllerBootId: scenario === "foreign_preparation" ? randomUUID() : legacyControllerBootId, controllerLeaseExpiresAt: scenario === "expired_preparation" ? new Date(0) : new Date(Date.now() + 60_000), ...(scenario === "foreign_native_preparation" ? { nativeIssueId: fixture.issueId } : {}), } : { runtimeMode: "native", nativeIssueId: fixture.issueId, nativeSessionId: randomUUID() }), - status: "running", invocationSource: "manual" }); - if (!preparing || scenario === "foreign_native_preparation") await fixture.db.insert(nativeRunFinalizations).values({ runId: nextRunId, companyId, issueId: fixture.issueId, + ...(waiting ? {} : { status: "running" }), invocationSource: "manual" }); + if ((!preparing && !waiting) || scenario === "foreign_native_preparation") await fixture.db.insert(nativeRunFinalizations).values({ runId: nextRunId, companyId, issueId: fixture.issueId, phase: "observed", ...(scenario === "claimed_successor" || scenario === "foreign_native_preparation" ? { controllerBootId: scenario === "foreign_native_preparation" ? randomUUID() : successor.bootId, controllerGeneration: 1, leaseOwner: "next-warm-owner", leaseExpiresAt: new Date(Date.now() + 60_000) } : {}), @@ -1383,8 +1385,8 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); } await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId)); const before = (await loadNativeRunnerCheckpoint(fixture.db, binding))!; - authority.queueCommand(scenario === "new_work" ? "turn.start" : command, {}, `early-retirement-${scenario}`); - if (!["idle", "preparing_successor", "claimed_successor"].includes(scenario)) { + authority.queueCommand(scenario === "new_work" || scenario === "queued_new_work" ? "turn.start" : command, {}, `early-retirement-${scenario}`); + if (!["idle", "preparing_successor", "claimed_successor", "queued_successor", "scheduled_retry_successor"].includes(scenario)) { await expect(authority.flushCheckpoint()).rejects.toThrow("native_runner_checkpoint_owner_lost"); expect((await loadNativeRunnerCheckpoint(fixture.db, binding))?.records).toEqual(before.records); } else { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 8f841d8be5a..36b0662d4ba 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -11318,6 +11318,44 @@ describe("native warm session supervision", () => { } }); + it("a follow-up that arrives while the idle timeout is closing the warm session waits for that close instead of failing busy", async () => { + const identity = "warm-idle-close-follow-up"; + const warmExecution = { ...execution, binding: { ...execution.binding, runId: identity, executionWorkspaceId: `${identity}-workspace` }, + session: { ...execution.session, normalizedSessionId: identity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 50 } }, + } as NativeExecutionInputV1; + let closeStarted!: () => void, finishClose!: () => void; + const closing = new Promise(resolve => { closeStarted = resolve; }); + const closeGate = new Promise(resolve => { finishClose = resolve; }); + const session = { close: vi.fn(async () => { closeStarted(); await closeGate; }) }; + state.execute.mockReset().mockImplementation(async (options) => { + await options.onSession?.(session); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: identity, normalizedSessionId: identity, providerSessionId: identity, + driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + const copy = (runId: string) => ({ runId, root: `/tmp/${identity}`, preparationKey: "same", hasChanges: async () => false, + collectStopped: async () => undefined }); + try { + await executePaperclipNativeSession({ db: leaseDb(warmExecution), execution: warmExecution, runnerInstanceId: identity, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId: identity, remoteCwd: `/tmp/${identity}` }, + instructionWorkingCopy: copy(identity) }); + await closing; + const followUpRunId = `${identity}-follow-up`; + const adopt = vi.fn(async () => copy(followUpRunId)); + const followUp = claimWarmNativeInstructionCopy({ priorExecution: warmExecution, + companyId: warmExecution.binding.companyId, agentId: warmExecution.binding.agentId, + executionWorkspaceId: warmExecution.binding.executionWorkspaceId, workspace: warmExecution.workspace, + environmentId: identity, runId: followUpRunId, preparationKey: "same", adopt }); + setTimeout(finishClose, 100); + await expect(followUp).resolves.toBeNull(); + expect(adopt).not.toHaveBeenCalled(); + expect(session.close).toHaveBeenCalledTimes(1); + } finally { + finishClose(); + await closeWarmNativeSessionsForEnvironment({ environmentId: identity, reason: "test cleanup" }); + } + }); + it("admits the follow-up's execution only after the previous owner's final cleanup hands its scopes back", async () => { const identity = "warm-release-scope-handback"; const warmExecution = { ...execution, binding: { ...execution.binding, runId: identity, executionWorkspaceId: `${identity}-workspace` }, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f8fc648a289..9fd14ac5af6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -935,6 +935,13 @@ async function waitForWarmNativeSessionSettled(scope: string): Promise { while (settlingNativeSessionScopes.has(scope) && Date.now() < deadline) { await new Promise(resolve => setTimeout(resolve, warmSessionReleaseWait.pollMs)); } + const closing = closingWarmNativeSessions.get(scope); + if (!closing) return; + let timer: ReturnType | undefined; + await Promise.race([ + closing.catch(() => undefined), + new Promise(resolve => { timer = setTimeout(resolve, Math.max(0, deadline - Date.now())); }), + ]).finally(() => clearTimeout(timer)); } /** Reserve the existing physical copy before heartbeat composes AGENT_HOME and @@ -961,7 +968,7 @@ export async function claimWarmNativeInstructionCopy(input: { })) || prior.workspace.cwd !== input.workspace.cwd) return null; const scope = nativeSessionScopeKey(prior); const busyEntry = warmNativeSessions.get(scope); - if (busyEntry?.busy || settlingNativeSessionScopes.has(scope)) await waitForWarmNativeSessionSettled(scope); + if (busyEntry?.busy || settlingNativeSessionScopes.has(scope) || closingWarmNativeSessions.has(scope)) await waitForWarmNativeSessionSettled(scope); // Re-read after the wait: the release may have retired or replaced the session. const entry = warmNativeSessions.get(scope); if (!entry || !entry.instructionWorkingCopy?.runId || entry.environmentId !== input.environmentId) return null; diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 3db04081944..9d2107a22ae 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1518,6 +1518,14 @@ tried again on the next poll with the same `clientRequestId`. The overlap window `staging-mix.ts`, which a swap driver registers. Without a probe, the window is any time a fault is in progress and the server still answers (a K15 overlap). The timeline records `staging_mix_follow_up`. +To aim a follow-up at the warm session's idle close instead, set `RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS=-` +(for example `298000-306000` for the 300 s idle timeout). The harness then posts the follow-up that long after the +task's last run ended, `RESTART_CHAOS_MIX_FOLLOW_UP_COUNT` times (default 2), and waits for every post before the task +counts as finished. A failed post is tried again with the same `clientRequestId`. The task budget grows by one +maximum delay plus one turn per post. The grade check `after_end_follow_ups_served` needs every post, an agent comment +after each post and the `FOLLOWUP-` ledger line. The timeline records `staging_mix_follow_up_after_end` with the +measured delay. + **Retries: hands-off first.** By default the harness never clicks Retry during the wave or the settle, so the product's own continuation is what gets measured. A task that shows a Retry banner is kept under watch, not rested. Every banner is recorded per task: when it first showed, how long it stayed, and the error code behind it, even when the task then diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 41f7e609faa..999dda01c42 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -53,6 +53,37 @@ describe("TaskObserver", () => { expect(observer.done).toBe(true); }); + it("does not finish a staging-mix task on the poll that posts its last after-end follow-up", async () => { + const previous = { ...process.env }; + const delay = REST_CONFIRM_MS + 50; + Object.assign(process.env, { RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS: `${delay}-${delay}`, RESTART_CHAOS_MIX_FOLLOW_UP_COUNT: "1" }); + try { + let endedAt = 0; + const { api, posted } = fakeApi({ "/runs": () => [{ runId: "r1", status: "succeeded", createdAt: new Date(endedAt - 1_000).toISOString(), + finishedAt: new Date(endedAt).toISOString() }] }); + const mixPlan: TaskPlan = { ...plan, workload: "SM", fault: "K0", followUpAfterMs: null, + stagingMix: { turnMinutes: 2, rounds: 2, subtasks: 0, followUpAfterMs: 1_000, arriveAtMs: 0, holdMs: 0 } }; + const observer = new TaskObserver(mixPlan, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + endedAt = start; + const comments = () => posted.filter(route => route.endsWith("/comments")).length; + await observer.poll(start + 100); + expect(comments()).toBe(0); + await observer.poll(start + 100 + REST_CONFIRM_MS + 1); + expect(comments()).toBe(1); + expect(observer.done).toBe(false); + await observer.poll(start + 200 + REST_CONFIRM_MS + 2); + expect(observer.done).toBe(false); + await observer.poll(start + 300 + 2 * REST_CONFIRM_MS + 3); + expect(observer.done).toBe(true); + } finally { + for (const key of ["RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS", "RESTART_CHAOS_MIX_FOLLOW_UP_COUNT"]) { + if (previous[key] === undefined) delete process.env[key]; else process.env[key] = previous[key]; + } + } + }); + it("keeps watching unexplained idle work until the stall limit, then records the stall", async () => { const { api } = fakeApi({}, "in_progress"); const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index c51c2917dd4..c24c7e7dd25 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -20,7 +20,7 @@ import { awaitingQuestionAnswer, collectQuestionArtifacts, questionStep } from " import { isSessionFault, rotateAgentSession } from "./session-faults.js"; import { awaitingRewake, collectRewakeArtifacts, isRewakeWorkload, rewakeStep } from "./rewake.js"; import { withLongTurn } from "./long-turn.js"; -import { collectStagingMixArtifacts, holdsRetry, stagingMixDefinition, stagingMixStep } from "./staging-mix.js"; +import { collectStagingMixArtifacts, followUpAfterEndRange, holdsRetry, stagingMixDefinition, stagingMixStep } from "./staging-mix.js"; import { captureRemoteWorkspaceFiles, readRemoteWorkspaceFile } from "./remote-workspace-files.js"; import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; @@ -492,7 +492,12 @@ export class TaskObserver { await this.captureRemoteFilesAtDone(sample); this.readySince = sample.reachable && sample.serverReady ? (this.readySince ?? now) : null; await this.observeBenchProgress(); + const afterEndPosts = this.record.stagingMix?.afterEndFollowUps?.length ?? 0; await this.governanceStep(sample, now); + if ((this.record.stagingMix?.afterEndFollowUps?.length ?? 0) > afterEndPosts) { + this.restCandidateSince = null; + return; + } if (this.journey && await this.journey.step({ sample, now, pendingFaults: this.pendingFaults, quiet: sample.reachable && sample.serverReady && !this.supervisor.faultInProgress })) this.restCandidateSince = null; if (await this.observeChildStalls(now)) { @@ -859,6 +864,8 @@ export class TaskObserver { if (this.plan.workload === "H1") return Boolean(governance?.budgetPausedAt) && !governance?.resume && !governance?.pauseLostAt; if (this.plan.workload === "Q1" || this.plan.workload === "Q4") return awaitingQuestionAnswer(this.record); if (isRewakeWorkload(this.plan.workload)) return awaitingRewake(this.record); + const afterEnd = followUpAfterEndRange(); + if (afterEnd && this.plan.stagingMix?.followUpAfterMs != null) return (this.record.stagingMix?.afterEndFollowUps?.length ?? 0) < afterEnd.count; return false; } diff --git a/tests/restart-chaos/staging-mix.test.ts b/tests/restart-chaos/staging-mix.test.ts index b2b08a8c4d7..7dcfd30df49 100644 --- a/tests/restart-chaos/staging-mix.test.ts +++ b/tests/restart-chaos/staging-mix.test.ts @@ -4,7 +4,7 @@ import { parseCommand } from "./config.js"; import { agentRunCaps, assignAdapters, childLine, childNonce, doneMarker, expectedRoundHash, followUpMarker, gradeStagingMix, holdsRetry, planArrivals, arrivalEndMs, followUpRequestId, planStagingMixWave, plannedTaskCount, posixCksum, retryDue, roundCommand, ROUND_SECONDS, setOverlapProbe, stagingMixDescription, - stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask, + stagingMixBudgetMs, stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask, } from "./staging-mix.js"; import { descendantsOf, parseMemInfo, parsePs, parseVmStat, StagingMixRun, stagingMixSeedOptions, type ResourceSample } from "./staging-mix-run.js"; import type { OracleArtifacts, TaskRecord, TaskSample } from "./types.js"; @@ -303,6 +303,43 @@ describe("staging-mix board step", () => { expect(keys[0]).not.toBe(followUpRequestId("i2", "n1")); }); + it("with an after-end delay, retries a failed post with the same key, and grades every post as answered and served", async () => { + const previous = { ...process.env }; + Object.assign(process.env, { RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS: "300000-300000", RESTART_CHAOS_MIX_FOLLOW_UP_COUNT: "1" }); + try { + const ended: TaskSample = { ...running, issueStatus: "done", + runs: [{ ...running.runs![0]!, status: "succeeded", finishedAt: new Date(10_000).toISOString() }] }; + const rec = record(); + expect((await step(rec, ended, 300_000, false)).posted).toEqual([]); + await step(rec, ended, 310_000, false, undefined, () => { throw new Error("HTTP 502"); }); + expect(rec.stagingMix!.afterEndFollowUps).toEqual([]); + expect(rec.stagingMix!.followUpErrors).toEqual(["HTTP 502"]); + expect((await step(rec, ended, 313_000, false)).posted.length).toBe(1); + expect(rec.stagingMix!.afterEndFollowUps).toEqual([{ afterRunEndAt: 10_000, at: 313_000, delayMs: 303_000, commentId: "c1" }]); + expect(new Set(requests.slice(-2)).size).toBe(1); + expect((await step(rec, ended, 320_000, false)).posted).toEqual([]); + + const nonce = "cafe0002"; + const plan = task({ followUpAfterMs: 1 }); + const afterEndFollowUps = [{ afterRunEndAt: 10_000, at: 313_000, delayMs: 303_000, commentId: "c1" }]; + const served = (comments: Array<{ body: string; runId: string | null; createdAt: number }>, extra: string[]) => gradeStagingMix(nonce, plan, artifacts({ + documents: { ledger: ledgerFor(nonce, 4, extra) }, agentComments: comments.map(comment => comment.body), + stagingMix: { followUp: { at: 313_000, commentId: "c1", duringRun: false, inOverlap: false, error: null }, followUpErrors: [], retries: [], comments, afterEndFollowUps } })) + .checks.find(check => check.name === "after_end_follow_ups_served")!; + expect(served([{ body: doneMarker(nonce), runId: "r1", createdAt: 9_000 }], [followUpMarker(nonce)]).ok).toBe(false); + expect(served([{ body: doneMarker(nonce), runId: "r2", createdAt: 400_000 }], []).ok).toBe(false); + expect(served([{ body: doneMarker(nonce), runId: "r2", createdAt: 400_000 }], [followUpMarker(nonce)]).ok).toBe(true); + const never = gradeStagingMix(nonce, plan, artifacts({ documents: { ledger: ledgerFor(nonce, 4) }, agentComments: [doneMarker(nonce)], + stagingMix: { followUp: null, followUpErrors: ["HTTP 502"], retries: [], comments: [] } })); + expect(never.checks.find(check => check.name === "after_end_follow_ups_served")).toMatchObject({ ok: false, detail: "posted=0/1; unanswered=none" }); + expect(stagingMixBudgetMs(plan) - stagingMixBudgetMs(task())).toBe(300_000 + 4 * ROUND_SECONDS * 1_000); + } finally { + for (const key of ["RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS", "RESTART_CHAOS_MIX_FOLLOW_UP_COUNT"]) { + if (previous[key] === undefined) delete process.env[key]; else process.env[key] = previous[key]; + } + } + }); + it("posts the follow-up before the turn's last rounds when no overlap comes", async () => { const rec = record(); const plan = task({ rounds: 4, followUpAfterMs: 60_000 }); diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index cff36dadc35..b84536c645a 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -50,6 +50,7 @@ export interface StagingMixTask { export interface StagingMixRecord { followUpErrors?: string[]; followUp?: { at: number; commentId: string | null; duringRun: boolean; inOverlap: boolean; error: string | null }; + afterEndFollowUps?: Array<{ afterRunEndAt: number; at: number; delayMs: number; commentId: string }>; retrySeenAt?: number | null; retries?: Array<{ at: number; inOverlap: boolean }>; retryBanner?: RetryBanner; @@ -71,6 +72,7 @@ export interface StagingMixArtifacts { comments?: Array<{ body: string; runId: string | null; createdAt?: number | null }>; commentsUnreadable?: boolean; completion?: CompletionEvidence; + afterEndFollowUps?: NonNullable; } export const DUPLICATE_CHECKS: ReadonlySet = new Set(["rounds_once", "done_comment_once", "completion_once", "subtasks_once", "subtask_rounds_once"]); @@ -478,6 +480,15 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: checks.push({ name: "follow_up_posted", ok: Boolean(followUp?.commentId), detail: followUp?.commentId ? `comment ${followUp.commentId}` : `never posted${errors.length ? `; errors: ${errors.slice(-3).join(" | ")}` : ""}` }); } + const afterEnd = artifacts.stagingMix?.afterEndFollowUps ?? []; + const range = task.followUpAfterMs !== null ? followUpAfterEndRange() : null; + if (range || afterEnd.length) { + const replies = artifacts.stagingMix?.comments ?? []; + const unanswered = afterEnd.filter(entry => !replies.some(comment => comment.createdAt != null && comment.createdAt >= entry.at)); + checks.push({ name: "after_end_follow_ups_served", + ok: afterEnd.length >= (range?.count ?? 1) && unanswered.length === 0 && (ledger ?? "").includes(followUpMarker(nonce)), + detail: `posted=${afterEnd.length}/${range?.count ?? afterEnd.length}; unanswered=${unanswered.map(entry => entry.commentId).join(", ") || "none"}` }); + } if (followUp?.commentId && followUp.duringRun) { checks.push({ name: "follow_up_served", ok: (ledger ?? "").includes(followUpMarker(nonce)), detail: `posted ${new Date(followUp.at).toISOString()}${followUp.inOverlap ? " in overlap" : ""}` }); @@ -503,7 +514,9 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: export function stagingMixBudgetMs(task: StagingMixTask): number { const turnMs = task.rounds * ROUND_SECONDS * 1_000; - return Math.round(turnMs * 1.5) + 20 * 60_000 + (task.subtasks ? 20 * 60_000 : 0); + const afterEnd = task.followUpAfterMs !== null ? followUpAfterEndRange() : null; + return Math.round(turnMs * 1.5) + 20 * 60_000 + (task.subtasks ? 20 * 60_000 : 0) + + (afterEnd ? afterEnd.count * (afterEnd.maxMs + turnMs) : 0); } export function stagingMixDefinition(task: StagingMixTask): WorkloadDefinition { @@ -559,13 +572,45 @@ export interface StagingMixStepInput { note: (kind: string, data: Record) => void; } +export function followUpAfterEndRange(environment: NodeJS.ProcessEnv = process.env): { minMs: number; maxMs: number; count: number } | null { + const raw = environment.RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS; + if (!raw) return null; + const [low, high] = parseRange(raw, "RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS", 0, 3_600_000); + const count = whole(environment.RESTART_CHAOS_MIX_FOLLOW_UP_COUNT, "RESTART_CHAOS_MIX_FOLLOW_UP_COUNT", 2, 1, 10); + return { minMs: low, maxMs: high, count }; +} + export async function stagingMixStep(input: StagingMixStepInput): Promise { const { record, task, sample, now } = input; const state = record.stagingMix ??= {}; if (!sample.reachable || !record.issueId || record.createdAt === null) return; const overlap = insideOverlapWindow(now, { faultInProgress: input.faultInProgress, reachable: sample.reachable }); const running = (sample.runs ?? []).some(run => run.status === "running"); - if (task.followUpAfterMs !== null && !state.followUp && running) { + const afterEnd = followUpAfterEndRange(); + if (afterEnd && task.followUpAfterMs !== null) { + const posted = state.afterEndFollowUps ??= []; + const runs = sample.runs ?? []; + const active = runs.some(run => ACTIVE.has(run.status)); + const lastEnd = Math.max(0, ...runs.map(run => Date.parse(run.finishedAt ?? "") || 0)); + if (!active && lastEnd > 0 && posted.length < afterEnd.count && !posted.some(entry => entry.afterRunEndAt === lastEnd)) { + const span = afterEnd.maxMs - afterEnd.minMs; + const delay = afterEnd.minMs + (span > 0 ? (Number.parseInt(input.nonce.slice(0, 6), 16) * (posted.length + 1)) % (span + 1) : 0); + if (now >= lastEnd + delay) { + const index = posted.length; + let commentId: string | null = null; let error: string | null = null; + try { + const comment = await input.api.post<{ id?: string }>(`/api/issues/${record.issueId}/comments`, { body: followUpComment(input.nonce), + clientRequestId: followUpRequestId(record.issueId, `${input.nonce}-after-end-${index}`) }); + commentId = typeof comment?.id === "string" ? comment.id : null; + if (!commentId) error = "the comment response had no id"; + } catch (caught) { error = caught instanceof Error ? caught.message : String(caught); } + if (commentId) posted.push({ afterRunEndAt: lastEnd, at: now, delayMs: now - lastEnd, commentId }); + else (state.followUpErrors ??= []).push(error!); + input.note("staging_mix_follow_up_after_end", { index, delayMs: now - lastEnd, commentId, error }); + if (commentId && !state.followUp) state.followUp = { at: now, commentId, duringRun: false, inOverlap: overlap, error: null }; + } + } + } else if (task.followUpAfterMs !== null && !state.followUp && running) { const plannedAt = record.createdAt + task.followUpAfterMs; const latestAt = Math.min(plannedAt + task.holdMs, record.createdAt + Math.round(task.rounds * ROUND_SECONDS * 1_000 * 0.85)); if (now >= plannedAt && (overlap || now >= latestAt)) { @@ -626,6 +671,7 @@ export function collectStagingMixArtifacts(record: TaskRecord, const time = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? Date.parse(value) : null; return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [], ...(record.stagingMix?.completion ? { completion: record.stagingMix.completion } : {}), + ...(record.stagingMix?.afterEndFollowUps?.length ? { afterEndFollowUps: record.stagingMix.afterEndFollowUps } : {}), ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null, createdAt: time(row.createdAt) })) } : { commentsUnreadable: true }) }; diff --git a/tests/runner-e2e/controller-database.test.ts b/tests/runner-e2e/controller-database.test.ts index 72e9360c9a2..5685812f6d6 100644 --- a/tests/runner-e2e/controller-database.test.ts +++ b/tests/runner-e2e/controller-database.test.ts @@ -60,6 +60,32 @@ it("keeps an unmigrated fixture database alive when its serving client dies", as } }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); +it("finishes cleanup when the database already stopped because a group signal reached it", async () => { + const root = await createRunnerE2ETemporaryRoot(); + const configPath = path.join(root, "instance", "config.json"); + let database: Awaited> | undefined; + try { + await (await prepareRunnerE2EServerConfig({ temporaryRoot: root, configPath, serverPort: 3100 }))!.close(); + const dataDirectory = JSON.parse(await readFile(configPath, "utf8")).database.embeddedPostgresDataDir as string; + database = await prepareControllerDatabase(configPath, root); + const pid = Number((await readFile(path.join(dataDirectory, "postmaster.pid"), "utf8")).split("\n")[0]); + process.kill(pid, "SIGINT"); + const alive = () => { try { process.kill(pid, 0); return true; } catch { return false; } }; + const deadline = Date.now() + 30_000; + while (alive() && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 50)); + expect(alive()).toBe(false); + const outcome = await Promise.race([ + database.cleanup().then(() => "cleaned"), + new Promise(resolve => setTimeout(resolve, 10_000, "hung")), + ]); + database = undefined; + expect(outcome).toBe("cleaned"); + } finally { + await database?.cleanup(); + await rm(root, { recursive: true, force: true }); + } +}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + it("rejects database directories outside the fixture and leaves external databases alone", async () => { const root = await createRunnerE2ETemporaryRoot(); const configPath = path.join(root, "config.json"); diff --git a/tests/runner-e2e/controller-database.ts b/tests/runner-e2e/controller-database.ts index eafd12d0c6e..a4c39766f8e 100644 --- a/tests/runner-e2e/controller-database.ts +++ b/tests/runner-e2e/controller-database.ts @@ -1,5 +1,6 @@ /** Qualification owns Postgres outside the server process tree. No migrations * run here: each serving artifact must apply its own schema on startup. */ +import type { ChildProcess } from "node:child_process"; import { existsSync } from "node:fs"; import { readFile, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; @@ -35,7 +36,10 @@ export async function prepareControllerDatabase(configPath: string, temporaryRoo connectionString: `${base}/paperclip` }); config.database = { ...config.database, mode: "postgres", connectionString: `${base}/paperclip` }; await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 }); - return { cleanup: async () => { await database.stop(); } }; + const postgres = (database as unknown as { process?: ChildProcess }).process; + let exited = !postgres || postgres.exitCode !== null || postgres.signalCode !== null; + postgres?.once("exit", () => { exited = true; }); + return { cleanup: async () => { if (!exited) await database.stop(); } }; } catch (error) { if (started) await database.stop().catch(() => undefined); throw new Error(`Qualification database startup failed: ${logs.getRecentLogs().join(" | ")}`, { cause: error });