From 94c475180341e82c5ce13b3d8eae96fcb2879fe0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 17:54:36 -0600 Subject: [PATCH] test(restart-chaos): count staging-mix completions only from DONE, and attribute runs after DONE Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/README.md | 7 ++- tests/restart-chaos/staging-mix-retry.test.ts | 59 ++++++++++++++++--- tests/restart-chaos/staging-mix-retry.ts | 54 ++++++++++++++++- tests/restart-chaos/staging-mix.ts | 19 ++++-- 4 files changed, 124 insertions(+), 15 deletions(-) diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index ac4025b0874..360a09b034e 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1555,6 +1555,11 @@ worked when the clicked task meets the same completion rule. The banner list fol `GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason, `agentsWithStalledSlots` and `openTasks`. +The summary also lists every run that started after a task's DONE, under "Runs started after the task's DONE" +(`extraTurns` in the JSON). The DONE time is the first bare `DONE-` comment, or else the task's completion time. +Each line gives the run's wake reason, source, trigger and requesting actor, taken from that task's saved wake +evidence. + **Grade.** The usual R1–R6, U1–U5 and T rules apply. A needed Retry still fails R3. The T-oracle checks the result, not the status: @@ -1565,7 +1570,7 @@ not the status: | `rounds_once` | a round line appears twice (a tool ran again) | | `done_comment_posted` | no comment's whole body is `DONE-`; a summary that quotes the marker does not count | | `done_comment_once` | that comment was posted more than once | -| `completion_once` | more than one run posted a comment that mentions `DONE-` (a duplicate completion), or the comments could not be read | +| `completion_once` | more than one run completed the task (a duplicate completion), or the comments could not be read. A run completes the task when it posts the bare `DONE-` comment, or a comment that mentions the marker at or after the first one. A progress or blocked comment that says it will post the marker later is not a completion | | `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) | | `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line | | `subtask__done`, `parent_read_subtask_` | a subtask is missing, not done or its ledger fails, or the parent did not report its last hash | diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 3d52bfd1a09..2009b91cf3f 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it } from "vitest"; import { parseCommand } from "./config.js"; import { collectStagingMixArtifacts, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js"; import { - bannerSink, issueView, readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, - type RetryStageObserver, + bannerSink, extraTurnLine, firstDoneAt, issueView, readAssignedWorkInvariant, retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle, + stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, } from "./staging-mix-retry.js"; import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js"; @@ -158,9 +158,10 @@ describe("staging-mix Retry policy", () => { describe("staging-mix completion comments", () => { const nonce = "b3d70e77"; - const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, + const base = (comments: Array<{ body: string; runId: string | null; at?: string }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null, - stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId }))) }); + stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId, + ...(comment.at ? { createdAt: comment.at } : {}) }))) }); const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!; it("counts only the DONE comment itself, not a closing summary that quotes it", () => { @@ -182,13 +183,57 @@ describe("staging-mix completion comments", () => { expect(check(unread, "done_comment_posted").ok).toBe(true); }); + it("does not count a blocked run's promise to post DONE later as a completion (w-20261112T223803-s7101 t001)", () => { + const comments = [ + { body: "Prepared Parent ledger log for this response.", runId: "37724b59-4e6a", at: "2026-11-12T23:12:54.696Z" }, + { body: doneMarker(nonce), runId: "37724b59-4e6a", at: "2026-11-12T23:12:44.587Z" }, + { body: "Blocked report accepted. Summary of progress so far:\n\n- Completed all 4 parent rounds.\n\nParent now waits on both subtasks to finish. " + + `When they complete, I'll read each subtask's ledger hash, save this task's \`ledger\` document, post \`${doneMarker(nonce)}\`, and mark the task done.`, + runId: "fcc55827-ab78", at: "2026-11-12T22:56:04.145Z" }, + ]; + expect(check(base(comments), "completion_once")).toMatchObject({ ok: true, detail: "runs=1" }); + expect(check(base(comments), "done_comment_once")).toMatchObject({ ok: true, detail: "comments=1" }); + expect(check(base(comments), "done_comment_posted").ok).toBe(true); + }); + it("flags a completion summary repeated by a different run as a duplicate completion", () => { const comments = [ - { body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce" }, - { body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13" }, - { body: doneMarker(nonce), runId: "13de31be-fb13" }, + { body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce", at: "2026-11-12T18:10:00.000Z" }, + { body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13", at: "2026-11-12T18:05:01.000Z" }, + { body: doneMarker(nonce), runId: "13de31be-fb13", at: "2026-11-12T18:05:00.000Z" }, ]; expect(check(base(comments), "done_comment_once").ok).toBe(true); expect(check(base(comments), "completion_once")).toMatchObject({ ok: false, detail: "duplicate completion: runs 413ddcf2, 13de31be each reported completion" }); + expect(check(base([...comments.slice(1), { body: doneMarker(nonce), runId: "run-z", at: "2026-11-12T18:20:00.000Z" }]), "completion_once").ok).toBe(false); + }); +}); + +describe("staging-mix runs after DONE", () => { + const evidence = { + runs: [{ runId: "6b825e48-aaaa", createdAt: 1_000, invocationSource: "assignment", triggerDetail: null, wakeupRequestId: "w1" }, + { runId: "37724b59-bbbb", createdAt: 5_000, invocationSource: "automation", triggerDetail: null, wakeupRequestId: "w2" }, + { runId: "91f0c3d2-cccc", createdAt: 9_000, invocationSource: "automation", triggerDetail: "system", wakeupRequestId: "w3" }, + { runId: "aa11bb22-dddd", createdAt: 9_500, invocationSource: "on_demand", triggerDetail: "manual", wakeupRequestId: null }], + wakes: [{ id: "w1", reason: "issue_assigned", source: "assignment", triggerDetail: null, requestedByActorType: "user", requestedByActorId: "u1" }, + { id: "w2", reason: "issue_children_completed", source: "automation", triggerDetail: null, requestedByActorType: "system", requestedByActorId: null }, + { id: "w3", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: "sweep" }], + }; + + it("lists each run that started after the task's DONE with its wake reason and actor", () => { + const turns = runsAfterDone("t001", evidence, 8_000); + expect(turns.map(turn => [turn.runId, turn.wakeReason, turn.actorType, turn.actorId])).toEqual([ + ["91f0c3d2-cccc", "issue_recovery_action_restored", "system", "sweep"], ["aa11bb22-dddd", null, null, null]]); + expect(extraTurnLine(turns[0]!)).toBe(`- t001: run 91f0c3d2 started ${new Date(9_000).toISOString()} after DONE; wake issue_recovery_action_restored (automation/system) by system sweep`); + expect(extraTurnLine(turns[1]!)).toBe(`- t001: run aa11bb22 started ${new Date(9_500).toISOString()} after DONE; wake unknown (on_demand/manual) by unknown`); + expect(runsAfterDone("t001", evidence, null)).toEqual([]); + expect(runsAfterDone("t001", null, 1)).toEqual([]); + }); + + it("anchors DONE on the first DONE comment, else on the issue's completion time", () => { + const comments = [{ body: "will post DONE-b3d70e77 later", authorAgentId: "a", createdAt: "2026-11-12T22:56:04.145Z" }, + { body: "DONE-b3d70e77", authorAgentId: "a", createdAt: "2026-11-12T23:12:44.587Z" }, { body: "DONE-b3d70e77", authorAgentId: null, createdAt: "2026-11-12T20:00:00.000Z" }]; + expect(firstDoneAt(comments, "b3d70e77", "2026-11-12T23:13:05.527Z")).toBe(Date.parse("2026-11-12T23:12:44.587Z")); + expect(firstDoneAt(comments.slice(0, 1), "b3d70e77", "2026-11-12T23:13:05.527Z")).toBe(Date.parse("2026-11-12T23:13:05.527Z")); + expect(firstDoneAt(null, "b3d70e77", null)).toBeNull(); }); }); diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index 78d51694ee5..1fb08093539 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -2,7 +2,7 @@ import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import type { ChaosApi } from "./api.js"; -import { bannerShownMs, DUPLICATE_CHECKS, noteRetryBanner } from "./staging-mix.js"; +import { bannerShownMs, DUPLICATE_CHECKS, isDoneComment, noteRetryBanner } from "./staging-mix.js"; import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js"; type Json = Record; @@ -170,6 +170,48 @@ export function stagingMixHeadline(result: StagingMixResult): string[] { ]; } +export interface ExtraTurn { + taskId: string; + runId: string | null; + createdAt: number | null; + invocationSource: string | null; + wakeReason: string | null; + wakeSource: string | null; + triggerDetail: string | null; + actorType: string | null; + actorId: string | null; +} + +interface WakeEvidenceFile { + runs?: Array<{ runId: string | null; createdAt: number | null; invocationSource: string | null; triggerDetail: string | null; wakeupRequestId: string | null }>; + wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null }>; +} + +export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, doneAt: number | null): ExtraTurn[] { + if (!evidence || doneAt === null) return []; + const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); + return (evidence.runs ?? []).filter(run => run.createdAt !== null && run.createdAt > doneAt).map(run => { + const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined; + return { taskId, runId: run.runId, createdAt: run.createdAt, invocationSource: run.invocationSource, wakeReason: wake?.reason ?? null, + wakeSource: wake?.source ?? null, triggerDetail: wake?.triggerDetail ?? run.triggerDetail, actorType: wake?.requestedByActorType ?? null, + actorId: wake?.requestedByActorId ?? null }; + }); +} + +export function firstDoneAt(comments: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdAt?: unknown }> | null, nonce: string, completedAt: unknown): number | null { + const times = (comments ?? []).filter(comment => comment.authorAgentId && isDoneComment(String(comment.body ?? ""), nonce)) + .map(comment => Date.parse(String(comment.createdAt))).filter(Number.isFinite); + if (times.length) return Math.min(...times); + const completed = typeof completedAt === "string" ? Date.parse(completedAt) : NaN; + return Number.isFinite(completed) ? completed : null; +} + +export function extraTurnLine(turn: ExtraTurn): string { + return `- ${turn.taskId}: run ${turn.runId?.slice(0, 8) ?? "?"} started ${turn.createdAt === null ? "?" : new Date(turn.createdAt).toISOString()} after DONE; ` + + `wake ${turn.wakeReason ?? "unknown"} (${turn.wakeSource ?? turn.invocationSource ?? "?"}${turn.triggerDetail ? `/${turn.triggerDetail}` : ""}) ` + + `by ${turn.actorType ?? "unknown"}${turn.actorId ? ` ${turn.actorId}` : ""}`; +} + export async function readAssignedWorkInvariant(api: Pick, companyId: string): Promise { try { const report = await api.get(`/api/companies/${companyId}/assigned-work-invariant`, 30_000); @@ -185,8 +227,13 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise { const extraClicks = new Set(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string").map(entry => String(entry.issueId))); const tasks: ResultTask[] = []; + const extraTurns: ExtraTurn[] = []; for (const task of input.tasks) { const issue = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null; + const comments = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}/comments`, 10_000).catch(() => null) : null; + const evidence = await readFile(path.join(input.waveDir, "wake-evidence", `${task.plan.taskId}.json`), "utf8") + .then(text => JSON.parse(text) as WakeEvidenceFile).catch(() => null); + extraTurns.push(...runsAfterDone(task.plan.taskId, evidence, firstDoneAt(comments, task.plan.nonce, issue?.completedAt))); const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8") .then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null); tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus, @@ -194,13 +241,14 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick } const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId)); await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1", - headline: stagingMixHeadline(result), ...result }, null, 2)}\n`, { mode: 0o600 }); + headline: stagingMixHeadline(result), ...result, extraTurns }, null, 2)}\n`, { mode: 0o600 }); const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}` : `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`; await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "", ...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`), ...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`), ...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s` - + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 }); + + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), + ...(extraTurns.length ? ["", `Runs started after the task's DONE: ${extraTurns.length}`, ...extraTurns.map(extraTurnLine)] : []), ""].join("\n"), { mode: 0o600 }); return result; } diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index 9e622047fda..d73350ce54c 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -62,7 +62,7 @@ export interface StagingMixArtifacts { followUp: StagingMixRecord["followUp"] | null; followUpErrors: string[]; retries: NonNullable; - comments?: Array<{ body: string; runId: string | null }>; + comments?: Array<{ body: string; runId: string | null; createdAt?: number | null }>; commentsUnreadable?: boolean; } @@ -373,6 +373,14 @@ export function stagingMixDescription(nonce: string, task: StagingMixTask, agent export const followUpRequestId = (issueId: string, nonce: string) => stableRequestId(`followup-${issueId}-${nonce}`); +export function completionRunIds(comments: ReadonlyArray<{ body: string; runId: string | null; createdAt?: number | null }>, nonce: string): string[] { + const doneTimes = comments.filter(comment => isDoneComment(comment.body, nonce) && typeof comment.createdAt === "number").map(comment => comment.createdAt!); + const doneAt = doneTimes.length ? Math.min(...doneTimes) : null; + const completions = comments.filter(comment => isDoneComment(comment.body, nonce) + || (doneAt !== null && typeof comment.createdAt === "number" && comment.createdAt >= doneAt && comment.body.includes(doneMarker(nonce)))); + return [...new Set(completions.flatMap(comment => comment.runId ? [comment.runId] : []))]; +} + export function isDoneComment(body: string, nonce: string): boolean { return body.trim().replace(/\.$/, "").replace(/^`+|`+$/g, "").trim() === doneMarker(nonce); } @@ -412,7 +420,7 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: const own = verifyLedger(nonce, task.rounds, ledger); const comments = artifacts.stagingMix?.comments ?? artifacts.agentComments.map(body => ({ body, runId: null })); const doneComments = comments.filter(comment => isDoneComment(comment.body, nonce)).length; - const completionRuns = [...new Set(comments.filter(comment => comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!))]; + const completionRuns = completionRunIds(comments, nonce); const followUp = artifacts.stagingMix?.followUp ?? null; const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(nonce)); const checks: OracleResult["checks"] = [ @@ -573,8 +581,11 @@ export function bannerShownMs(banner: RetryBanner, now: number): number { return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince); } -export function collectStagingMixArtifacts(record: TaskRecord, rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown }> | null): StagingMixArtifacts { +export function collectStagingMixArtifacts(record: TaskRecord, + rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown; createdAt?: unknown }> | null): StagingMixArtifacts { + const time = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? Date.parse(value) : null; return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [], - ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null })) } + ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), + runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null, createdAt: time(row.createdAt) })) } : { commentsUnreadable: true }) }; }