mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
test(restart-chaos): count staging-mix completions only from DONE, and attribute runs after DONE
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b1ef1f3022
commit
94c4751803
4 files changed
+124
-15
No files matched your search
@@ -1555,6 +1555,11 @@ worked when the clicked task meets the same completion rule. The banner list fol
|
||||
`GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason,
|
||||
`agentsWithStalledSlots` and `openTasks`.
|
||||
|
||||
The summary also lists every run that started after a task's DONE, under "Runs started after the task's DONE"
|
||||
(`extraTurns` in the JSON). The DONE time is the first bare `DONE-<nonce>` comment, or else the task's completion time.
|
||||
Each line gives the run's wake reason, source, trigger and requesting actor, taken from that task's saved wake
|
||||
evidence.
|
||||
|
||||
**Grade.** The usual R1–R6, U1–U5 and T rules apply. A needed Retry still fails R3. The T-oracle checks the result,
|
||||
not the status:
|
||||
|
||||
@@ -1565,7 +1570,7 @@ not the status:
|
||||
| `rounds_once` | a round line appears twice (a tool ran again) |
|
||||
| `done_comment_posted` | no comment's whole body is `DONE-<nonce>`; a summary that quotes the marker does not count |
|
||||
| `done_comment_once` | that comment was posted more than once |
|
||||
| `completion_once` | more than one run posted a comment that mentions `DONE-<nonce>` (a duplicate completion), or the comments could not be read |
|
||||
| `completion_once` | more than one run completed the task (a duplicate completion), or the comments could not be read. A run completes the task when it posts the bare `DONE-<nonce>` comment, or a comment that mentions the marker at or after the first one. A progress or blocked comment that says it will post the marker later is not a completion |
|
||||
| `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) |
|
||||
| `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line |
|
||||
| `subtask_<k>_done`, `parent_read_subtask_<k>` | a subtask is missing, not done or its ledger fails, or the parent did not report its last hash |
|
||||
|
||||
@@ -2,8 +2,8 @@ import { describe, expect, it } from "vitest";
|
||||
import { parseCommand } from "./config.js";
|
||||
import { collectStagingMixArtifacts, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js";
|
||||
import {
|
||||
bannerSink, issueView, readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix,
|
||||
type RetryStageObserver,
|
||||
bannerSink, extraTurnLine, firstDoneAt, issueView, readAssignedWorkInvariant, retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle,
|
||||
stagingMixHeadline, summarizeStagingMix, type RetryStageObserver,
|
||||
} from "./staging-mix-retry.js";
|
||||
import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js";
|
||||
|
||||
@@ -158,9 +158,10 @@ describe("staging-mix Retry policy", () => {
|
||||
|
||||
describe("staging-mix completion comments", () => {
|
||||
const nonce = "b3d70e77";
|
||||
const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) },
|
||||
const base = (comments: Array<{ body: string; runId: string | null; at?: string }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) },
|
||||
agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null,
|
||||
stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId }))) });
|
||||
stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId,
|
||||
...(comment.at ? { createdAt: comment.at } : {}) }))) });
|
||||
const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!;
|
||||
|
||||
it("counts only the DONE comment itself, not a closing summary that quotes it", () => {
|
||||
@@ -182,13 +183,57 @@ describe("staging-mix completion comments", () => {
|
||||
expect(check(unread, "done_comment_posted").ok).toBe(true);
|
||||
});
|
||||
|
||||
it("does not count a blocked run's promise to post DONE later as a completion (w-20261112T223803-s7101 t001)", () => {
|
||||
const comments = [
|
||||
{ body: "Prepared Parent ledger log for this response.", runId: "37724b59-4e6a", at: "2026-11-12T23:12:54.696Z" },
|
||||
{ body: doneMarker(nonce), runId: "37724b59-4e6a", at: "2026-11-12T23:12:44.587Z" },
|
||||
{ body: "Blocked report accepted. Summary of progress so far:\n\n- Completed all 4 parent rounds.\n\nParent now waits on both subtasks to finish. "
|
||||
+ `When they complete, I'll read each subtask's ledger hash, save this task's \`ledger\` document, post \`${doneMarker(nonce)}\`, and mark the task done.`,
|
||||
runId: "fcc55827-ab78", at: "2026-11-12T22:56:04.145Z" },
|
||||
];
|
||||
expect(check(base(comments), "completion_once")).toMatchObject({ ok: true, detail: "runs=1" });
|
||||
expect(check(base(comments), "done_comment_once")).toMatchObject({ ok: true, detail: "comments=1" });
|
||||
expect(check(base(comments), "done_comment_posted").ok).toBe(true);
|
||||
});
|
||||
|
||||
it("flags a completion summary repeated by a different run as a duplicate completion", () => {
|
||||
const comments = [
|
||||
{ body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce" },
|
||||
{ body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13" },
|
||||
{ body: doneMarker(nonce), runId: "13de31be-fb13" },
|
||||
{ body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce", at: "2026-11-12T18:10:00.000Z" },
|
||||
{ body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13", at: "2026-11-12T18:05:01.000Z" },
|
||||
{ body: doneMarker(nonce), runId: "13de31be-fb13", at: "2026-11-12T18:05:00.000Z" },
|
||||
];
|
||||
expect(check(base(comments), "done_comment_once").ok).toBe(true);
|
||||
expect(check(base(comments), "completion_once")).toMatchObject({ ok: false, detail: "duplicate completion: runs 413ddcf2, 13de31be each reported completion" });
|
||||
expect(check(base([...comments.slice(1), { body: doneMarker(nonce), runId: "run-z", at: "2026-11-12T18:20:00.000Z" }]), "completion_once").ok).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("staging-mix runs after DONE", () => {
|
||||
const evidence = {
|
||||
runs: [{ runId: "6b825e48-aaaa", createdAt: 1_000, invocationSource: "assignment", triggerDetail: null, wakeupRequestId: "w1" },
|
||||
{ runId: "37724b59-bbbb", createdAt: 5_000, invocationSource: "automation", triggerDetail: null, wakeupRequestId: "w2" },
|
||||
{ runId: "91f0c3d2-cccc", createdAt: 9_000, invocationSource: "automation", triggerDetail: "system", wakeupRequestId: "w3" },
|
||||
{ runId: "aa11bb22-dddd", createdAt: 9_500, invocationSource: "on_demand", triggerDetail: "manual", wakeupRequestId: null }],
|
||||
wakes: [{ id: "w1", reason: "issue_assigned", source: "assignment", triggerDetail: null, requestedByActorType: "user", requestedByActorId: "u1" },
|
||||
{ id: "w2", reason: "issue_children_completed", source: "automation", triggerDetail: null, requestedByActorType: "system", requestedByActorId: null },
|
||||
{ id: "w3", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: "sweep" }],
|
||||
};
|
||||
|
||||
it("lists each run that started after the task's DONE with its wake reason and actor", () => {
|
||||
const turns = runsAfterDone("t001", evidence, 8_000);
|
||||
expect(turns.map(turn => [turn.runId, turn.wakeReason, turn.actorType, turn.actorId])).toEqual([
|
||||
["91f0c3d2-cccc", "issue_recovery_action_restored", "system", "sweep"], ["aa11bb22-dddd", null, null, null]]);
|
||||
expect(extraTurnLine(turns[0]!)).toBe(`- t001: run 91f0c3d2 started ${new Date(9_000).toISOString()} after DONE; wake issue_recovery_action_restored (automation/system) by system sweep`);
|
||||
expect(extraTurnLine(turns[1]!)).toBe(`- t001: run aa11bb22 started ${new Date(9_500).toISOString()} after DONE; wake unknown (on_demand/manual) by unknown`);
|
||||
expect(runsAfterDone("t001", evidence, null)).toEqual([]);
|
||||
expect(runsAfterDone("t001", null, 1)).toEqual([]);
|
||||
});
|
||||
|
||||
it("anchors DONE on the first DONE comment, else on the issue's completion time", () => {
|
||||
const comments = [{ body: "will post DONE-b3d70e77 later", authorAgentId: "a", createdAt: "2026-11-12T22:56:04.145Z" },
|
||||
{ body: "DONE-b3d70e77", authorAgentId: "a", createdAt: "2026-11-12T23:12:44.587Z" }, { body: "DONE-b3d70e77", authorAgentId: null, createdAt: "2026-11-12T20:00:00.000Z" }];
|
||||
expect(firstDoneAt(comments, "b3d70e77", "2026-11-12T23:13:05.527Z")).toBe(Date.parse("2026-11-12T23:12:44.587Z"));
|
||||
expect(firstDoneAt(comments.slice(0, 1), "b3d70e77", "2026-11-12T23:13:05.527Z")).toBe(Date.parse("2026-11-12T23:13:05.527Z"));
|
||||
expect(firstDoneAt(null, "b3d70e77", null)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import type { ChaosApi } from "./api.js";
|
||||
import { bannerShownMs, DUPLICATE_CHECKS, noteRetryBanner } from "./staging-mix.js";
|
||||
import { bannerShownMs, DUPLICATE_CHECKS, isDoneComment, noteRetryBanner } from "./staging-mix.js";
|
||||
import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js";
|
||||
|
||||
type Json = Record<string, any>;
|
||||
@@ -170,6 +170,48 @@ export function stagingMixHeadline(result: StagingMixResult): string[] {
|
||||
];
|
||||
}
|
||||
|
||||
export interface ExtraTurn {
|
||||
taskId: string;
|
||||
runId: string | null;
|
||||
createdAt: number | null;
|
||||
invocationSource: string | null;
|
||||
wakeReason: string | null;
|
||||
wakeSource: string | null;
|
||||
triggerDetail: string | null;
|
||||
actorType: string | null;
|
||||
actorId: string | null;
|
||||
}
|
||||
|
||||
interface WakeEvidenceFile {
|
||||
runs?: Array<{ runId: string | null; createdAt: number | null; invocationSource: string | null; triggerDetail: string | null; wakeupRequestId: string | null }>;
|
||||
wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null }>;
|
||||
}
|
||||
|
||||
export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, doneAt: number | null): ExtraTurn[] {
|
||||
if (!evidence || doneAt === null) return [];
|
||||
const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake]));
|
||||
return (evidence.runs ?? []).filter(run => run.createdAt !== null && run.createdAt > doneAt).map(run => {
|
||||
const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined;
|
||||
return { taskId, runId: run.runId, createdAt: run.createdAt, invocationSource: run.invocationSource, wakeReason: wake?.reason ?? null,
|
||||
wakeSource: wake?.source ?? null, triggerDetail: wake?.triggerDetail ?? run.triggerDetail, actorType: wake?.requestedByActorType ?? null,
|
||||
actorId: wake?.requestedByActorId ?? null };
|
||||
});
|
||||
}
|
||||
|
||||
export function firstDoneAt(comments: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdAt?: unknown }> | null, nonce: string, completedAt: unknown): number | null {
|
||||
const times = (comments ?? []).filter(comment => comment.authorAgentId && isDoneComment(String(comment.body ?? ""), nonce))
|
||||
.map(comment => Date.parse(String(comment.createdAt))).filter(Number.isFinite);
|
||||
if (times.length) return Math.min(...times);
|
||||
const completed = typeof completedAt === "string" ? Date.parse(completedAt) : NaN;
|
||||
return Number.isFinite(completed) ? completed : null;
|
||||
}
|
||||
|
||||
export function extraTurnLine(turn: ExtraTurn): string {
|
||||
return `- ${turn.taskId}: run ${turn.runId?.slice(0, 8) ?? "?"} started ${turn.createdAt === null ? "?" : new Date(turn.createdAt).toISOString()} after DONE; `
|
||||
+ `wake ${turn.wakeReason ?? "unknown"} (${turn.wakeSource ?? turn.invocationSource ?? "?"}${turn.triggerDetail ? `/${turn.triggerDetail}` : ""}) `
|
||||
+ `by ${turn.actorType ?? "unknown"}${turn.actorId ? ` ${turn.actorId}` : ""}`;
|
||||
}
|
||||
|
||||
export async function readAssignedWorkInvariant(api: Pick<ChaosApi, "get">, companyId: string): Promise<StagingMixResult["invariant"]> {
|
||||
try {
|
||||
const report = await api.get<Json>(`/api/companies/${companyId}/assigned-work-invariant`, 30_000);
|
||||
@@ -185,8 +227,13 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
|
||||
tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise<StagingMixResult> {
|
||||
const extraClicks = new Set(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string").map(entry => String(entry.issueId)));
|
||||
const tasks: ResultTask[] = [];
|
||||
const extraTurns: ExtraTurn[] = [];
|
||||
for (const task of input.tasks) {
|
||||
const issue = task.record.issueId ? await input.api.get<Json>(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null;
|
||||
const comments = task.record.issueId ? await input.api.get<Json[]>(`/api/issues/${task.record.issueId}/comments`, 10_000).catch(() => null) : null;
|
||||
const evidence = await readFile(path.join(input.waveDir, "wake-evidence", `${task.plan.taskId}.json`), "utf8")
|
||||
.then(text => JSON.parse(text) as WakeEvidenceFile).catch(() => null);
|
||||
extraTurns.push(...runsAfterDone(task.plan.taskId, evidence, firstDoneAt(comments, task.plan.nonce, issue?.completedAt)));
|
||||
const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8")
|
||||
.then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null);
|
||||
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus,
|
||||
@@ -194,13 +241,14 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
|
||||
}
|
||||
const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId));
|
||||
await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1",
|
||||
headline: stagingMixHeadline(result), ...result }, null, 2)}\n`, { mode: 0o600 });
|
||||
headline: stagingMixHeadline(result), ...result, extraTurns }, null, 2)}\n`, { mode: 0o600 });
|
||||
const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}`
|
||||
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
|
||||
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
|
||||
...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`),
|
||||
...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`),
|
||||
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
|
||||
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
|
||||
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`),
|
||||
...(extraTurns.length ? ["", `Runs started after the task's DONE: ${extraTurns.length}`, ...extraTurns.map(extraTurnLine)] : []), ""].join("\n"), { mode: 0o600 });
|
||||
return result;
|
||||
}
|
||||
@@ -62,7 +62,7 @@ export interface StagingMixArtifacts {
|
||||
followUp: StagingMixRecord["followUp"] | null;
|
||||
followUpErrors: string[];
|
||||
retries: NonNullable<StagingMixRecord["retries"]>;
|
||||
comments?: Array<{ body: string; runId: string | null }>;
|
||||
comments?: Array<{ body: string; runId: string | null; createdAt?: number | null }>;
|
||||
commentsUnreadable?: boolean;
|
||||
}
|
||||
|
||||
@@ -373,6 +373,14 @@ export function stagingMixDescription(nonce: string, task: StagingMixTask, agent
|
||||
|
||||
export const followUpRequestId = (issueId: string, nonce: string) => stableRequestId(`followup-${issueId}-${nonce}`);
|
||||
|
||||
export function completionRunIds(comments: ReadonlyArray<{ body: string; runId: string | null; createdAt?: number | null }>, nonce: string): string[] {
|
||||
const doneTimes = comments.filter(comment => isDoneComment(comment.body, nonce) && typeof comment.createdAt === "number").map(comment => comment.createdAt!);
|
||||
const doneAt = doneTimes.length ? Math.min(...doneTimes) : null;
|
||||
const completions = comments.filter(comment => isDoneComment(comment.body, nonce)
|
||||
|| (doneAt !== null && typeof comment.createdAt === "number" && comment.createdAt >= doneAt && comment.body.includes(doneMarker(nonce))));
|
||||
return [...new Set(completions.flatMap(comment => comment.runId ? [comment.runId] : []))];
|
||||
}
|
||||
|
||||
export function isDoneComment(body: string, nonce: string): boolean {
|
||||
return body.trim().replace(/\.$/, "").replace(/^`+|`+$/g, "").trim() === doneMarker(nonce);
|
||||
}
|
||||
@@ -412,7 +420,7 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts:
|
||||
const own = verifyLedger(nonce, task.rounds, ledger);
|
||||
const comments = artifacts.stagingMix?.comments ?? artifacts.agentComments.map(body => ({ body, runId: null }));
|
||||
const doneComments = comments.filter(comment => isDoneComment(comment.body, nonce)).length;
|
||||
const completionRuns = [...new Set(comments.filter(comment => comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!))];
|
||||
const completionRuns = completionRunIds(comments, nonce);
|
||||
const followUp = artifacts.stagingMix?.followUp ?? null;
|
||||
const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(nonce));
|
||||
const checks: OracleResult["checks"] = [
|
||||
@@ -573,8 +581,11 @@ export function bannerShownMs(banner: RetryBanner, now: number): number {
|
||||
return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince);
|
||||
}
|
||||
|
||||
export function collectStagingMixArtifacts(record: TaskRecord, rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown }> | null): StagingMixArtifacts {
|
||||
export function collectStagingMixArtifacts(record: TaskRecord,
|
||||
rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown; createdAt?: unknown }> | null): StagingMixArtifacts {
|
||||
const time = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? Date.parse(value) : null;
|
||||
return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [],
|
||||
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null })) }
|
||||
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""),
|
||||
runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null, createdAt: time(row.createdAt) })) }
|
||||
: { commentsUnreadable: true }) };
|
||||
}
|
||||
Reference in new issue
Block a user