test(evals): prove discard-queued-message rejects a prompt without the direction

Add negative cases: a prompt carrying only other comments, raw or JSON-escaped,
a prefix of the direction, and an empty body must not count as delivery.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-08 03:34:17 -06:00
1 parent d3102c577c
commit d3b353c80a
2 files changed
+14 -1

No files matched your search

@@ -56,3 +56,16 @@ it("reads task links saved in the reply even before they render", () => {
expect(replyLinkHrefs("Discard it on [RUN-1](/RUN/issues/RUN-1).", [])).toEqual(["/RUN/issues/RUN-1"]);
expect(replyLinkHrefs("Discard it on RUN-1.", [])).toEqual([]);
});
it("rejects a native prompt that carries other comments but not the direction comment, raw or JSON-escaped", () => {
const body = "New direction.\n\n- Remove the contact section.\n- Keep the font guidance.";
const other = "I have changed my mind: add a contact section.\nKeep the font requirement.";
const prompt = `{"messages":[{"body":${JSON.stringify(other)}}]}\n${other}`;
expect(promptCarries(prompt, body)).toBe(false);
// A prefix of the direction is not the direction.
expect(promptCarries(`{"body":${JSON.stringify("New direction.\n\n- Remove the contact section.")}}`, body)).toBe(false);
expect(promptCarries(prompt, "")).toBe(false);
});
it("rejects an unbound later worker in the grade when its prompt lacks the direction", () => {
const o = valid(); o.state.cancelDirectionRunBound = promptCarries(`{"body":${JSON.stringify("unrelated")}}`, "Remove the contact section.\nKeep fonts.");
expect(gradeDiscard(o).passed).toBe(false);
});
@@ -15,7 +15,7 @@ import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime
const contactText = "Also add a contact section to the website.";
/** Native prompts embed task comments as JSON, so a multi-line body appears with escaped newlines. */
export function promptCarries(prompt: string | undefined, body: string) {
return !!prompt && (prompt.includes(body) || prompt.includes(JSON.stringify(body).slice(1, -1)));
return !!prompt && body.trim().length > 0 && (prompt.includes(body) || prompt.includes(JSON.stringify(body).slice(1, -1)));
}
/** Links the reply saved in Markdown, plus links already rendered when the reply was observed. */
export function replyLinkHrefs(reply: string, rendered: readonly string[]) {