From 6be8ef50d1cbc379b406e29c92d43576074b896c Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 22:27:07 -0600 Subject: [PATCH] fix(evals): scope negated skill-update completion claims Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../integrator-skill-update.test.ts | 13 +++++++++++++ .../chat-catalog-domains/integrator-skill-update.ts | 8 ++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index dcaf5f0aec3..0fce4bf113f 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5219,7 +5219,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.integrator.v2", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts index bd354ab2e12..ddd18685f36 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts @@ -33,6 +33,19 @@ describe("genuine immutable public skill update qualification", () => { expect(gradeIntegratorSkillUpdate(valid(bytes.old, bytes.next)).passed).toBe(true); expect(gradeIntegratorSkillUpdate(undefined).passed).toBe(false); const natural = valid(bytes.old, bytes.next); natural.reply = `${natural.state.reviewCheckpoint.reply}\nThe accessibility skill is now on revision 0.5; our local fork is preserved.`; expect(gradeIntegratorSkillUpdate(natural).passed).toBe(true); }); + it("accepts a truthful not-applied review but rejects a later positive claim or changed saved version", () => { + const truthful = valid(bytes.old, bytes.next); + truthful.state.reviewCheckpoint.reply = truthful.state.reviewCheckpoint.reply.replace("I have not applied it.", "No update applied yet."); + truthful.reply = `${truthful.state.reviewCheckpoint.reply}\nI updated the accessibility skill to the reviewed source version. The local copy is unchanged.`; + expect(gradeIntegratorSkillUpdate(truthful).passed).toBe(true); + const contradictory = structuredClone(truthful); + contradictory.state.reviewCheckpoint.reply += " The update is applied."; + expect(gradeIntegratorSkillUpdate(contradictory).checks.find(check => check.id === "review-diff-before-update")?.passed).toBe(false); + const changed = structuredClone(truthful); + changed.state.reviewCheckpoint.upstream.currentVersionId = "premature"; + expect(gradeIntegratorSkillUpdate(changed).checks.find(check => check.id === "review-diff-before-update")?.passed).toBe(false); + }); + it("rejects premature apply, listing-only review, stale installation and overwritten local work", () => { const early = valid(bytes.old, bytes.next); early.state.reviewCheckpoint.upstream.currentVersionId = "new"; expect(gradeIntegratorSkillUpdate(early).passed).toBe(false); const listing = valid(bytes.old, bytes.next); listing.state.reviewCheckpoint.reply = "Current installed 0.4. Latest 0.5 is available."; expect(gradeIntegratorSkillUpdate(listing).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts index e6e06b68873..8a69d9f9b2d 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts @@ -44,13 +44,17 @@ function substantiveDiff(reply: string) { const clutter = clauses.some(line => /clutter|overus|redundan|crowd|overload|breath|spacing/i.test(line) && /layout|pattern|control|design|interface|element/i.test(line)); return [recurring, forms, clutter].filter(Boolean).length >= 2; } +function claimsAppliedUpdate(reply: string) { + return [...reply.matchAll(/\b(?:I (?:have )?(?:updated|applied|installed)|update (?:is |has been )?(?:applied|installed|complete))\b/gi)] + .some(match => !/^update /i.test(match[0]) || !/\bno\s+$/i.test(reply.slice(0, match.index))); +} function previewCorrect(o: UpdateObservation) { const before = o.state.upstream as CompanySkillDetail; return o.upstream.currentVersionId === before.currentVersionId && o.upstream.sourceRef === accessibilityUpdatePins.oldCommit && hash(o.upstream.markdown) === accessibilityUpdatePins.oldSha256 && o.source.lastScanCommit === accessibilityUpdatePins.oldCommit && o.source.trackingRef === accessibilityUpdatePins.newCommit && o.source.automaticUpdates === false && o.updateStatus.supported && o.updateStatus.hasUpdate && o.updateStatus.currentRef === accessibilityUpdatePins.oldCommit && o.updateStatus.latestRef === accessibilityUpdatePins.newCommit && (o.reply.includes(accessibilityUpdatePins.oldCommit.slice(0, 7)) || /(?:installed|current|currently)[^\n]{0,80}\b0\.4\b|\b0\.4\b[^\n]{0,80}(?:installed|current)/i.test(o.reply)) - && substantiveDiff(o.reply) && !/\b(?:I (?:have )?(?:updated|applied|installed)|update (?:is |has been )?(?:applied|installed|complete))\b/i.test(o.reply); + && substantiveDiff(o.reply) && !claimsAppliedUpdate(o.reply); } export function gradeIntegratorSkillUpdate(value: unknown) { const c = checks(); const o = updateObservation(value); c.add("real-tracked-source-and-saved-versions", !!o, "Requires actual public GitHub source tracking, installed snapshot versions and preserved editable fork."); if (!o) return c.result(); @@ -93,7 +97,7 @@ export const integratorSkillUpdateCases = [integratorDefinition("skills", "skill const selected = await saveSkillConversationContext(ctx, { description: `Selected upstream accessibility skill: ${upstream.name} (${upstream.id}), tracked source ${source.id}. Installed source revision ${upstream.sourceRef}; tracking reference ${source.trackingRef}. Separate editable local copy: ${localFork.name} (${localFork.id}).` }); return { upstream, localFork, source, factualContextAttachment: selected.factualContextAttachment, localVersions: await readSkillVersionHistory(ctx.input.api, ctx.input.fixtures.company.id, localFork.id, localFork.currentVersionId) }; }, observeUpdate, gradeIntegratorSkillUpdate, async (ctx, state) => { const skill = state.upstream as CompanySkillDetail; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`, skill.name); }, -{ authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000, +{ oracleVersion: "paperclip.chat-catalog.integrator.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000, sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-update.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"], async act(ctx, state) { updateReceipts.delete(ctx);