test(evals): observe skill updates at semantic checkpoints

Poll local native readiness before reading GitHub. Preserve real provider reads at preview and installation, reusing only an unchanged exact immutable-source checkpoint during grade settling.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/integrator-skill-update-checkpoint.test.ts

@@ -0,0 +1,70 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { readFileSync } from "node:fs";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+const port = vi.hoisted(() => ({ observe: () => ({} as Record<string, unknown>), versions: (_id: string): unknown[] => [], send: () => {} }));
+vi.mock("./integrator-support.js", async original => ({ ...await original<typeof import("./integrator-support.js")>(), observeInternal: async () => port.observe() }));
+vi.mock("./integrator-skill-versions.js", () => ({ readSkillVersionHistory: async (_api: unknown, _company: string, id: string) => port.versions(id) }));
+vi.mock("../chat-flow.js", () => ({ sendChatMessage: async () => port.send() }));
+vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value }));
+import { SkillSourceUpdateReadError } from "../api.js";
+import { integratorSkillUpdateCases, gradeIntegratorSkillUpdate, accessibilityUpdatePins as pins } from "./integrator-skill-update.js";
+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });
+const old = readFileSync(new URL("../fixtures/accessibility-source/0.4.SKILL.md", import.meta.url), "utf8");
+const next = readFileSync(new URL("../fixtures/accessibility-source/0.5.SKILL.md", import.meta.url), "utf8");
+function fixture(failAt?: number) {
+  vi.useFakeTimers(); vi.setSystemTime(10_000);
+  let phase = 0, changed = 0, wrongVersion = false, missingHistory = false, earlyCalls = 0, mutable = false, foreignRepo = false, noInstall = false;
+  const ready = () => Date.now() >= (phase ? 19_000 : 15_000), installed = () => phase === 1 && ready() && !noInstall;
+  const upstream = () => ({ id: "up", companyId: "company", editable: false, sourceRef: installed() ? pins.newCommit : pins.oldCommit,
+    currentVersionId: installed() ? "new" : "old", markdown: installed() && !wrongVersion ? next : old });
+  const local = { id: "fork", companyId: "company", editable: true, forkedFromSkillId: "up", currentVersionId: "local", markdown: old + "\nLocal notes" };
+  const versions = () => missingHistory ? [] : [{ id: "old", fileInventory: [{ path: "SKILL.md", content: old }] }, ...(installed() ? [{ id: "new", fileInventory: [{ path: "SKILL.md", content: next }] }] : [])];
+  port.versions = id => id === "up" ? versions() : [{ id: "local" }];
+  const source = () => ({ id: "source", companyId: "company", fullName: foreignRepo ? "foreign/repository" : "KreerC/ACCESSIBILITY.md", revision: changed, lastScanCommit: installed() ? pins.newCommit : pins.oldCommit, trackingRef: mutable ? "main" : pins.newCommit, automaticUpdates: false });
+  const preview = "The currently installed version is 0.4. The new revision adds continuous accessibility regression tests and splits long forms into smaller steps. I have not applied it.";
+  const state: Record<string, unknown> = { upstream: upstream(), localFork: local, source: source(), localVersions: [{ id: "local" }] };
+  const runs = () => Array.from({ length: phase ? 2 : 1 }, (_, i) => ({ id: "native-" + i, status: i === phase && !ready() ? "running" : "succeeded" }));
+  port.observe = () => ({ scenarioId: "skills.review-update", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state, runs: runs(),
+    reply: installed() ? preview + "\nI updated the accessibility skill; the local copy is unchanged." : preview,
+    tasks: [], projects: [], goals: [], documents: [], revisions: [], comments: [], humanQueries: [], links: [],
+    activity: installed() ? [{ agentId: "ceo", action: "company.skill_source_refreshed", entityId: "source" }] : [] });
+  port.send = () => { phase = 1; };
+  const provider = vi.fn(async () => {
+    if (!ready()) earlyCalls++;
+    if (failAt === phase) throw new SkillSourceUpdateReadError(422, "skill_source_github_rate_limited", 403);
+    return { supported: true, hasUpdate: !installed(), currentRef: installed() ? pins.newCommit : pins.oldCommit, latestRef: pins.newCommit, trackingRef: pins.newCommit };
+  });
+  const localReads: string[] = [];
+  const ctx = { observationDeadlineAt: 520_000, input: { fixtures: { company: { id: "company" } }, page: {}, evidence: async () => {}, api: {
+    get: async (path: string) => { localReads.push(path); if (path.endsWith("/skills/up")) return upstream(); if (path.endsWith("/skills/fork")) return local;
+      if (path.endsWith("/skill-sources/source")) return source(); throw Error("Unexpected nonlocal read"); }, getSkillSourceUpdateStatus: provider,
+  } }, issue: () => ({ conversationState: ready() ? "waiting" : "active" }), refreshIssue: async () => {}, allRuns: async () => runs() } as unknown as ChatCatalogContext;
+  const definition = integratorSkillUpdateCases[0]!;
+  async function act() { const pending = definition.act!(ctx, state).then(() => undefined, (error: Error) => error); await vi.advanceTimersByTimeAsync(10_000); return pending; }
+  return { act, observe: () => definition.observe(ctx, state), provider, earlyCalls: () => earlyCalls, localReads, changeSource: () => { changed++; },
+    wrongVersion: () => { wrongVersion = true; }, missingHistory: () => { missingHistory = true; }, mutable: () => { mutable = true; }, foreignRepo: () => { foreignRepo = true; }, noInstall: () => { noInstall = true; } };
+}
+it("actual caller makes no provider request while native work is running and one genuine request at each settled semantic checkpoint", async () => {
+  const f = fixture(); expect(await f.act()).toBeUndefined(); expect(f.earlyCalls()).toBe(0); expect(f.provider).toHaveBeenCalledTimes(2);
+  for (let i = 0; i < 5; i++) expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(true);
+  expect(f.provider).toHaveBeenCalledTimes(2); expect(f.localReads.length).toBeGreaterThan(6);
+});
+it.each([0, 1])("propagates real HTTP422/provider403 at semantic checkpoint%s without caching success", async phase => {
+  const f = fixture(phase); expect(await f.act()).toMatchObject({ status: 422, providerStatus: 403, reason: "skill_source_github_rate_limited" });
+  expect(f.earlyCalls()).toBe(0); expect(f.provider).toHaveBeenCalledTimes(phase + 1);
+});
+it("requires another actual provider checkpoint after the new human intent even when installation and persisted source/version tuple did not change", async () => {
+  const f = fixture(); f.noInstall(); expect(await f.act()).toBeUndefined();
+  expect(f.provider).toHaveBeenCalledTimes(2); expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(false);
+  expect(f.provider).toHaveBeenCalledTimes(2);
+});
+it("never reuses a checkpoint receipt across changed source or installed bytes/history", async () => {
+  const f = fixture(); expect(await f.act()).toBeUndefined();
+  f.changeSource(); await f.observe(); expect(f.provider).toHaveBeenCalledTimes(3);
+  f.wrongVersion(); expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(false); expect(f.provider).toHaveBeenCalledTimes(4);
+  f.missingHistory(); expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(false); expect(f.provider).toHaveBeenCalledTimes(5);
+});
+it.each(["mutable", "foreignRepo"] as const)("does not reuse a checkpoint for %s even when its local tuple remains unchanged", async fault => {
+  const f = fixture(); expect(await f.act()).toBeUndefined(); f[fault]();
+  await f.observe(); await f.observe(); expect(f.provider).toHaveBeenCalledTimes(4);
+});

tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts

@@ -5,7 +5,7 @@ import { createHash } from "node:crypto";
 import type { CompanySkillDetail, CompanySkillVersion, CompanySkillUpdateStatus } from "../../../packages/shared/src/types/company-skill.js";
 import type { SkillSource, SkillSourceRefreshResult } from "../../../packages/shared/src/types/skill-source.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
-import { pollUntil, SkillSourceUpdateReadError } from "../api.js";
+import { pollUntil } from "../api.js";
 import { remainingIdentityObservationBudget } from "./identity-support.js";
 import { checks, observation, type InternalObservation } from "./internal-support.js";
 import { editableLocalNotes } from "./integrator-skill-source.js";
@@ -17,11 +17,24 @@ export { accessibilityUpdatePins } from "./integrator-skill-pins.js";
 const hash = (text: string) => createHash("sha256").update(text).digest("hex");
 const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`;
 interface UpdateObservation extends InternalObservation { upstream: CompanySkillDetail; localFork: CompanySkillDetail; upstreamVersions: CompanySkillVersion[]; forkVersions: CompanySkillVersion[]; source: SkillSource; updateStatus: Pick<CompanySkillUpdateStatus, "supported" | "currentRef" | "latestRef" | "hasUpdate"> }
-async function observeUpdate(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<UpdateObservation> {
+const updateReceipts = new WeakMap<ChatCatalogContext, { localDigest: string; status: UpdateObservation["updateStatus"] }>();
+async function observeUpdate(ctx: ChatCatalogContext, state: Record<string, unknown>, checkpoint = false): Promise<UpdateObservation> {
   const base = await observeInternal(ctx, state); const api = ctx.input.api; const upstream = state.upstream as CompanySkillDetail; const local = state.localFork as CompanySkillDetail;
-  return { ...base, upstream: await api.get<CompanySkillDetail>(`${root(ctx)}/skills/${upstream.id}`), localFork: await api.get<CompanySkillDetail>(`${root(ctx)}/skills/${local.id}`),
+  const saved = { upstream: await api.get<CompanySkillDetail>(`${root(ctx)}/skills/${upstream.id}`), localFork: await api.get<CompanySkillDetail>(`${root(ctx)}/skills/${local.id}`),
     upstreamVersions: await readSkillVersionHistory(api, ctx.input.fixtures.company.id, upstream.id), forkVersions: await readSkillVersionHistory(api, ctx.input.fixtures.company.id, local.id),
-    source: await api.get<SkillSource>(`${root(ctx)}/skill-sources/${(state.source as SkillSource).id}`), updateStatus: await api.getSkillSourceUpdateStatus<UpdateObservation["updateStatus"]>(ctx.input.fixtures.company.id, upstream.id) };
+    source: await api.get<SkillSource>(`${root(ctx)}/skill-sources/${(state.source as SkillSource).id}`) };
+  // This is the same genuine semantic checkpoint, not another provider read.
+  // Only the exact immutable public commit can retain it during grade settling;
+  // mutable remote refs must be read again even if local state is unchanged.
+  const localDigest = hash(JSON.stringify({ companyId: ctx.input.fixtures.company.id, sourceId: ctx.issue()?.id,
+    humanQueries: base.humanQueries, runs: base.runs.map(run => ({ id: run.id, companyId: run.companyId, agentId: run.agentId,
+      status: run.status, issueId: run.contextSnapshot?.issueId, commentId: run.contextSnapshot?.commentId })), ...saved })), prior = updateReceipts.get(ctx);
+  const immutable = /^[a-f0-9]{40}$/.test(saved.source.trackingRef) && saved.source.trackingRef === accessibilityUpdatePins.newCommit
+    && saved.source.fullName.toLowerCase() === new URL(accessibilityUpdatePins.repositoryURL).pathname.slice(1).toLowerCase();
+  const status = !checkpoint && immutable && prior?.localDigest === localDigest && prior.status.latestRef === accessibilityUpdatePins.newCommit ? prior.status
+    : await api.getSkillSourceUpdateStatus<UpdateObservation["updateStatus"]>(ctx.input.fixtures.company.id, upstream.id);
+  updateReceipts.set(ctx, { localDigest, status: { ...status } });
+  return { ...base, ...saved, updateStatus: status };
 }
 function updateObservation(value: unknown): UpdateObservation | undefined { const o = value as UpdateObservation; return observation(value) && !!o.upstream && !!o.localFork && Array.isArray(o.upstreamVersions) && Array.isArray(o.forkVersions) && !!o.source && !!o.updateStatus ? o : undefined; }
 function substantiveDiff(reply: string) {
@@ -83,9 +96,23 @@ export const integratorSkillUpdateCases = [integratorDefinition("skills", "skill
 { authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000,
   sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-update.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"],
   async act(ctx, state) {
-    const first = await pollUntil({ label: "actual upstream diff presented before installation", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => observeUpdate(ctx, state), retryOnError: error => !(error instanceof SkillSourceUpdateReadError), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" });
+    updateReceipts.delete(ctx);
+    const waitForSource = async (count: number, deadlineAt: number) => pollUntil({ label: "actual skill-update source response", deadlineAt, intervalMs: 1000,
+      load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); },
+      accept: runs => Date.now() < deadlineAt && runs.length === count && runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting",
+      reject: runs => runs.length > count || runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Skill update source has failed or excess native work" : undefined });
+    remainingIdentityObservationBudget(ctx);
+    await waitForSource(1, Math.min(Date.now() + 180_000, ctx.observationDeadlineAt!));
+    remainingIdentityObservationBudget(ctx);
+    const first = await observeUpdate(ctx, state, true);
+    remainingIdentityObservationBudget(ctx);
     state.reviewCheckpoint = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-skill-source-review-before-update.json", state.reviewCheckpoint);
     if (!previewCorrect(first)) throw new Error("The initial actual diff review was wrong or installed prematurely; later updates cannot repair it");
+    updateReceipts.delete(ctx);
     const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "Apply that reviewed upstream accessibility update now, keeping our editable local copy unchanged.");
+    await waitForSource(2, ctx.observationDeadlineAt!);
+    remainingIdentityObservationBudget(ctx);
+    await observeUpdate(ctx, state, true);
+    remainingIdentityObservationBudget(ctx);
   },
 })];