diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update-checkpoint.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update-checkpoint.test.ts new file mode 100644 index 00000000000..92200628e8c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update-checkpoint.test.ts @@ -0,0 +1,70 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { readFileSync } from "node:fs"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const port = vi.hoisted(() => ({ observe: () => ({} as Record), versions: (_id: string): unknown[] => [], send: () => {} })); +vi.mock("./integrator-support.js", async original => ({ ...await original(), observeInternal: async () => port.observe() })); +vi.mock("./integrator-skill-versions.js", () => ({ readSkillVersionHistory: async (_api: unknown, _company: string, id: string) => port.versions(id) })); +vi.mock("../chat-flow.js", () => ({ sendChatMessage: async () => port.send() })); +vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value })); +import { SkillSourceUpdateReadError } from "../api.js"; +import { integratorSkillUpdateCases, gradeIntegratorSkillUpdate, accessibilityUpdatePins as pins } from "./integrator-skill-update.js"; +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); }); +const old = readFileSync(new URL("../fixtures/accessibility-source/0.4.SKILL.md", import.meta.url), "utf8"); +const next = readFileSync(new URL("../fixtures/accessibility-source/0.5.SKILL.md", import.meta.url), "utf8"); +function fixture(failAt?: number) { + vi.useFakeTimers(); vi.setSystemTime(10_000); + let phase = 0, changed = 0, wrongVersion = false, missingHistory = false, earlyCalls = 0, mutable = false, foreignRepo = false, noInstall = false; + const ready = () => Date.now() >= (phase ? 19_000 : 15_000), installed = () => phase === 1 && ready() && !noInstall; + const upstream = () => ({ id: "up", companyId: "company", editable: false, sourceRef: installed() ? pins.newCommit : pins.oldCommit, + currentVersionId: installed() ? "new" : "old", markdown: installed() && !wrongVersion ? next : old }); + const local = { id: "fork", companyId: "company", editable: true, forkedFromSkillId: "up", currentVersionId: "local", markdown: old + "\nLocal notes" }; + const versions = () => missingHistory ? [] : [{ id: "old", fileInventory: [{ path: "SKILL.md", content: old }] }, ...(installed() ? [{ id: "new", fileInventory: [{ path: "SKILL.md", content: next }] }] : [])]; + port.versions = id => id === "up" ? versions() : [{ id: "local" }]; + const source = () => ({ id: "source", companyId: "company", fullName: foreignRepo ? "foreign/repository" : "KreerC/ACCESSIBILITY.md", revision: changed, lastScanCommit: installed() ? pins.newCommit : pins.oldCommit, trackingRef: mutable ? "main" : pins.newCommit, automaticUpdates: false }); + const preview = "The currently installed version is 0.4. The new revision adds continuous accessibility regression tests and splits long forms into smaller steps. I have not applied it."; + const state: Record = { upstream: upstream(), localFork: local, source: source(), localVersions: [{ id: "local" }] }; + const runs = () => Array.from({ length: phase ? 2 : 1 }, (_, i) => ({ id: "native-" + i, status: i === phase && !ready() ? "running" : "succeeded" })); + port.observe = () => ({ scenarioId: "skills.review-update", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state, runs: runs(), + reply: installed() ? preview + "\nI updated the accessibility skill; the local copy is unchanged." : preview, + tasks: [], projects: [], goals: [], documents: [], revisions: [], comments: [], humanQueries: [], links: [], + activity: installed() ? [{ agentId: "ceo", action: "company.skill_source_refreshed", entityId: "source" }] : [] }); + port.send = () => { phase = 1; }; + const provider = vi.fn(async () => { + if (!ready()) earlyCalls++; + if (failAt === phase) throw new SkillSourceUpdateReadError(422, "skill_source_github_rate_limited", 403); + return { supported: true, hasUpdate: !installed(), currentRef: installed() ? pins.newCommit : pins.oldCommit, latestRef: pins.newCommit, trackingRef: pins.newCommit }; + }); + const localReads: string[] = []; + const ctx = { observationDeadlineAt: 520_000, input: { fixtures: { company: { id: "company" } }, page: {}, evidence: async () => {}, api: { + get: async (path: string) => { localReads.push(path); if (path.endsWith("/skills/up")) return upstream(); if (path.endsWith("/skills/fork")) return local; + if (path.endsWith("/skill-sources/source")) return source(); throw Error("Unexpected nonlocal read"); }, getSkillSourceUpdateStatus: provider, + } }, issue: () => ({ conversationState: ready() ? "waiting" : "active" }), refreshIssue: async () => {}, allRuns: async () => runs() } as unknown as ChatCatalogContext; + const definition = integratorSkillUpdateCases[0]!; + async function act() { const pending = definition.act!(ctx, state).then(() => undefined, (error: Error) => error); await vi.advanceTimersByTimeAsync(10_000); return pending; } + return { act, observe: () => definition.observe(ctx, state), provider, earlyCalls: () => earlyCalls, localReads, changeSource: () => { changed++; }, + wrongVersion: () => { wrongVersion = true; }, missingHistory: () => { missingHistory = true; }, mutable: () => { mutable = true; }, foreignRepo: () => { foreignRepo = true; }, noInstall: () => { noInstall = true; } }; +} +it("actual caller makes no provider request while native work is running and one genuine request at each settled semantic checkpoint", async () => { + const f = fixture(); expect(await f.act()).toBeUndefined(); expect(f.earlyCalls()).toBe(0); expect(f.provider).toHaveBeenCalledTimes(2); + for (let i = 0; i < 5; i++) expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(true); + expect(f.provider).toHaveBeenCalledTimes(2); expect(f.localReads.length).toBeGreaterThan(6); +}); +it.each([0, 1])("propagates real HTTP422/provider403 at semantic checkpoint%s without caching success", async phase => { + const f = fixture(phase); expect(await f.act()).toMatchObject({ status: 422, providerStatus: 403, reason: "skill_source_github_rate_limited" }); + expect(f.earlyCalls()).toBe(0); expect(f.provider).toHaveBeenCalledTimes(phase + 1); +}); +it("requires another actual provider checkpoint after the new human intent even when installation and persisted source/version tuple did not change", async () => { + const f = fixture(); f.noInstall(); expect(await f.act()).toBeUndefined(); + expect(f.provider).toHaveBeenCalledTimes(2); expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(false); + expect(f.provider).toHaveBeenCalledTimes(2); +}); +it("never reuses a checkpoint receipt across changed source or installed bytes/history", async () => { + const f = fixture(); expect(await f.act()).toBeUndefined(); + f.changeSource(); await f.observe(); expect(f.provider).toHaveBeenCalledTimes(3); + f.wrongVersion(); expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(false); expect(f.provider).toHaveBeenCalledTimes(4); + f.missingHistory(); expect(gradeIntegratorSkillUpdate(await f.observe()).passed).toBe(false); expect(f.provider).toHaveBeenCalledTimes(5); +}); +it.each(["mutable", "foreignRepo"] as const)("does not reuse a checkpoint for %s even when its local tuple remains unchanged", async fault => { + const f = fixture(); expect(await f.act()).toBeUndefined(); f[fault](); + await f.observe(); await f.observe(); expect(f.provider).toHaveBeenCalledTimes(4); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts index 6d715779c1d..e6e06b68873 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts @@ -5,7 +5,7 @@ import { createHash } from "node:crypto"; import type { CompanySkillDetail, CompanySkillVersion, CompanySkillUpdateStatus } from "../../../packages/shared/src/types/company-skill.js"; import type { SkillSource, SkillSourceRefreshResult } from "../../../packages/shared/src/types/skill-source.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; -import { pollUntil, SkillSourceUpdateReadError } from "../api.js"; +import { pollUntil } from "../api.js"; import { remainingIdentityObservationBudget } from "./identity-support.js"; import { checks, observation, type InternalObservation } from "./internal-support.js"; import { editableLocalNotes } from "./integrator-skill-source.js"; @@ -17,11 +17,24 @@ export { accessibilityUpdatePins } from "./integrator-skill-pins.js"; const hash = (text: string) => createHash("sha256").update(text).digest("hex"); const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`; interface UpdateObservation extends InternalObservation { upstream: CompanySkillDetail; localFork: CompanySkillDetail; upstreamVersions: CompanySkillVersion[]; forkVersions: CompanySkillVersion[]; source: SkillSource; updateStatus: Pick } -async function observeUpdate(ctx: ChatCatalogContext, state: Record): Promise { +const updateReceipts = new WeakMap(); +async function observeUpdate(ctx: ChatCatalogContext, state: Record, checkpoint = false): Promise { const base = await observeInternal(ctx, state); const api = ctx.input.api; const upstream = state.upstream as CompanySkillDetail; const local = state.localFork as CompanySkillDetail; - return { ...base, upstream: await api.get(`${root(ctx)}/skills/${upstream.id}`), localFork: await api.get(`${root(ctx)}/skills/${local.id}`), + const saved = { upstream: await api.get(`${root(ctx)}/skills/${upstream.id}`), localFork: await api.get(`${root(ctx)}/skills/${local.id}`), upstreamVersions: await readSkillVersionHistory(api, ctx.input.fixtures.company.id, upstream.id), forkVersions: await readSkillVersionHistory(api, ctx.input.fixtures.company.id, local.id), - source: await api.get(`${root(ctx)}/skill-sources/${(state.source as SkillSource).id}`), updateStatus: await api.getSkillSourceUpdateStatus(ctx.input.fixtures.company.id, upstream.id) }; + source: await api.get(`${root(ctx)}/skill-sources/${(state.source as SkillSource).id}`) }; + // This is the same genuine semantic checkpoint, not another provider read. + // Only the exact immutable public commit can retain it during grade settling; + // mutable remote refs must be read again even if local state is unchanged. + const localDigest = hash(JSON.stringify({ companyId: ctx.input.fixtures.company.id, sourceId: ctx.issue()?.id, + humanQueries: base.humanQueries, runs: base.runs.map(run => ({ id: run.id, companyId: run.companyId, agentId: run.agentId, + status: run.status, issueId: run.contextSnapshot?.issueId, commentId: run.contextSnapshot?.commentId })), ...saved })), prior = updateReceipts.get(ctx); + const immutable = /^[a-f0-9]{40}$/.test(saved.source.trackingRef) && saved.source.trackingRef === accessibilityUpdatePins.newCommit + && saved.source.fullName.toLowerCase() === new URL(accessibilityUpdatePins.repositoryURL).pathname.slice(1).toLowerCase(); + const status = !checkpoint && immutable && prior?.localDigest === localDigest && prior.status.latestRef === accessibilityUpdatePins.newCommit ? prior.status + : await api.getSkillSourceUpdateStatus(ctx.input.fixtures.company.id, upstream.id); + updateReceipts.set(ctx, { localDigest, status: { ...status } }); + return { ...base, ...saved, updateStatus: status }; } function updateObservation(value: unknown): UpdateObservation | undefined { const o = value as UpdateObservation; return observation(value) && !!o.upstream && !!o.localFork && Array.isArray(o.upstreamVersions) && Array.isArray(o.forkVersions) && !!o.source && !!o.updateStatus ? o : undefined; } function substantiveDiff(reply: string) { @@ -83,9 +96,23 @@ export const integratorSkillUpdateCases = [integratorDefinition("skills", "skill { authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000, sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-update.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"], async act(ctx, state) { - const first = await pollUntil({ label: "actual upstream diff presented before installation", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => observeUpdate(ctx, state), retryOnError: error => !(error instanceof SkillSourceUpdateReadError), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" }); + updateReceipts.delete(ctx); + const waitForSource = async (count: number, deadlineAt: number) => pollUntil({ label: "actual skill-update source response", deadlineAt, intervalMs: 1000, + load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, + accept: runs => Date.now() < deadlineAt && runs.length === count && runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting", + reject: runs => runs.length > count || runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Skill update source has failed or excess native work" : undefined }); + remainingIdentityObservationBudget(ctx); + await waitForSource(1, Math.min(Date.now() + 180_000, ctx.observationDeadlineAt!)); + remainingIdentityObservationBudget(ctx); + const first = await observeUpdate(ctx, state, true); + remainingIdentityObservationBudget(ctx); state.reviewCheckpoint = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-skill-source-review-before-update.json", state.reviewCheckpoint); if (!previewCorrect(first)) throw new Error("The initial actual diff review was wrong or installed prematurely; later updates cannot repair it"); + updateReceipts.delete(ctx); const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "Apply that reviewed upstream accessibility update now, keeping our editable local copy unchanged."); + await waitForSource(2, ctx.observationDeadlineAt!); + remainingIdentityObservationBudget(ctx); + await observeUpdate(ctx, state, true); + remainingIdentityObservationBudget(ctx); }, })];