From b59e295c7ff8be95274a53e3f209818f7f10cfff Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 6 Nov 2026 05:54:35 -0600 Subject: [PATCH] Calibrate four chat oracles and qualify coherent candidate --- server/src/routes/openapi.ts | 2 +- .../native-runtime/runner-api.test.ts | 4 +- tests/runner-e2e/CHAT-FEATURES.md | 66 +++++++++++++++++++ .../runner-e2e/chat-catalog-descriptors.json | 8 +-- .../external-personal-secret.test.ts | 6 ++ .../external-personal-secret.ts | 2 +- .../integrator-skill-authoring-v2.test.ts | 1 + .../integrator-skill-update.test.ts | 1 + .../integrator-skill-update.ts | 2 +- .../chat-catalog-domains/integrator-skills.ts | 2 +- ...internal-timeline-native-milestone.test.ts | 2 +- .../internal-work-timeline.ts | 2 +- 12 files changed, 87 insertions(+), 11 deletions(-) diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 807faef2032..b09caa92cb2 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -5731,7 +5731,7 @@ registry.registerPath({ method: "post", path: "/api/approvals/{id}/resubmit", tags: ["approvals"], - summary: "Revise and resubmit an approval or proposed hire", + summary: "Revise and resubmit a changes-requested approval", description: "Replace the payload of an approval whose status is revision_requested, or resubmit its existing payload. " + "Use this operation to revise a changes-requested hire proposal, such as changing its agent configuration to a cheaper model, and send it back for board review. " + diff --git a/server/src/services/native-runtime/runner-api.test.ts b/server/src/services/native-runtime/runner-api.test.ts index ed76e86a157..74e04ac1acf 100644 --- a/server/src/services/native-runtime/runner-api.test.ts +++ b/server/src/services/native-runtime/runner-api.test.ts @@ -175,7 +175,7 @@ describe("runner API catalog", () => { expect(previewBody.properties).not.toHaveProperty("adapterOverrides"); }); - it("discovers proposal revision without confusing it with approval decisions or reads", () => { + it("discovers proposal revision without confusing it with hiring, approval decisions, or reads", () => { const resubmit = "POST /api/approvals/{id}/resubmit"; for (const query of [ "revise changes-requested hire proposal", @@ -184,6 +184,8 @@ describe("runner API catalog", () => { ]) expect(searchRunnerApi({ query }).results.map(row => row.operationId), query).toContain(resubmit); for (const [query, expected] of [ ["GET /api/approvals/{id}", "GET /api/approvals/{id}"], + ["get approval", "GET /api/approvals/{id}"], + ["hire agent", "POST /api/companies/{companyId}/agent-hires"], ["approve approval", "POST /api/approvals/{id}/approve"], ["reject approval", "POST /api/approvals/{id}/reject"], [resubmit, resubmit], diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 946585242e8..59067881297 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -564,3 +564,69 @@ with cap 2, zero automatic retries and original 480000ms/660000ms cells. Team re calibrated locally; the final uniform ten is still required on this same source. All original grades, historical conditions, unmerged core holds and the 196/244 full-catalog gate remain intact. + + +Prospective timeline and provider-routing conditions + +Timeline v13 accepts a same-identified-task account that orders the first worker's +real command failure and pause before the different saved repair worker's verified +repair, without requiring an explicit assignment phrase. Independent same-task +saved handoff, actor, command, chronology, link, privacy and read-only proofs remain +mandatory. Launch 3376's original timeline FAILED result remains unchanged; its +completed truthful account motivated this prospective calibration, not a regrade. + +The reviewed OpenCode chat profile now supplies model-specific +`OPENCODE_CONFIG_CONTENT` selecting OpenRouter `provider.sort: "throughput"` only +for `deepseek/deepseek-v4.1-flash-1017`. This changes the profile environment and +provider-routing condition: it is a PROSPECTIVE performance experiment, with +benefit unknown. The exact model, default reasoning behavior, permissions, output +options, auxiliary-model configuration and all original native/human bounds and +clocks are unchanged. Previous default-routing failures are retained; no runtime +latency repair, reliability improvement or new native pass is claimed. + +Approval resubmission metadata describes the existing revision/resubmission +operation and its unchanged authority boundary so ordinary API discovery can find +it. This does not approve or execute a hire. Candidate core changes remain open +and unmerged; the 196/244 full-catalog gate remains held. All 244 YAML identities, +queries, contexts and outcomes remain unchanged. Combined qualification and any +next native measurement remain pending the coherent release decision. + +Personal secret v17 accepts an asserted current personal value, key, token or +credential that worked or works. This is a narrative calibration, not permission +to count save-only, planned, negated, unrelated-task or another person's success. +The original launch 3376 failure remains FAILED even though its plain-comment +path reached secure current-owner save, actual worker delivery with a 200 receipt +and a separate CEO acknowledgment. Independent consumer delivery, actor/owner +isolation, receipt, privacy and no-leak proofs remain mandatory. No new native +pass or latency improvement is claimed; the original three native/two human inputs +and 660000ms cell remain unchanged. Timeline and Personal are two of the four oracle version +leaves changed in this prospective batch. + +Skill authoring v6 observes the four required checklist points within their +actual heading-and-list sections rather than requiring every term in one text +block. The launch 3376 native authored and saved the skill, but its original +FAILED observer stopped before UI reload/readback qualification. This repair +provides no native pass credit: actual file, author, version, editable Studio, +reload and matching readback proofs remain required. Unrelated sections, missing +requirements and waivers remain rejected. This is the third prospective oracle +version leaf; skill review/update is the fourth calibration below. + +Any next throughput-routing measurement first uses a small representative +announcement case before expanding to the uniform ten. That experiment remains +unrun and requires a separate release decision; neither local projection nor +these assertion calibrations demonstrate a performance benefit. + + +Skill review/update v3 preserves a complete persisted preview on its original +path. Only an incomplete persisted preview may fall back to the current rendered +final response, anchored after the actual human review request and corroborated +by the same committed successful native run. Hidden, stale, foreign, reasoning, +tool-output and API-only summaries remain excluded. Hydration waits consume the +original observation deadline without renewing it. The later human apply request, +actual upstream version change and unchanged local fork remain mandatory. Launch +3376's original FAILED review/update result stays unchanged; its later apply +phase was never reached, so this calibration supplies no completed native pass. +Together the four prospective oracle leaves are Timeline v13, Personal v17, +Skill authoring v6 and Integrator v3; every other descriptor field and sourceFiles +array remains unchanged. The routing experiment remains unrun and its performance +benefit is unproved. No main pass credit or release acceptance is claimed. diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 6fb983527bc..b50df3afaf2 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -259,7 +259,7 @@ "../../server/src/services/work-timeline.ts", "../../server/src/services/native-runtime/runner-api-catalog.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v12", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v13", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, @@ -5009,7 +5009,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v16", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v17", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, @@ -5113,7 +5113,7 @@ "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts" ], - "oracleVersion": "paperclip.chat-catalog.skill-authoring.v5", + "oracleVersion": "paperclip.chat-catalog.skill-authoring.v6", "expectedRunCount": 1, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, @@ -5231,7 +5231,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v2", + "oracleVersion": "paperclip.chat-catalog.integrator.v3", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts index ceed4c3ea51..69cdd5345fd 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts @@ -3,6 +3,7 @@ import { questionSetToAskUserQuestionsPayload } from "../../../packages/shared/s import { expect, it } from "vitest"; import { gradePersonalSecret, + personalSecretExternalCases, personalSecretGuidance, personalSecretConfirmation, personalSecretQuestionHandoff, @@ -316,3 +317,8 @@ it("keeps successful worked wording dependent on actual worker delivery",()=>{ o.state.workerReceiptBound=false; expect(gradePersonalSecret(o).passed).toBe(false); }); + +it("declares the prospective personal acknowledgment calibration without changing its bounds", () => { + expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v17", + expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 660000 }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index 3990b781f88..4b9bff4ddc2 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -461,7 +461,7 @@ export const personalSecretExternalCases = [ if (!o.state.initialQuestionId) return; return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; }, - oracleVersion: "paperclip.chat-catalog.personal-secret.v16", + oracleVersion: "paperclip.chat-catalog.personal-secret.v17", setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3, minimumRunCount: 3, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts index 190b9402d65..35ed48645d2 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts @@ -62,6 +62,7 @@ it("rejects invented inspection receipts and keeps other skill cases on their or f.state.authoredSkillInspection={companyId:"company",skillId:"other",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true}; expect(gradeIntegratorSkills(f.o).passed).toBe(false); expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.explicit-restriction")!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v1"); + expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!.oracleVersion).toBe("paperclip.chat-catalog.skill-authoring.v6"); }); it("cannot extend an exhausted original case budget to inspect a finished artifact",async()=>{ const f=fixture();Object.assign(f.ctx,{observationDeadlineAt:Date.now()-1}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts index 9f2e1eaebb3..be42c30a259 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.test.ts @@ -130,6 +130,7 @@ it("actual review-update setup keeps the real prerequisite and then prepares the vi.stubGlobal("fetch", vi.fn(async (url: string) => new Response(url.includes(pins.oldCommit) ? bytes.old : bytes.next))); try { const f = setupFixture(false); const result = await integratorSkillUpdateCases[0]!.setup(f.ctx); + expect(integratorSkillUpdateCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v3"); expect(f.prerequisite).toHaveBeenCalledOnce(); expect(saveSkillConversationContext).toHaveBeenCalledOnce(); expect(result.source).toMatchObject({ trackingRef: pins.newCommit, lastScanCommit: pins.oldCommit }); } finally { vi.unstubAllGlobals(); } }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts index b538f2fa615..9060db5f538 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts @@ -163,7 +163,7 @@ export const integratorSkillUpdateCases = [integratorDefinition("skills", "skill const selected = await saveSkillConversationContext(ctx, { description: `Selected upstream accessibility skill: ${upstream.name} (${upstream.id}), tracked source ${source.id}. Installed source revision ${upstream.sourceRef}; tracking reference ${source.trackingRef}. Separate editable local copy: ${localFork.name} (${localFork.id}).` }); return { upstream, localFork, source, factualContextAttachment: selected.factualContextAttachment, localVersions: await readSkillVersionHistory(ctx.input.api, ctx.input.fixtures.company.id, localFork.id, localFork.currentVersionId) }; }, observeUpdate, gradeIntegratorSkillUpdate, async (ctx, state) => { const skill = state.upstream as CompanySkillDetail; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`, skill.name); }, -{ oracleVersion: "paperclip.chat-catalog.integrator.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000, +{ oracleVersion: "paperclip.chat-catalog.integrator.v3", authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000, sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-update.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"], async act(ctx, state) { updateReceipts.delete(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index 97005dab5ed..7bf3ccee0df 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -201,7 +201,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => { const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill); base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"]; - return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v5", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), + return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v6", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); }, async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); }, } : base; diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 49e9a9bb0ea..fecce3d6471 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -269,7 +269,7 @@ it("admits the selected live timeline metadata without invoking fixture callback const spies = callbacks.map(key => vi.spyOn(implementation, key)); try { expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 }); - expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v12"); + expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v13"); expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles); expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 }); for (const spy of spies) expect(spy).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 6fe60bef9f0..187a6e34808 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase { const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); - return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v12", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, + return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v13", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts","../../server/src/services/native-runtime/runner-api-catalog.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },