diff --git a/tests/runner-e2e/automatic-repository-flow.ts b/tests/runner-e2e/automatic-repository-flow.ts index 2a1b4693568..89e57f4c823 100644 --- a/tests/runner-e2e/automatic-repository-flow.ts +++ b/tests/runner-e2e/automatic-repository-flow.ts @@ -24,7 +24,8 @@ export async function runAutomaticRepositoryFlow(input: { const files = automaticRepositoryFiles(nonce); let issue: Row | undefined; let runs: Row[] = []; - const observation: AutomaticRepositoryObservation = { reference: "", issueDone: false, runCount: 0, + const observation: AutomaticRepositoryObservation = { expectedModel: execution.profile.model, + expectedProvider: execution.profile.provider === "codex" ? "codex" : "acpx:claude", actualProviders: [], reference: "", issueDone: false, runCount: 0, successfulCommittedRuns: 0, sourceContinuation: null, successorSource: null, workspaceIds: [], readyRepository: false, savedContext: null, firstSavedContext: null, firstContextAttributed: false, firstContextObservedAt: null, successorStartedAt: null, predecessorFinishedAt: null, predecessorRetagged: true, continuationReceipt: null, @@ -56,6 +57,10 @@ export async function runAutomaticRepositoryFlow(input: { observation.firstContextObservedAt = Date.now(); } } + observation.actualProviders = runs.map(run => { + const provider = run.runnerProfileJson?.nativeExecutionInput?.provider; + return { model: provider?.model ?? null, kind: provider ? provider.kind === "acpx" ? `acpx:${provider.agent}` : provider.kind : null }; + }); observation.workspaceIds = runs.map(run => run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId ?? null); observation.monitorScheduled ||= Boolean(issue.monitorNextCheckAt || runs.some(run => String(run.resultJson?.finalizationReasonCode).startsWith("scheduled_monitor_"))); } diff --git a/tests/runner-e2e/automatic-repository.test.ts b/tests/runner-e2e/automatic-repository.test.ts index 825ed08a0f7..a55b97624c5 100644 --- a/tests/runner-e2e/automatic-repository.test.ts +++ b/tests/runner-e2e/automatic-repository.test.ts @@ -3,7 +3,8 @@ import { automaticRepositoryApiState, automaticRepositoryStopReason, gradeAutoma const sha = "a".repeat(40); const reference = "c".repeat(64); -const valid: AutomaticRepositoryObservation = { reference, issueDone: true, runCount: 2, +const valid: AutomaticRepositoryObservation = { expectedModel: "gpt-6.1-sol", expectedProvider: "codex", + actualProviders: [{ model: "gpt-6.1-sol", kind: "codex" }, { model: "gpt-6.1-sol", kind: "codex" }], reference, issueDone: true, runCount: 2, successfulCommittedRuns: 2, sourceContinuation: "same_agent", successorSource: "native_status_decision", workspaceIds: ["workspace", "workspace"], readyRepository: true, savedContext: `${reference}\n`, firstSavedContext: `${reference}\n`, firstContextAttributed: true, firstContextObservedAt: 1000, successorStartedAt: 2000, predecessorFinishedAt: 1900, predecessorRetagged: false, @@ -37,6 +38,9 @@ it.each>([ { workspaceIds: ["old", "new"] }, { savedContext: null }, { firstSavedContext: null }, { firstContextAttributed: false }, { firstSavedContext: `${"d".repeat(64)}\n` }, { firstSavedContext: "REFERENCE-fresh\n" }, { firstContextObservedAt: null }, { firstContextObservedAt: 2000 }, { firstContextObservedAt: 3000 }, { successorStartedAt: null }, + { actualProviders: [] }, { actualProviders: [{ model: "gpt-6.1-sol", kind: "codex" }] }, + { actualProviders: [{ model: "gpt-5.6-sol", kind: "codex" }, valid.actualProviders[1]!] }, + { actualProviders: [valid.actualProviders[0]!, { model: "gpt-6.1-sol", kind: "acpx:claude" }] }, { readyRepository: false }, { downloaded: null }, { predecessorFinishedAt: null }, { predecessorFinishedAt: 2001 }, { predecessorRetagged: true }, { continuationReceipt: null }, { continuationReceipt: { ...valid.continuationReceipt!, dispatchId: valid.continuationReceipt!.intentId } }, diff --git a/tests/runner-e2e/automatic-repository.ts b/tests/runner-e2e/automatic-repository.ts index 321ed86b907..b8cf9168b70 100644 --- a/tests/runner-e2e/automatic-repository.ts +++ b/tests/runner-e2e/automatic-repository.ts @@ -29,6 +29,9 @@ export const automaticRepositoryTask: RunnerTaskFixture = { }; export interface AutomaticRepositoryObservation { + expectedModel: string; + expectedProvider: string; + actualProviders: Array<{ model: string | null; kind: string | null }>; reference: string; issueDone: boolean; runCount: number; @@ -94,6 +97,9 @@ export function gradeAutomaticRepository(value: AutomaticRepositoryObservation) /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value.continuationReceipt.dispatchId) && value.continuationReceipt.intentId !== value.continuationReceipt.dispatchId, "The saved same-agent intent must bind a distinct successor dispatch after the unmodified predecessor finishes, without a timer or user message."), + check("configured-provider", value.actualProviders.length === 2 && value.actualProviders.every(provider => + provider.model === value.expectedModel && provider.kind === value.expectedProvider), + "Both admitted runs must retain the configured provider and model."), check("retained-workspace", value.workspaceIds.length === 2 && Boolean(value.workspaceIds[0]) && value.workspaceIds.every(id => id === value.workspaceIds[0]) && value.readyRepository, "Both admissions must retain the same task workspace and its prepared repository."), check("retained-context", value.firstContextAttributed && value.firstContextObservedAt !== null && value.successorStartedAt !== null &&