mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
evals: select OpenCode DeepSeek for remote chat catalog
This commit is contained in:
1 parent
c911d97f4a
commit
9079e0f20d
19 files changed
+154
-34
No files matched your search
@@ -3002,3 +3002,22 @@ PAPERCLIP_RUNNER_E2E_SOURCE_SHA=<snapshot-sha> node --import ./cli/node_modules/
|
||||
Its receipt is retained under `results/chat-catalog-campaign/`; it is never a
|
||||
scenario measurement or feature acceptance. Missing/changed ownership or an
|
||||
unknown inspection result is not proof that a resource is absent.
|
||||
|
||||
### Chat catalog remote OpenCode campaign
|
||||
|
||||
The explicit `chat-features` matrix now selects `runner-opencode` with native
|
||||
`paperclip_runner`, provider `opencode`, and exact model
|
||||
`openrouter/deepseek/deepseek-v4.1-flash-1017`. Its credential is an encrypted
|
||||
`OPENROUTER_API_KEY` reference. OpenCode remains pinned to the runner-owned
|
||||
1.18.34 executable; this candidate does not change the previously qualified
|
||||
OpenCode model for other suites. `local` means the selected execution host,
|
||||
including an isolated AWS worker, rather than a requirement to use a developer laptop.
|
||||
|
||||
This is a new 244-cell behavioral campaign. Historical Codex results and passes
|
||||
remain separate and do not qualify the new provider/model. Named-provider feature
|
||||
requests retain their original meaning. Unavailable-model history requires an
|
||||
actually admitted failed OpenCode run with the exact configured provider/model;
|
||||
a preflight rejection does not establish that prerequisite. Provider/account
|
||||
qualification, paid-job approval, and owned AWS Spot capacity must be established
|
||||
before remote admission. Discovery and credential-free controls establish none
|
||||
of those live outcomes.
|
||||
@@ -1,3 +1,4 @@
|
||||
import { CHAT_OPENCODE_MODEL, CHAT_OPENCODE_PROFILE } from "./chat-runtime-profile.js";
|
||||
import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js";
|
||||
import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js";
|
||||
import { finalizationRecoveryTask, finalizationRecoveryDigest } from "./finalization-recovery.js";
|
||||
@@ -1608,8 +1609,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
id: CHAT_FEATURE_SUITE, label: "Chat feature journeys", manualOnly: true,
|
||||
description: "Executable catalog requests through Agent Chat, with independently graded persisted state.",
|
||||
groups: ["chat", "native"],
|
||||
profiles: [chatFeatureProfile(nativeProfile({ id: "runner-codex-subscription", label: "Native Codex subscription",
|
||||
provider: "codex", model: DEFAULT_CODEX_LOCAL_MODEL, credential: "CODEX_AUTH_JSON", supportedEnvironments: ["local"] }))],
|
||||
profiles: [chatFeatureProfile(nativeProfile({ id: CHAT_OPENCODE_PROFILE, label: "Native OpenCode OpenRouter DeepSeek V4.1 Flash 1017",
|
||||
provider: "opencode", model: CHAT_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"],
|
||||
modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-catalog:opencode-1.18.34:deepseek-v4.1-flash-1017:remote-initial" } }))],
|
||||
environments: [localEnvironment], tasks: [...chatFeatureTasks, ...chatCatalogTasks], expectedMatrixSize: 2 + chatCatalogTasks.length,
|
||||
definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, inventoryDigest: chatCatalogInventoryDigest, frozenScenarioCount: 244, scheduling: "explicit-only",
|
||||
instructions: "production-role-defaults", companyAndAgentBudgetCents: CHAT_FEATURE_BUDGET_CENTS,
|
||||
|
||||
@@ -11,7 +11,8 @@ import { ARCHIVE_CONTROL_ENV, ARCHIVE_SCENARIO, archiveFaultClient, archiveServe
|
||||
import { buildPaperclipServerEnvironment } from "./harness-env.js";
|
||||
import { reserveRunnerE2EServerPort } from "./ports.js";
|
||||
const id = `chat-features.runner-codex-subscription.local.${ARCHIVE_SCENARIO}`;
|
||||
it("starts before backend and gives only the test child a private authenticated control handoff", async () => {
|
||||
it.each(["runner-codex-subscription", "runner-opencode"])("binds %s archive control to its own selected execution and private handoff", async profile => {
|
||||
const id = `chat-features.${profile}.local.${ARCHIVE_SCENARIO}`;
|
||||
const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "paperclip-runner-e2e-"))); await mkdir(path.join(temporaryRoot, "selected-private-inputs"), { mode: 0o700 });
|
||||
const port = await reserveRunnerE2EServerPort(), backendOrigin = `http://127.0.0.1:${port}`, secrets: string[] = [];
|
||||
const control = await startArchiveFaultControl({ selected: [id], temporaryRoot, backendOrigin, privateSecrets: secrets });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { reviewedChatExecutionIds } from "./chat-runtime-profile.js";
|
||||
import { createServer } from "node:http";
|
||||
import { randomBytes, timingSafeEqual } from "node:crypto";
|
||||
import { lstat, readFile, realpath, rm, writeFile } from "node:fs/promises";
|
||||
@@ -6,7 +7,7 @@ import { listenOnFetchAllowedPort } from "../e2e/fetch-allowed-port.js";
|
||||
import { startArchiveFaultProxy, validArchiveBinding, type ArchiveBinding, type ArchiveFaultTransport, type ArchiveTransportReceipt } from "./chat-catalog-domains/internal-archive-transport.js";
|
||||
export const ARCHIVE_CONTROL_ENV = "PAPERCLIP_RUNNER_E2E_ARCHIVE_CONTROL";
|
||||
export const ARCHIVE_SCENARIO = "inbox-search.archive-partial-failure";
|
||||
const executionId = `chat-features.runner-codex-subscription.local.${ARCHIVE_SCENARIO}`;
|
||||
const executionIds = reviewedChatExecutionIds(ARCHIVE_SCENARIO);
|
||||
interface Binding { version: 1; executionId: string; temporaryRoot: string; backendOrigin: string; origin: string; controlOrigin: string; token: string }
|
||||
const origin = (value: unknown): value is string => { try { const u = new URL(String(value)); return u.protocol === "http:" && u.hostname === "127.0.0.1" && !!u.port && u.origin === value; } catch { return false; } };
|
||||
async function privateDirectory(root: string) {
|
||||
@@ -17,14 +18,14 @@ async function privateDirectory(root: string) {
|
||||
} catch { throw new Error("Private archive root is unavailable or unsafe"); }
|
||||
}
|
||||
async function privateBinding(filename: string, root: string, selected: readonly string[]) {
|
||||
if (selected.length !== 1 || selected[0] !== executionId || filename !== path.join(root, "selected-private-inputs", "archive-fault.json")) throw new Error("Archive transport requires its exact selected owned case");
|
||||
if (selected.length !== 1 || !executionIds.includes(selected[0]!) || filename !== path.join(root, "selected-private-inputs", "archive-fault.json")) throw new Error("Archive transport requires its exact selected owned case");
|
||||
try {
|
||||
await privateDirectory(root);
|
||||
const directory = path.dirname(filename), dir = await lstat(directory), file = await lstat(filename);
|
||||
if (await realpath(root) !== root || await realpath(directory) !== directory || !dir.isDirectory() || dir.mode & 0o077 || file.isSymbolicLink() || !file.isFile() || (file.mode & 0o777) !== 0o600 || file.size > 4096
|
||||
|| typeof process.getuid === "function" && (dir.uid !== process.getuid() || file.uid !== process.getuid())) throw new Error();
|
||||
const value = JSON.parse(await readFile(filename, "utf8")) as Binding;
|
||||
if (Object.keys(value).sort().join() !== "backendOrigin,controlOrigin,executionId,origin,temporaryRoot,token,version" || value.version !== 1 || value.executionId !== executionId || value.temporaryRoot !== root
|
||||
if (Object.keys(value).sort().join() !== "backendOrigin,controlOrigin,executionId,origin,temporaryRoot,token,version" || value.version !== 1 || value.executionId !== selected[0] || value.temporaryRoot !== root
|
||||
|| !origin(value.backendOrigin) || !origin(value.origin) || !origin(value.controlOrigin) || new Set([value.backendOrigin, value.origin, value.controlOrigin]).size !== 3 || !/^[a-f0-9]{64}$/.test(value.token)) throw new Error();
|
||||
return value;
|
||||
} catch { throw new Error("Private archive transport binding is unavailable or unsafe"); }
|
||||
@@ -40,7 +41,7 @@ function validReceipt(value: unknown): value is ArchiveTransportReceipt {
|
||||
&& (row.status === undefined || Number.isInteger(row.status) && Number(row.status) >= 100 && Number(row.status) <= 599);
|
||||
}
|
||||
export async function startArchiveFaultControl(input: { selected: readonly string[]; temporaryRoot: string; backendOrigin: string; privateSecrets: string[] }) {
|
||||
if (input.selected.length !== 1 || input.selected[0] !== executionId || !origin(input.backendOrigin)) throw new Error("Archive fault is allowlisted only for its owned local execution");
|
||||
if (input.selected.length !== 1 || !executionIds.includes(input.selected[0]!) || !origin(input.backendOrigin)) throw new Error("Archive fault is allowlisted only for its owned local execution");
|
||||
await privateDirectory(input.temporaryRoot);
|
||||
if (/(?:^|[\s,])(?:http|net|tls|undici|\*)(?:$|[\s,])/i.test(process.env.NODE_DEBUG ?? "")) throw new Error("Archive private control refuses raw transport debug output");
|
||||
const proxy = await startArchiveFaultProxy(input.backendOrigin), token = randomBytes(32).toString("hex");
|
||||
@@ -59,7 +60,7 @@ export async function startArchiveFaultControl(input: { selected: readonly strin
|
||||
const port = await listenOnFetchAllowedPort(control), controlOrigin = `http://127.0.0.1:${port}`;
|
||||
filename = path.join(input.temporaryRoot, "selected-private-inputs", "archive-fault.json");
|
||||
input.privateSecrets.push(token, controlOrigin, filename);
|
||||
const binding: Binding = { version: 1, executionId, temporaryRoot: input.temporaryRoot, backendOrigin: input.backendOrigin, origin: proxy.origin, controlOrigin, token };
|
||||
const binding: Binding = { version: 1, executionId: input.selected[0]!, temporaryRoot: input.temporaryRoot, backendOrigin: input.backendOrigin, origin: proxy.origin, controlOrigin, token };
|
||||
await writeFile(filename, `${JSON.stringify(binding)}\n`, { mode: 0o600, flag: "wx" }); created = true;
|
||||
await privateBinding(filename, input.temporaryRoot, input.selected);
|
||||
let closed = false;
|
||||
@@ -70,14 +71,14 @@ export async function startArchiveFaultControl(input: { selected: readonly strin
|
||||
* normal API-origin override; never a control filename, URL or bearer. */
|
||||
export async function archiveServerEnvironment(input: { selected: readonly string[]; temporaryRoot: string; backendOrigin: string; env: NodeJS.ProcessEnv }) {
|
||||
const source = { ...input.env }, filename = source[ARCHIVE_CONTROL_ENV]; delete source[ARCHIVE_CONTROL_ENV];
|
||||
if (!filename) { if (input.selected.includes(executionId)) throw new Error("Selected archive case is missing pre-server fault transport"); return { source, overrides: {} }; }
|
||||
if (!filename) { if (input.selected.some(id => executionIds.includes(id))) throw new Error("Selected archive case is missing pre-server fault transport"); return { source, overrides: {} }; }
|
||||
const binding = await privateBinding(filename, input.temporaryRoot, input.selected);
|
||||
if (binding.backendOrigin !== input.backendOrigin) throw new Error("Archive backend origin differs from owned server");
|
||||
return { source, overrides: { PAPERCLIP_API_URL: binding.origin } };
|
||||
}
|
||||
export async function archiveFaultClient(input: { selected: string; backendOrigin: string; privateSecrets?: string[]; env?: NodeJS.ProcessEnv }): Promise<ArchiveFaultTransport | undefined> {
|
||||
const env = input.env ?? process.env, filename = env[ARCHIVE_CONTROL_ENV];
|
||||
if (!filename) { if (input.selected === executionId) throw new Error("Selected archive case is missing private transport handoff"); return; }
|
||||
if (!filename) { if (executionIds.includes(input.selected)) throw new Error("Selected archive case is missing private transport handoff"); return; }
|
||||
const binding = await privateBinding(filename, env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "", [input.selected]);
|
||||
if (binding.backendOrigin !== input.backendOrigin) throw new Error("Archive client belongs to another backend");
|
||||
input.privateSecrets?.push(binding.token, binding.controlOrigin, filename);
|
||||
|
||||
@@ -49,26 +49,27 @@ export function gradeTeamConfigure(value: unknown) {
|
||||
return c.result();
|
||||
}
|
||||
export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => {
|
||||
if (ctx.input.execution.profile.credential !== "CODEX_AUTH_JSON") throw new Error("Setup blocker: this disposable team fixture requires the selected Codex subscription account, not an inferred personal login");
|
||||
if (!["CODEX_AUTH_JSON", "OPENROUTER_API_KEY"].includes(ctx.input.execution.profile.credential)) throw new Error("Setup blocker: this disposable team fixture requires the selected campaign account, not an inferred personal login");
|
||||
const team = (await ctx.input.api.get<CatalogTeam[]>("/api/teams/catalog")).find(row => row.slug === "product-engineering" && row.compatibility === "compatible");
|
||||
if (!team) throw new Error("Setup blocker: compatible real Product Engineering package is unavailable");
|
||||
const built = ctx.input.execution.profile.buildAgent({ environmentId: ctx.input.fixtures.environment.id, environmentFixtureId: ctx.input.execution.environment.id, workspacePath: ctx.input.workspacePath, secretRefs: ctx.input.fixtures.secretRefs, executionId: ctx.input.nonce });
|
||||
if (typeof built.adapterType !== "string") throw new Error("Setup blocker: selected profile does not declare a supported runtime adapter");
|
||||
const configured = codexSubscriptionAgentConfig(built, process.env, ctx.input.fixtures.company.id);
|
||||
const override: CompanyPortabilityAdapterOverride = { adapterType: built.adapterType, adapterConfig: configured.adapterConfig };
|
||||
const subscription = ctx.input.execution.profile.credential === "CODEX_AUTH_JSON";
|
||||
const configured = subscription ? codexSubscriptionAgentConfig(built, process.env, ctx.input.fixtures.company.id) : built;
|
||||
const override: CompanyPortabilityAdapterOverride = { adapterType: built.adapterType, adapterConfig: configured.adapterConfig as Record<string, unknown> };
|
||||
const emptyHome = await mkdtemp(path.join(tmpdir(), "paperclip-missing-team-login-")); fixtures.set(ctx, { emptyHome, override });
|
||||
try {
|
||||
const install = await ctx.input.api.postSensitive<CatalogTeamInstallResult>(`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/install`, { targetManagerAgentId: ctx.input.fixtures.agent.id, adapterOverrides: Object.fromEntries(team.agentSlugs.map(slug => [slug, override])) });
|
||||
const missingId = install.portabilityImport.agents.find(row => row.slug === "qa")?.id, workingId = install.portabilityImport.agents.find(row => row.slug === "senior-coder")?.id;
|
||||
if (!missingId || !workingId) throw new Error("Setup blocker: installer did not produce the exact QA and senior-coder members");
|
||||
await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record<string, unknown> ?? {}), CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } } });
|
||||
await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record<string, unknown> ?? {}), ...missingTeamAccountEnvironment(subscription, emptyHome) } } });
|
||||
const members = catalogTeamMembers(await observeTeamResources(ctx), team.id);
|
||||
const initialProbes = await Promise.all(members.map(member => probe(ctx, member)));
|
||||
if (!initialProbes.some(row => row.agentId === missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) || !initialProbes.some(row => row.agentId === workingId && row.status === "pass")) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split");
|
||||
const working = await ctx.input.api.get<Agent>(`/api/agents/${workingId}`);
|
||||
const workingRevisionIdsBefore = (await ctx.input.api.get<Array<{ id: string }>>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort();
|
||||
await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
|
||||
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company Codex subscription account. The senior coder (${workingId}) is already configured. The existing company subscription is authorized for the QA member too; its value must not enter chat.` });
|
||||
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company ${subscription ? "Codex subscription" : "OpenRouter"} account. The senior coder (${workingId}) is already configured. The existing company account is authorized for the QA member too; its value must not enter chat.` });
|
||||
return { teamId: team.id, installedFromCatalog: true, memberIdsBefore: members.map(row => row.id).sort(), expectedMembers: members.length, missingId, missingName: members.find(row => row.id === missingId)!.name,
|
||||
workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes };
|
||||
} catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; }
|
||||
@@ -105,3 +106,7 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as
|
||||
async verifyResult(ctx, state, value) { await verifyRenderedReply(ctx, value, String(state.initialRunId)); },
|
||||
async cleanup(ctx) { const fixture = fixtures.get(ctx); fixtures.delete(ctx); if (fixture) await rm(fixture.emptyHome, { recursive: true, force: true }); },
|
||||
})];
|
||||
|
||||
export function missingTeamAccountEnvironment(subscription: boolean, emptyHome: string) {
|
||||
return subscription ? { CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } : { OPENROUTER_API_KEY: "" };
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { it, expect } from "vitest";
|
||||
import { admittedModelFailure } from "./runtime-native-failure.js";
|
||||
import { admittedModelFailure, unavailableNativeModel } from "./runtime-native-failure.js";
|
||||
function valid() { return { companyId: "company", agentId: "morgan", issueId: "website", model: "unavailable",
|
||||
run: { id: "actual", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "failed", startedAt: "2026-10-26T00:00:00Z",
|
||||
errorCode: "native_provider_model_rejected", error: "The selected model was rejected", nativeIssueId: "website",
|
||||
@@ -12,3 +12,13 @@ it("rejects configuration rejection before native admission", () => { const o =
|
||||
it("rejects an unrelated prior failure", () => { const o = valid(); o.run.nativeIssueId = "other"; expect(admittedModelFailure(o)).toBe(false); });
|
||||
it("rejects a changed or substituted provider model", () => { const o = valid(); o.run.runnerProfileJson.nativeExecutionInput.provider.model = "supported"; expect(admittedModelFailure(o)).toBe(false); });
|
||||
it("rejects absent durable terminal events", () => { const o = valid(); o.events = []; expect(admittedModelFailure(o)).toBe(false); });
|
||||
|
||||
it("uses an explicit unavailable OpenRouter ID and requires actual OpenCode admission, never a Codex failure", () => {
|
||||
const model = unavailableNativeModel("opencode", "owned-nonce");
|
||||
expect(model).toBe("openrouter/paperclip-eval/paperclip-eval-unavailable-owned-nonce");
|
||||
const o = { ...valid(), provider: "opencode" as const, model };
|
||||
o.run.runnerProfileJson.nativeExecutionInput.provider = { kind: "opencode", model };
|
||||
expect(admittedModelFailure(o)).toBe(true);
|
||||
o.run.runnerProfileJson.nativeExecutionInput.provider.kind = "codex";
|
||||
expect(admittedModelFailure(o)).toBe(false);
|
||||
});
|
||||
@@ -9,12 +9,12 @@ interface NativeFailureRun extends ChatRun {
|
||||
runnerProfileJson?: { nativeExecutionInput?: { provider?: { kind?: string; model?: string | null } } };
|
||||
}
|
||||
interface NativeEvent { seq?: number; eventType?: string; payload?: { prpEvent?: { eventType?: string } } }
|
||||
export function admittedModelFailure(input: { run?: NativeFailureRun; events: NativeEvent[]; companyId: string; agentId: string; issueId: string; model: string }) {
|
||||
export function admittedModelFailure(input: { run?: NativeFailureRun; events: NativeEvent[]; companyId: string; agentId: string; issueId: string; model: string; provider?: "codex" | "opencode" }) {
|
||||
const r = input.run, provider = r?.runnerProfileJson?.nativeExecutionInput?.provider;
|
||||
return !!r && r.runtimeMode === "native" && r.status === "failed" && !!r.startedAt && !!r.errorCode && !!r.error
|
||||
&& r.companyId === input.companyId && r.agentId === input.agentId
|
||||
&& [r.nativeIssueId, r.contextSnapshot?.issueId, r.contextSnapshot?.taskId].includes(input.issueId)
|
||||
&& provider?.kind === "codex" && provider.model === input.model
|
||||
&& provider?.kind === (input.provider ?? "codex") && provider.model === input.model
|
||||
&& input.events.some(event => [event.eventType, event.payload?.prpEvent?.eventType].some(type => type === "run.terminal" || type === "turn.failed"));
|
||||
}
|
||||
/** A real public configuration/invocation fixture, never a synthetic failure.
|
||||
@@ -28,9 +28,10 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor
|
||||
const endpoint = `/api/agents/${worker.id}`;
|
||||
const original = await ctx.input.api.get<RuntimeAgent>(endpoint);
|
||||
if (original.lifecycleState !== "ready" || !["active", "idle"].includes(original.status ?? "")
|
||||
|| original.adapterType !== "paperclip_runner" || original.adapterConfig?.provider !== "codex")
|
||||
throw new Error("Native failure prerequisite requires an actually ready native Codex worker");
|
||||
const before = await ctx.allRuns(); const model = `paperclip-eval-unavailable-${ctx.input.nonce}`;
|
||||
|| original.adapterType !== "paperclip_runner" || !["codex", "opencode"].includes(String(original.adapterConfig?.provider)))
|
||||
throw new Error("Native failure prerequisite requires an actually ready selected native Codex or OpenCode worker");
|
||||
const provider = original.adapterConfig!.provider as "codex" | "opencode";
|
||||
const before = await ctx.allRuns(); const model = unavailableNativeModel(provider, ctx.input.nonce);
|
||||
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
|
||||
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true,
|
||||
isActive: true, unpricedUsagePolicy: "allow" });
|
||||
@@ -48,7 +49,7 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor
|
||||
: rows.some(run => ["succeeded", "cancelled", "timed_out"].includes(run.status)) ? "Invocation did not produce genuine native failed history" : undefined });
|
||||
failed = await ctx.input.api.get<NativeFailureRun>(`/api/heartbeat-runs/${settled[0]!.id}`);
|
||||
const events = await collectRunEvents<NativeEvent>((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${failed!.id}/events?afterSeq=${afterSeq}&limit=${limit}`));
|
||||
if (!admittedModelFailure({ run: failed, events, companyId: ctx.input.fixtures.company.id, agentId: worker.id, issueId: receipt.issueId, model }))
|
||||
if (!admittedModelFailure({ run: failed, events, companyId: ctx.input.fixtures.company.id, agentId: worker.id, issueId: receipt.issueId, model, provider }))
|
||||
throw new Error("Actual invocation lacks admitted model-bound native failure evidence; no failure coverage claimed");
|
||||
return { failedRun: failed, failedRunEvents: events, failedIssueId: receipt.issueId, failedModel: model,
|
||||
failureKind: "native-failure-under-explicit-unavailable-model-configuration", originalConfiguration: original };
|
||||
@@ -60,3 +61,9 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor
|
||||
if (!isDeepStrictEqual(restored.adapterConfig, original.adapterConfig)) throw new Error("Original admitted model configuration was not restored");
|
||||
}
|
||||
}
|
||||
|
||||
/** OpenCode requires a provider-qualified ID; actual admission and persisted failure remain mandatory. */
|
||||
export function unavailableNativeModel(provider: "codex" | "opencode", nonce: string) {
|
||||
const id = `paperclip-eval-unavailable-${nonce}`;
|
||||
return provider === "opencode" ? `openrouter/paperclip-eval/${id}` : id;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { modelDiscoveryEndpoint } from "./runtime-runs-adapters.js";
|
||||
import { it, expect } from "vitest";
|
||||
import { gradeModelOptions } from "./runtime-runs-adapters.js";
|
||||
function valid() {
|
||||
@@ -19,3 +20,9 @@ it("rejects silently changing Morgan's selected model", () => {
|
||||
it("rejects leaked private account credentials", () => {
|
||||
const o = valid(); o.state.secretLeaked = true; expect(gradeModelOptions(o).passed).toBe(false);
|
||||
});
|
||||
|
||||
it("discovers the actual selected native provider/environment rather than a Codex alias", () => {
|
||||
const worker = { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" }, defaultEnvironmentId: "owned-environment" };
|
||||
expect(modelDiscoveryEndpoint("company", worker as never)).toBe("/api/companies/company/adapters/paperclip_runner/models?provider=opencode&environmentId=owned-environment");
|
||||
expect(() => modelDiscoveryEndpoint("company", { ...worker, adapterConfig: { provider: "unknown" } } as never)).toThrow();
|
||||
});
|
||||
@@ -27,9 +27,7 @@ export function gradeModelOptions(value: unknown) {
|
||||
}
|
||||
export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-model-options", async ctx => {
|
||||
const worker = await runtimeWorker(ctx);
|
||||
if (worker.adapterConfig?.provider !== "codex" && worker.adapterType !== "codex_local")
|
||||
throw new Error("Current runtime model-discovery fixture requires a genuinely qualified Codex account");
|
||||
const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/adapters/codex_local/models`;
|
||||
const endpoint = modelDiscoveryEndpoint(ctx.input.fixtures.company.id, worker);
|
||||
const discovered = await readAgentResource(ctx, endpoint);
|
||||
const restricted = await readAgentResource(ctx, `/api/agents/${worker.id}/runtime-state`);
|
||||
if (discovered.status !== 200 || !Array.isArray(discovered.data)) throw new Error("Scenario requires admitted real coding model discovery");
|
||||
@@ -51,3 +49,9 @@ export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-m
|
||||
await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").filter({ hasText: match.id }).last()).toBeVisible();
|
||||
},
|
||||
})];
|
||||
|
||||
export function modelDiscoveryEndpoint(companyId: string, worker: RuntimeAgent) {
|
||||
if (worker.adapterType !== "paperclip_runner" || !["codex", "opencode"].includes(String(worker.adapterConfig?.provider)) || !worker.defaultEnvironmentId)
|
||||
throw new Error("Model discovery requires the actually selected native provider and environment");
|
||||
return `/api/companies/${encodeURIComponent(companyId)}/adapters/paperclip_runner/models?provider=${worker.adapterConfig!.provider}&environmentId=${encodeURIComponent(worker.defaultEnvironmentId)}`;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
|
||||
import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -118,3 +119,11 @@ it("rejects a missing extensionless component instead of binding an unrelated TS
|
||||
f.write("OtherComponent.tsx", 'export const value = "unrelated";');
|
||||
expect(f.fingerprint).toThrow("Unresolved fingerprint dependency: ./MissingComponent");
|
||||
});
|
||||
|
||||
it("admits installed OpenCode only for its exact reviewed model, case and source-local lane", () => {
|
||||
const lane = { suiteId: "chat-features", flow: "agent_chat", environment: "local", scenarioId: "instance-operations.update-instance", installedSource: "published-global-npm",
|
||||
profile: { id: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, adapterType: "paperclip_runner", credential: "OPENROUTER_API_KEY" } };
|
||||
expect(assertChatCatalogFingerprintLane(lane, {}, []).installedLane).toBe(true);
|
||||
expect(() => assertChatCatalogFingerprintLane({ ...lane, profile: { ...lane.profile, provider: "codex" } }, {}, [])).toThrow();
|
||||
expect(() => assertChatCatalogFingerprintLane({ ...lane, scenarioId: "tasks.create-backlog" }, { PAPERCLIP_RUNNER_E2E_INSTALLED_CLI: "/candidate" }, [])).toThrow();
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { reviewedChatProfile } from "./chat-runtime-profile.js";
|
||||
import { publicOriginFingerprintIdentity } from "./chat-public-origin.js";
|
||||
import { databaseFingerprintIdentity } from "./docker-postgres.js";
|
||||
import { verifyRetainedChatOracleIdentity } from "./chat-private-inputs.js";
|
||||
@@ -80,14 +81,14 @@ export function fingerprintChatDependencyClosure(input: {
|
||||
}
|
||||
|
||||
export function assertChatCatalogFingerprintLane(selected: unknown, env: NodeJS.ProcessEnv = process.env, args: readonly string[] = process.argv) {
|
||||
const execution = selected as { suiteId?: string; flow?: string; environment?: string; executionId?: string; scenarioId?: string; installedSource?: string; databaseLane?: unknown; profile?: { id?: string } };
|
||||
const execution = selected as { suiteId?: string; flow?: string; environment?: string; executionId?: string; scenarioId?: string; installedSource?: string; databaseLane?: unknown; profile?: { id?: string; provider?: string; model?: string; adapterType?: string; credential?: string } };
|
||||
const database = databaseFingerprintIdentity(env);
|
||||
if (execution.databaseLane !== undefined && JSON.stringify(execution.databaseLane) !== JSON.stringify(database)) throw new Error("Selected database lane differs from fingerprint environment");
|
||||
const installed = execution.installedSource === "published-global-npm" && (installedChatScenarioIds as readonly string[]).includes(String(execution.scenarioId));
|
||||
const installedKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_CLI", "PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256",
|
||||
"PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY"];
|
||||
const forbiddenArguments = ["--target-config", "--github-review-config", "--connection-config", "--installed-startup-only", "product-journeys"];
|
||||
if (installed && execution.profile?.id !== "runner-codex-subscription" || execution.suiteId !== "chat-features" || execution.flow !== "agent_chat" || execution.environment !== "local"
|
||||
if (installed && !reviewedChatProfile(execution.profile) || execution.suiteId !== "chat-features" || execution.flow !== "agent_chat" || execution.environment !== "local"
|
||||
|| !installed && installedKeys.some(key => env[key] && env[key] !== "0") || args.some(argument => forbiddenArguments.includes(argument)))
|
||||
throw new Error("Reviewed dynamic resolutions require source-local Agent Chat without external dispatch overrides");
|
||||
return { suiteId: execution.suiteId, flow: execution.flow, environment: execution.environment,
|
||||
|
||||
@@ -61,7 +61,7 @@ export function chatFeatureProfile(profile: RunnerProfileFixture): RunnerProfile
|
||||
return { ...payload, name: "CEO", title: "CEO", role: "ceo",
|
||||
capabilities: "Coordinates company work and delegates tasks to the team.",
|
||||
budgetMonthlyCents: CHAT_FEATURE_BUDGET_CENTS,
|
||||
adapterConfig: { ...adapterConfig, timeoutSec: 120, modelReasoningEffort: "low" },
|
||||
adapterConfig: { ...adapterConfig, timeoutSec: 120, ...(profile.provider === "codex" ? { modelReasoningEffort: "low" } : {}) },
|
||||
runtimeConfig: { ...(payload.runtimeConfig as Record<string, unknown>),
|
||||
heartbeat: { enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 3, maxDailyCostCents: CHAT_FEATURE_BUDGET_CENTS } },
|
||||
};
|
||||
|
||||
@@ -76,12 +76,13 @@ describe("chat feature saved-state oracle", () => {
|
||||
});
|
||||
|
||||
describe("chat feature selection", () => {
|
||||
it("preserves the two original queries as explicit local subscription cells", () => {
|
||||
it("preserves the two original queries as explicit host-local OpenCode cells", () => {
|
||||
const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && ["create-backlog", "delegate-announcement"].includes(cell.task.id));
|
||||
expect(cells).toHaveLength(2);
|
||||
for (const cell of cells) {
|
||||
expect(cell.environment.id).toBe("local");
|
||||
expect(cell.requiredCredentials).toEqual(["CODEX_AUTH_JSON"]);
|
||||
expect(cell.requiredCredentials).toEqual(["OPENROUTER_API_KEY"]);
|
||||
expect(cell.profile).toMatchObject({ id: "runner-opencode", provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017" });
|
||||
expect(cell.task.automaticRetryPolicy).toBe("single_attempt");
|
||||
expect(cell.task.buildPrompt("nonce")).toBe(chatFeatureScenarios.find(row => row.id === `tasks.${cell.task.id}`)!.query);
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id])).map(row => row.id)).toEqual([cell.id]);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, symlinkSync, rmSync, readFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import path from "node:path";
|
||||
@@ -15,6 +16,9 @@ it("admits only exact published pins and the two selected source-local Codex ope
|
||||
expect(() => validateChatInstallConfiguration({ schemaVersion: 1, disposable: true, source })).toThrow();
|
||||
const execution = { suite: { id: "chat-features", manualOnly: true }, task: { id: "instance-operations.update-instance", flow: "agent_chat" }, profile: { id: "runner-codex-subscription" }, environment: { id: "local" } } as MatrixExecution;
|
||||
expect(admitsInstalledChat(execution)).toBe(true);
|
||||
const remote = { ...execution, profile: { ...execution.profile, id: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, adapterType: "paperclip_runner", credential: "OPENROUTER_API_KEY" as const } };
|
||||
expect(admitsInstalledChat(remote)).toBe(true);
|
||||
expect(admitsInstalledChat({ ...remote, profile: { ...remote.profile, model: "other" } })).toBe(false);
|
||||
expect(admitsInstalledChat({ ...execution, task: { ...execution.task, id: "tasks.create-backlog" } })).toBe(false);
|
||||
expect(admitsInstalledChat({ ...execution, environment: { ...execution.environment, id: "daytona" } })).toBe(false);
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { reviewedChatProfile } from "./chat-runtime-profile.js";
|
||||
import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";
|
||||
import { spawn, execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -71,7 +72,7 @@ export function installedChatTreeIdentity(prefix: string) {
|
||||
}
|
||||
export function admitsInstalledChat(execution: Pick<MatrixExecution, "suite" | "task" | "profile" | "environment">) {
|
||||
return execution.suite.id === "chat-features" && execution.suite.manualOnly && execution.task.flow === "agent_chat"
|
||||
&& execution.profile.id === "runner-codex-subscription" && execution.environment.id === "local"
|
||||
&& reviewedChatProfile(execution.profile) && execution.environment.id === "local"
|
||||
&& (installedChatScenarioIds as readonly string[]).includes(execution.task.id);
|
||||
}
|
||||
/** Runs only inside an explicitly selected governed attempt; never during discovery. */
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { CHAT_OPENCODE_MODEL, reviewedChatProfile, reviewedChatExecutionIds } from "./chat-runtime-profile.js";
|
||||
import { runnerMatrix, runnerProfiles } from "./catalog.js";
|
||||
import { missingTeamAccountEnvironment } from "./chat-catalog-domains/external-team-configure.js";
|
||||
const profile = { id: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, adapterType: "paperclip_runner", credential: "OPENROUTER_API_KEY" };
|
||||
it("pins the new controller identity and rejects model/provider/credential substitution", () => {
|
||||
expect(reviewedChatProfile(profile)).toBe(true);
|
||||
for (const patch of [{ id: "runner-unknown" }, { provider: "codex" }, { model: "openrouter/other/model" }, { credential: "CODEX_AUTH_JSON" }, { adapterType: "opencode_local" }])
|
||||
expect(reviewedChatProfile({ ...profile, ...patch })).toBe(false);
|
||||
expect(reviewedChatExecutionIds("secrets.vault-reference")).toEqual(["chat-features.runner-codex-subscription.local.secrets.vault-reference", "chat-features.runner-opencode.local.secrets.vault-reference"]);
|
||||
});
|
||||
it("changes the complete chat matrix without relabeling the existing global qualification model", () => {
|
||||
const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features");
|
||||
expect(cells.length).toBe(244);
|
||||
expect(cells.every(cell => cell.profile.model === CHAT_OPENCODE_MODEL && reviewedChatProfile(cell.profile))).toBe(true);
|
||||
expect(runnerProfiles.find(row => row.id === "runner-opencode")!.model).not.toBe(CHAT_OPENCODE_MODEL);
|
||||
const cell = cells[0]!;
|
||||
const payload = cell.profile.buildAgent({ environmentId: "environment", environmentFixtureId: "local", workspacePath: "/owned", secretRefs: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: "22222222-2222-4222-8222-222222222222", version: "latest" } }, executionId: "owned" });
|
||||
expect(payload.adapterConfig).toMatchObject({ provider: "opencode", model: CHAT_OPENCODE_MODEL });
|
||||
expect(payload.adapterConfig).not.toHaveProperty("modelReasoningEffort");
|
||||
expect(payload.adapterConfig).not.toHaveProperty("codexPermissionMode");
|
||||
});
|
||||
it("withholds only the selected account from the missing OpenRouter member", () => {
|
||||
expect(missingTeamAccountEnvironment(false, "/owned/empty")).toEqual({ OPENROUTER_API_KEY: "" });
|
||||
expect(missingTeamAccountEnvironment(true, "/owned/empty")).toEqual({ CODEX_HOME: "/owned/empty", OPENAI_API_KEY: "", CODEX_API_KEY: "" });
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
/** Exact controller identity for the new remote campaign; historical Codex cells remain distinct. */
|
||||
export const CHAT_OPENCODE_PROFILE = "runner-opencode";
|
||||
export const CHAT_OPENCODE_MODEL = "openrouter/deepseek/deepseek-v4.1-flash-1017";
|
||||
export function reviewedChatProfile(profile: { id?: string; provider?: string; model?: string; adapterType?: string; credential?: string } | undefined) {
|
||||
return profile?.id === "runner-codex-subscription" || !!profile && profile.id === CHAT_OPENCODE_PROFILE
|
||||
&& profile.provider === "opencode" && profile.model === CHAT_OPENCODE_MODEL
|
||||
&& profile.adapterType === "paperclip_runner" && profile.credential === "OPENROUTER_API_KEY";
|
||||
}
|
||||
export function reviewedChatExecutionIds(scenarioId: string) {
|
||||
return ["runner-codex-subscription", CHAT_OPENCODE_PROFILE].map(profile => `chat-features.${profile}.local.${scenarioId}`);
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
validVaultBinding,
|
||||
vaultPrivateFields,
|
||||
vaultExecutionId,
|
||||
vaultExecutionIds,
|
||||
} from "./chat-vault-private-server.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
|
||||
@@ -216,3 +217,11 @@ it("scrubs ambient AWS and vault SDK config while preserving explicit unrelated
|
||||
]);
|
||||
expect(other.AWS_ACCESS_KEY_ID).toBe("ambient");
|
||||
});
|
||||
|
||||
it("binds the OpenCode vault envelope to the exact selected new execution, never the prior campaign", async () => {
|
||||
const f = await fixture();
|
||||
f.input.executionIds = [vaultExecutionIds[1]!]; f.envelope.executionId = vaultExecutionIds[1]!; await f.save();
|
||||
expect(await readSelectedVaultServerEnvironment(f.input)).toHaveProperty("AWS_ACCESS_KEY_ID", f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_ACCESS);
|
||||
f.envelope.executionId = vaultExecutionId; await f.save();
|
||||
await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow();
|
||||
});
|
||||
@@ -1,10 +1,12 @@
|
||||
import { reviewedChatExecutionIds } from "./chat-runtime-profile.js";
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, realpath, open, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
|
||||
|
||||
export const vaultScenarioId = "secrets.vault-reference";
|
||||
export const vaultExecutionId = `chat-features.runner-codex-subscription.local.${vaultScenarioId}`;
|
||||
export const vaultExecutionId = reviewedChatExecutionIds(vaultScenarioId)[0]!;
|
||||
export const vaultExecutionIds = reviewedChatExecutionIds(vaultScenarioId);
|
||||
export const vaultPrivateFields = [
|
||||
"provider",
|
||||
"region",
|
||||
@@ -102,7 +104,7 @@ export async function readSelectedVaultServerEnvironment(input: {
|
||||
backendOrigin: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): Promise<NodeJS.ProcessEnv> {
|
||||
if (!input.executionIds.includes(vaultExecutionId)) return {};
|
||||
if (!input.executionIds.some(id => vaultExecutionIds.includes(id))) return {};
|
||||
if (input.executionIds.length !== 1)
|
||||
throw new Error(
|
||||
"Vault runtime inputs require the exact single selected source execution",
|
||||
@@ -169,7 +171,7 @@ export async function readSelectedVaultServerEnvironment(input: {
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
if (envelope.version !== 1 || envelope.executionId !== vaultExecutionId)
|
||||
if (envelope.version !== 1 || envelope.executionId !== input.executionIds[0])
|
||||
throw new Error("Vault runtime input belongs to another execution");
|
||||
// Ordinary reserved setup handles missing selected provider inputs. It never
|
||||
// receives an ambient fallback merely because the private binding is absent.
|
||||
|
||||
Reference in new issue
Block a user