mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
fix(evals): keep fast response initial observation on original clock
This commit is contained in:
1 parent
1c7cbdc25b
commit
8cd9e8bc6b
3 files changed
+8
-6
No files matched your search
@@ -48,6 +48,7 @@ const originalClockExecutionIds = new Set([
|
||||
"chat-features.runner-opencode.local.secrets.configure-company-secret",
|
||||
"chat-features.runner-opencode.local.teams.configure-team",
|
||||
"chat-features.runner-opencode.local.steering.interrupt-with-direction",
|
||||
"chat-features.runner-opencode.local.companies.fast-responses",
|
||||
]);
|
||||
function usesOriginalNativePhaseClock(ctx: ChatCatalogContext): boolean {
|
||||
return originalClockExecutionIds.has(ctx.input?.execution?.id);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
|
||||
import { expect } from "@playwright/test";
|
||||
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import type { AiConnectionList, AiManagedConnectionSummary } from "../../../packages/shared/src/ai-connections.js";
|
||||
@@ -104,7 +105,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s
|
||||
}, gradeCompanyModel, {
|
||||
sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000,
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
|
||||
await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
|
||||
state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled;
|
||||
state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId,
|
||||
...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) });
|
||||
|
||||
@@ -17,22 +17,22 @@ describe("selected native phases share the once-created original flow clock", ()
|
||||
vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000);
|
||||
expect(ctx.observationDeadlineAt).toBe(870_000);
|
||||
});
|
||||
it.each(["instance-operations.backup-and-restore", "instance-operations.restart-service", "routines.webhook-trigger", "status-cards.explain-card", "secrets.configure-company-secret", "teams.configure-team", "steering.interrupt-with-direction"])("future13 %s shares the setup-inclusive original clock", scenario => {
|
||||
it.each(["instance-operations.backup-and-restore", "instance-operations.restart-service", "routines.webhook-trigger", "status-cards.explain-card", "secrets.configure-company-secret", "teams.configure-team", "steering.interrupt-with-direction", "companies.fast-responses"])("future13 %s shares the setup-inclusive original clock", scenario => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario);
|
||||
expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000);
|
||||
expect(chatCatalogOriginalPhaseDeadline(ctx)).toBe(870_000);
|
||||
vi.setSystemTime(869_999); expect(chatCatalogNativePhaseRemainingBudget(ctx)!()).toBe(1);
|
||||
vi.setSystemTime(870_000); expect(() => chatCatalogOriginalPhaseDeadline(ctx)).toThrow(/exhausted/);
|
||||
});
|
||||
it("actual identity helper settles after its old150s cap using only the original remaining clock", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(50_000); const ctx = context("secrets.configure-company-secret", 450_000);
|
||||
it.each(["secrets.configure-company-secret", "companies.fast-responses"])("actual %s identity helper settles after its old150s cap using only the original remaining clock", async scenario => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(50_000); const originalDeadline = scenario === "companies.fast-responses" ? 390_000 : 450_000; const ctx = context(scenario, originalDeadline);
|
||||
Object.assign(ctx, { refreshIssue: async () => {}, allRuns: async () => [{ status: Date.now() >= 230_000 ? "succeeded" : "running" }] });
|
||||
let completed = false; const pending = waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)).then(() => { completed = true; });
|
||||
await vi.advanceTimersByTimeAsync(152_000); expect(completed).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(28_000); await pending; expect(completed).toBe(true);
|
||||
expect(ctx.observationDeadlineAt).toBe(450_000);
|
||||
expect(ctx.observationDeadlineAt).toBe(originalDeadline);
|
||||
});
|
||||
it.each(["onboarding.first-work", "routines.plugin-managed-routine", "status-cards.archive-card", "status-cards.other-authors-card"])("future13 %s does not receive new phase or UI behavior", scenario => {
|
||||
it.each(["onboarding.first-work", "routines.plugin-managed-routine", "status-cards.archive-card", "status-cards.other-authors-card", "companies.prompt-injection-guard", "companies.configure-decision-model"])("future13 %s does not receive new phase or UI behavior", scenario => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario, 1);
|
||||
expect(chatCatalogNativePhaseDeadline(ctx, 900_000)).toBe(900_000);
|
||||
expect(chatCatalogOriginalPhaseDeadline(ctx)).toBeUndefined();
|
||||
|
||||
Reference in new issue
Block a user