mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
fix(evals): admit verified missing personal secret prerequisite
Prove the controller and worker ready before introducing the missing required current-person secret. Preserve that specific scenario only after exact owner-scoped missing-binding resolution; reject unrelated lifecycle, hold, identity and configuration failures. Keep feature grading, human secure entry, native delivery, clocks and privacy assertions unchanged. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
bb33df5320
commit
78d91b0e2f
2 files changed
+120
-8
No files matched your search
@@ -0,0 +1,74 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import { personalSecretExternalCases, personalSecretUnboundConfigHash, preparePersonalSecretWorker, provePersonalSecretMissingBoundary } from "./external-personal-secret.js";
|
||||
afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); });
|
||||
function fixture() {
|
||||
vi.useFakeTimers(); vi.setSystemTime(100);
|
||||
const config = { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", env: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: "controller", version: "latest" } } };
|
||||
const ready = { id: "morgan", companyId: "company", lifecycleState: "ready", lifecycleHolds: [] as string[], lifecycleError: null as string | null, status: "idle", adapterType: "paperclip_runner", adapterConfig: config };
|
||||
const agent = { ...ready, adapterConfig: { ...config, env: { ...config.env, PUBLISHING_API_KEY: { type: "user_secret_ref", key: "PERSONAL_PUBLISHING_KEY", version: "latest", required: true, allowMissingOverride: false } } } };
|
||||
const lead = { ...ready, id: "ceo" }, evidence = vi.fn();
|
||||
const get = vi.fn(async (route: string) => route === "/api/agents/ceo" ? lead : ready);
|
||||
const ctx = { observationDeadlineAt: 10000, input: { api: { baseURL: "http://127.0.0.1:3100", get }, evidence, fixtures: { company: { id: "company" }, agent: { id: "ceo" } }, page: { context: () => ({ cookies: async () => [{ name: "session", value: "synthetic-private-session" }] }) } } } as unknown as ChatCatalogContext;
|
||||
const state = { morganId: "morgan", morganReadyBeforeRequiredBinding: true, morganUnboundConfigHash: personalSecretUnboundConfigHash(config), definitionId: "definition", currentPersonId: "person" };
|
||||
const fetcher = vi.fn(async (_input: string | URL | Request, _init?: RequestInit) => new Response(JSON.stringify({ code: "user_secret_missing", details: { code: "user_secret_missing", definitionId: "definition", responsibleUserId: "person" } }), { status: 422 }));
|
||||
vi.stubGlobal("fetch", fetcher);
|
||||
return { ctx, ready, agent, lead, state, get, fetcher, evidence };
|
||||
}
|
||||
it("preserves the exact missing-value scenario instead of demanding every configured worker ready", () => {
|
||||
expect(personalSecretExternalCases[0]!.setupPolicy).toEqual({ readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] });
|
||||
expect(personalSecretExternalCases[0]!.verifySetup).toBeTypeOf("function");
|
||||
});
|
||||
it("proves the worker ready before adding the missing required value and does no provider probe", async () => {
|
||||
const f = fixture(); await expect(preparePersonalSecretWorker(f.ctx, "morgan")).resolves.toEqual(f.ready);
|
||||
expect(f.fetcher).not.toHaveBeenCalled();
|
||||
f.ready.lifecycleError = "Authentication failed";
|
||||
await expect(preparePersonalSecretWorker(f.ctx, "morgan")).rejects.toThrow("unrelated setup failure");
|
||||
});
|
||||
it("requires exact current-person missing-secret resolution and ready CEO with unchanged worker configuration", async () => {
|
||||
const f = fixture(); await provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent);
|
||||
expect(f.fetcher).toHaveBeenCalledOnce();
|
||||
expect(f.fetcher.mock.calls[0]?.[0]).toEqual(new URL("http://127.0.0.1:3100/api/companies/company/adapters/paperclip_runner/test-environment"));
|
||||
expect(f.evidence).toHaveBeenCalledWith("personal-secret-missing-binding-prerequisite.json", expect.objectContaining({ status: 422, code: "user_secret_missing", prospectiveResolutionOnly: true }));
|
||||
expect(JSON.stringify(f.evidence.mock.calls)).not.toContain("synthetic-private-session");
|
||||
});
|
||||
it.each(["bad CEO", "bad worker", "budget hold", "wrong company", "changed model", "optional", "override", "wrong definition", "never ready"])("rejects unrelated %s before any resolution request", async invalid => {
|
||||
const f = fixture();
|
||||
if (invalid === "bad CEO") f.lead.lifecycleError = "Authentication failed";
|
||||
if (invalid === "bad worker") f.agent.lifecycleError = "Authentication failed";
|
||||
if (invalid === "budget hold") f.agent.lifecycleHolds = ["budget"];
|
||||
if (invalid === "wrong company") f.agent.companyId = "foreign";
|
||||
if (invalid === "changed model") f.agent.adapterConfig.model = "other";
|
||||
if (invalid === "optional") f.agent.adapterConfig.env.PUBLISHING_API_KEY.required = false;
|
||||
if (invalid === "override") f.agent.adapterConfig.env.PUBLISHING_API_KEY.allowMissingOverride = true;
|
||||
if (invalid === "wrong definition") f.agent.adapterConfig.env.PUBLISHING_API_KEY.key = "OTHER";
|
||||
if (invalid === "never ready") f.state.morganReadyBeforeRequiredBinding = false;
|
||||
await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow();
|
||||
expect(f.fetcher).not.toHaveBeenCalled(); expect(f.evidence).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each(["other status", "other code", "other definition", "other person", "missing details"])("refuses %s rather than treating any failure as the intended prerequisite", async invalid => {
|
||||
const f = fixture(), body = { code: "user_secret_missing", details: { code: "user_secret_missing", definitionId: "definition", responsibleUserId: "person" } };
|
||||
if (invalid === "other code") body.code = "harness_auth_failed";
|
||||
if (invalid === "other definition") body.details.definitionId = "other";
|
||||
if (invalid === "other person") body.details.responsibleUserId = "other";
|
||||
if (invalid === "missing details") delete (body as { details?: unknown }).details;
|
||||
f.fetcher.mockImplementation(async () => new Response(JSON.stringify(body), { status: invalid === "other status" ? 500 : 422 }));
|
||||
await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow("exact current-person missing binding");
|
||||
expect(f.evidence).not.toHaveBeenCalled();
|
||||
});
|
||||
it("does not let a slow resolution reset the original observation clock", async () => {
|
||||
const f = fixture(); f.fetcher.mockImplementation(async () => { vi.setSystemTime(10001); return new Response(JSON.stringify({ code: "user_secret_missing", details: { code: "user_secret_missing", definitionId: "definition", responsibleUserId: "person" } }), { status: 422 }); });
|
||||
await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow("budget");
|
||||
expect(f.evidence).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("accepts only the source-owned missing-binding lifecycle state once the precise resolution guard proves it", async () => {
|
||||
const f = fixture(); f.agent.lifecycleState = "verifying"; f.agent.status = "paused";
|
||||
f.agent.lifecycleError = "Execution configuration check failed. Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator.";
|
||||
await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).resolves.toBeUndefined();
|
||||
});
|
||||
it("does not expose malformed private response contents", async () => {
|
||||
const f = fixture(); f.fetcher.mockImplementation(async () => new Response("synthetic-private-invalid-json", { status: 422 }));
|
||||
await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow("contents withheld");
|
||||
expect(f.evidence).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { expect } from "@playwright/test";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import { chatCatalogAgentIsReady, type ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import type {
|
||||
CompanySecret,
|
||||
UserSecretDefinition,
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
openStandingChat,
|
||||
probeAgentPermission,
|
||||
waitForReply,
|
||||
remainingIdentityObservationBudget,
|
||||
} from "./identity-support.js";
|
||||
import {
|
||||
createAuthenticatedPerson,
|
||||
@@ -47,6 +48,43 @@ interface PrivateFixture {
|
||||
endpoint: Awaited<ReturnType<typeof secretTestEndpoint>>;
|
||||
}
|
||||
const fixtures = new WeakMap<ChatCatalogContext, PrivateFixture>();
|
||||
interface PersonalSetupAgent { id: string; companyId: string; lifecycleState: string; status: string; lifecycleError?: string | null; lifecycleHolds: string[]; adapterType: string; adapterConfig: Record<string, unknown> }
|
||||
const missingBindingLifecycleError = "Execution configuration check failed. Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator.";
|
||||
export function personalSecretUnboundConfigHash(config: Record<string, unknown>): string {
|
||||
const env = { ...(config.env as Record<string, unknown> | undefined) }; delete env.PUBLISHING_API_KEY;
|
||||
return createHash("sha256").update(JSON.stringify({ ...config, env })).digest("hex");
|
||||
}
|
||||
/** A configured worker must be healthy before introducing the intentional missing value. */
|
||||
export async function preparePersonalSecretWorker(ctx: ChatCatalogContext, id: string): Promise<PersonalSetupAgent> {
|
||||
const deadlineAt = Date.now() + Math.min(45_000, remainingIdentityObservationBudget(ctx));
|
||||
const agent = await pollUntil<PersonalSetupAgent>({ label: "personal-secret worker ready before required binding", deadlineAt, intervalMs: 500,
|
||||
load: () => ctx.input.api.get(`/api/agents/${id}`),
|
||||
accept: value => chatCatalogAgentIsReady(value), reject: value => value.lifecycleError ? "Personal-secret worker has an unrelated setup failure before its required binding" : undefined });
|
||||
remainingIdentityObservationBudget(ctx);
|
||||
if (agent.id !== id || agent.companyId !== ctx.input.fixtures.company.id || agent.adapterType !== "paperclip_runner" || Object.hasOwn(agent.adapterConfig.env ?? {}, "PUBLISHING_API_KEY")) throw new Error("Personal-secret prerequisite worker identity or initial binding differs");
|
||||
return agent;
|
||||
}
|
||||
/** This is a prospective saved-config resolution check. The exact missing user
|
||||
* secret response is raised before adapter execution or environment acquisition. */
|
||||
export async function provePersonalSecretMissingBoundary(ctx: ChatCatalogContext, state: Record<string, unknown>, agent: PersonalSetupAgent): Promise<void> {
|
||||
remainingIdentityObservationBudget(ctx);
|
||||
const lead = await ctx.input.api.get<PersonalSetupAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
|
||||
if (!chatCatalogAgentIsReady(lead) || lead.id !== ctx.input.fixtures.agent.id || lead.companyId !== ctx.input.fixtures.company.id || lead.lifecycleError) throw new Error("Personal-secret CEO must remain independently ready");
|
||||
const ref = (agent.adapterConfig.env as Record<string, unknown> | undefined)?.PUBLISHING_API_KEY as { type?: string; key?: string; required?: boolean; allowMissingOverride?: boolean } | undefined;
|
||||
if (!["ready", "preparing", "verifying", "pausing"].includes(agent.lifecycleState) || !Array.isArray(agent.lifecycleHolds) || agent.lifecycleHolds.some(hold => hold !== "setup") || state.morganReadyBeforeRequiredBinding !== true || agent.id !== state.morganId || agent.companyId !== ctx.input.fixtures.company.id || agent.adapterType !== "paperclip_runner" || personalSecretUnboundConfigHash(agent.adapterConfig) !== state.morganUnboundConfigHash || ref?.type !== "user_secret_ref" || ref.key !== "PERSONAL_PUBLISHING_KEY" || ref.required !== true || ref.allowMissingOverride !== false || (agent.lifecycleError && agent.lifecycleError !== missingBindingLifecycleError)) throw new Error("Personal-secret worker has an unrelated lifecycle or configuration failure");
|
||||
const origin = ctx.input.api.baseURL, cookies = await ctx.input.page.context().cookies(origin);
|
||||
if (!cookies.length) throw new Error("Personal-secret setup probe requires the real current-person session");
|
||||
const response = await fetch(new URL(`/api/companies/${ctx.input.fixtures.company.id}/adapters/paperclip_runner/test-environment`, origin), { method: "POST", redirect: "error", signal: AbortSignal.timeout(Math.min(15_000, remainingIdentityObservationBudget(ctx))), headers: { "content-type": "application/json", Origin: origin, Cookie: cookies.map(cookie => `${cookie.name}=${cookie.value}`).join("; ") }, body: JSON.stringify({ agentId: agent.id, adapterConfig: agent.adapterConfig }) });
|
||||
try {
|
||||
let body: { code?: string; details?: { code?: string; definitionId?: string; responsibleUserId?: string } };
|
||||
try { body = await response.json(); } catch { throw new Error("Personal-secret setup response could not be verified; contents withheld"); }
|
||||
if (!body || typeof body !== "object" || response.status !== 422 || body.code !== "user_secret_missing" || body.details?.code !== "user_secret_missing" || body.details.definitionId !== state.definitionId || body.details.responsibleUserId !== state.currentPersonId) throw new Error("Personal-secret setup did not prove the exact current-person missing binding");
|
||||
} finally { if (!response.bodyUsed) await response.body?.cancel(); }
|
||||
remainingIdentityObservationBudget(ctx);
|
||||
state.currentOwnerMissingBindingVerified = true;
|
||||
await ctx.input.evidence("personal-secret-missing-binding-prerequisite.json", { agentId: agent.id, definitionId: state.definitionId, responsibleUserId: state.currentPersonId, status: 422, code: "user_secret_missing", unchangedWorkerConfiguration: true, controllerReady: true, prospectiveResolutionOnly: true });
|
||||
}
|
||||
|
||||
export function personalSecretGuidance(reply: string) {
|
||||
return (
|
||||
/personal|your own|my secrets/i.test(reply) &&
|
||||
@@ -235,9 +273,7 @@ export const personalSecretExternalCases = [
|
||||
"Personal publishing definition must initially lack the current person's value",
|
||||
);
|
||||
const morgan = await identityWorker(ctx),
|
||||
agent = await ctx.input.api.get<{
|
||||
adapterConfig: Record<string, unknown>;
|
||||
}>(`/api/agents/${morgan.id}`);
|
||||
agent = await preparePersonalSecretWorker(ctx, morgan.id);
|
||||
await ctx.input.api.patch(`/api/agents/${morgan.id}`, {
|
||||
adapterConfig: {
|
||||
...agent.adapterConfig,
|
||||
@@ -277,6 +313,8 @@ export const personalSecretExternalCases = [
|
||||
description: `This request concerns publishing task ${task.identifier ?? task.id}, assigned to Morgan. It is in backlog because the current responsible person has not set the required Personal publishing credential definition ${definition.id} (PERSONAL_PUBLISHING_KEY). Morgan's required PUBLISHING_API_KEY user-secret reference is already configured. Riley is another company member with their own separate personal value.`,
|
||||
});
|
||||
return {
|
||||
morganReadyBeforeRequiredBinding: true,
|
||||
morganUnboundConfigHash: personalSecretUnboundConfigHash(agent.adapterConfig),
|
||||
definitionId: definition.id,
|
||||
currentPersonId: session.user.id,
|
||||
otherPersonId: other.id,
|
||||
@@ -307,6 +345,7 @@ export const personalSecretExternalCases = [
|
||||
gradePersonalSecret,
|
||||
{
|
||||
authentication: "human-session",
|
||||
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] },
|
||||
expectedRunCount: 3,
|
||||
minimumRunCount: 3,
|
||||
expectedHumanMessageCount: 2,
|
||||
@@ -340,9 +379,7 @@ export const personalSecretExternalCases = [
|
||||
session = await ctx.input.api.get<{ user: { id: string } }>(
|
||||
"/api/auth/get-session",
|
||||
),
|
||||
agent = await ctx.input.api.get<{
|
||||
adapterConfig: { env?: Record<string, unknown> };
|
||||
}>(`/api/agents/${state.morganId}`),
|
||||
agent = await ctx.input.api.get<PersonalSetupAgent>(`/api/agents/${state.morganId}`),
|
||||
ref = agent.adapterConfig.env?.PUBLISHING_API_KEY as
|
||||
| {
|
||||
type?: string;
|
||||
@@ -351,6 +388,7 @@ export const personalSecretExternalCases = [
|
||||
allowMissingOverride?: boolean;
|
||||
}
|
||||
| undefined;
|
||||
await provePersonalSecretMissingBoundary(ctx, state, agent);
|
||||
state.requiredReferenceBefore =
|
||||
ref?.type === "user_secret_ref" &&
|
||||
ref.key === "PERSONAL_PUBLISHING_KEY" &&
|
||||
|
||||
Reference in new issue
Block a user