mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
test(evals): grade four chat cases by saved state and setup literals only
Remove the remaining reply-vocabulary checks. Reply meaning is read by a person; the oracles keep: - update-case-facts: saved browser tokens, plus a follow-up reply that names the case or its task and the setup's escalation literal; - read-history: links to both tasks and the recorded blocked/failed and done/completed statuses; - managed-profile: the 403s, no profile or workaround written, and a reference to the place the board acts (managed-agent setup or the profile route); - gated-lab: the route literal and no link to an unrelated task. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed
1 parent
1b48ecd285
commit
524e2f3c76
9 files changed
+57
-169
No files matched your search
@@ -697,7 +697,7 @@
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-scenarios/agents.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v4",
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v5",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -1442,7 +1442,7 @@
|
||||
"chat-catalog-domains/internal-definition.ts",
|
||||
"chat-scenarios/cases.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.case-facts.v2",
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.case-facts.v3",
|
||||
"expectedRunCount": 4,
|
||||
"expectedHumanMessageCount": 2,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -2696,7 +2696,7 @@
|
||||
"../../ui/src/pages/NotFound.tsx",
|
||||
"chat-query-attachments.ts"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.privileged.gated-lab.v3",
|
||||
"oracleVersion": "paperclip.chat-catalog.privileged.gated-lab.v4",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -4688,7 +4688,7 @@
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-scenarios/routines.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v4",
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.routine-work-failure.v5",
|
||||
"expectedRunCount": 3,
|
||||
"expectedHumanMessageCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
|
||||
@@ -36,10 +36,8 @@ describe("current case facts and real subsequent context", () => {
|
||||
const moved = valid(); moved.brief.issueId = "case"; expect(gradeIntegratorCaseFacts(moved).passed).toBe(false);
|
||||
const missing = valid(); missing.briefRevisions = []; expect(gradeIntegratorCaseFacts(missing).passed).toBe(false);
|
||||
});
|
||||
it("rejects a plausible contrary chat report and missing collaborative acknowledgement", () => {
|
||||
for (const reply of ["No. Your mobile Safari only correction on COM-C1 was overwritten by support. Escalation: support-123.",
|
||||
"COM-C1 no longer includes your browser correction. Support escalation is support-123.",
|
||||
"Yes, COM-C1 still says mobile Safari only. The escalation is support-999.",
|
||||
it("rejects a follow-up without the case reference or the saved escalation literal", () => {
|
||||
for (const reply of ["Yes, COM-C1 still says mobile Safari only. The escalation is support-999.",
|
||||
"Yes, COM-C1 still says mobile Safari only.",
|
||||
"Yes, it still says mobile Safari only. Support escalation is support-123.",
|
||||
`${FIRST}`])
|
||||
|
||||
@@ -40,8 +40,8 @@ export function corrected(fields: unknown) {
|
||||
|| platformSplit && typeof value === "string" && tokens(value).join(" ") === "safari");
|
||||
}
|
||||
/** The follow-up reply is the text added after the first checkpoint. It must name the case (its identifier or its
|
||||
* primary task) and the setup-chosen support escalation, and must not contradict the saved state: no other
|
||||
* escalation value, no "No" answer, and no sentence saying the correction was lost. Saved fields are the proof. */
|
||||
* primary task) and the setup-chosen support escalation literal. Whether its wording is right is free-form text that a
|
||||
* person reads; the saved fields above are the proof of the outcome. */
|
||||
export function latestReply(fresh: InternalObservation, first: InternalObservation | undefined) {
|
||||
return first && fresh.reply.startsWith(first.reply) ? fresh.reply.slice(first.reply.length) : fresh.reply;
|
||||
}
|
||||
@@ -50,11 +50,7 @@ export function reportsCurrentFacts(reply: string, o: FactsObservation) {
|
||||
const caseRow = o.state.case as CaseRecord & { identifier?: string }; const task = o.tasks.find(row => row.id === caseRow.workIssueId);
|
||||
const names = [caseRow.identifier, task?.identifier, caseRow.id, caseRow.workIssueId].filter((value): value is string => !!value);
|
||||
const refersToCase = names.some(name => new RegExp(`(?<![\\w-])${escapeRe(name)}(?![\\w-])`, "i").test(reply));
|
||||
const value = String(o.state.collaboratorValue); const otherEscalation = (reply.match(/\bsupport-[\w-]+/gi) ?? []).some(found => found !== value);
|
||||
const answersNo = /^\W*no\b/i.test(reply.trim());
|
||||
const lost = reply.split(/\n+|(?<=[.!?])\s+/).some(sentence => /\b(?:overwr[io]te|overwritten|lost|removed|reverted|dropped|discarded|no longer|missing|(?:is|was|are|were)n['’]?t (?:saved|kept|included|there|present)|not (?:saved|kept|included|there|present)|does(?:n['’]t| not) (?:include|contain|have))\b/i.test(sentence)
|
||||
&& /\b(?:correction|mobile Safari|browser|reproduction step)\b/i.test(sentence) && !/\b(?:nothing|not|never|wasn['’]?t|none)\b[^.!?]{0,30}\b(?:overwr|lost|removed|reverted|dropped|discarded)/i.test(sentence));
|
||||
return refersToCase && reply.includes(value) && !otherEscalation && !answersNo && !lost;
|
||||
return refersToCase && reply.includes(String(o.state.collaboratorValue));
|
||||
}
|
||||
function retained(o: FactsObservation, concurrent: boolean) {
|
||||
const original = o.state.case as CaseRecord; const fields = o.caseRecord.fields as Record<string, unknown>;
|
||||
@@ -69,7 +65,7 @@ export function gradeIntegratorCaseFacts(value: unknown) {
|
||||
const first = factsObservation(o.state.firstCorrection); const fresh = factsObservation(o.state.collaborationCheckpoint);
|
||||
c.add("initial-correction-preserves-unrelated-fields", !!first && retained(first, false), "The unchanged first query must already correct the saved facts without discarding unrelated fields; a later turn cannot repair an initially wrong mutation.");
|
||||
c.add("fresh-collaborative-change-not-stale-replacement", !!fresh && retained(fresh, true) && retained(o, true)
|
||||
&& reportsCurrentFacts(latestReply(fresh, first), o), "A real independently saved support change survives the explicit reread; the follow-up reply names the case and the actual concurrent field and does not contradict the saved correction.");
|
||||
&& reportsCurrentFacts(latestReply(fresh, first), o), "A real independently saved support change survives the explicit reread; the follow-up reply names the case and the actual concurrent field.");
|
||||
const before = o.state.brief as IssueDocument; const original = o.state.case as CaseRecord;
|
||||
c.add("ordinary-task-document-ownership-and-history", o.brief.id === before.id && o.brief.issueId === original.workIssueId
|
||||
&& o.briefRevisions.some(revision => revision.id === before.latestRevisionId && revision.issueId === original.workIssueId && revision.body === before.body), "The original ordinary-task document and its actual saved revision remain owned and accessible on the same primary task.");
|
||||
@@ -114,7 +110,7 @@ export const integratorCaseFactsCases = [integratorDefinition("cases", "cases.up
|
||||
const row = state.case as CaseRecord; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title);
|
||||
const task = await ctx.input.api.get<Issue>(`/api/issues/${row.workIssueId}`);
|
||||
await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier}`, task.title);
|
||||
}, { oracleVersion: "paperclip.chat-catalog.integrator.case-facts.v2", expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } },
|
||||
}, { oracleVersion: "paperclip.chat-catalog.integrator.case-facts.v3", expectedRunCount: 4, expectedHumanMessageCount: 2, attemptTimeoutMs: 900_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 3 } },
|
||||
sourceFiles: ["chat-catalog-domains/integrator-case-facts.ts", "chat-catalog-domains/internal-run-brief.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], cleanup,
|
||||
async act(ctx, state) {
|
||||
const settle = (count: number) => pollUntil({ label: `saved case correction phase ${count}`, deadlineAt: Date.now() + 180_000, intervalMs: 1000,
|
||||
|
||||
@@ -72,13 +72,13 @@ describe("real routine work failure with successful native turns", () => {
|
||||
["forged worker report", p => { p.snapshot.comments[0]!.createdByRunId = "other"; }],
|
||||
];
|
||||
it.each(mutations)("rejects %s", (_name, mutate) => { const p = phase(true); mutate(p); expect(genuineHistoryWork(p.identity, p.snapshot, p.calls, true)).toBe(false); });
|
||||
it("retains week, separate task, failure/success, link and schedule-grounding controls", () => {
|
||||
it("retains week, separate task, recorded failure/success and link controls", () => {
|
||||
for (const mutate of [
|
||||
(v: ReturnType<typeof valid>) => { v.executions[0]!.triggeredAt = "2026-10-19T00:00:00Z" as never; },
|
||||
(v: ReturnType<typeof valid>) => { v.executions.push(v.executions[0]!); },
|
||||
(v: ReturnType<typeof valid>) => { v.links.shift(); },
|
||||
(v: ReturnType<typeof valid>) => { v.reply = "All runs succeeded this week."; },
|
||||
(v: ReturnType<typeof valid>) => { v.reply += " Both scheduled morning summaries ran successfully."; },
|
||||
(v: ReturnType<typeof valid>) => { v.tasks[0]!.status = "done"; },
|
||||
(v: ReturnType<typeof valid>) => { v.executions[1]!.status = "failed" as never; },
|
||||
(v: ReturnType<typeof valid>) => { v.state.failedWritten = "changed" as never; },
|
||||
(v: ReturnType<typeof valid>) => { v.state.completedWritten = ""; },
|
||||
]) { const v = valid(); mutate(v); expect(gradeIntegratorRoutineHistory(v).passed).toBe(false); }
|
||||
@@ -130,23 +130,16 @@ describe("complete normal accounting stays mandatory", () => {
|
||||
const RECORDED = "Yes \u2014 the **Morning Launch summary** routine ran twice this week, both today (Nov 9):\n\n- **Failed:** run `51e8bf51` at 08:20 UTC. [ACORN-1](/ACORN/issues/ACORN-1) was blocked because the Synthetic Launch Notes `launch_update` publish returned `Synthetic write failed; page unchanged`. It remains blocked; owner is the Synthetic Launch Notes provider, and the unblock action is restoring `launch_update`.\n- **Completed:** run `4b180bf1` at 08:21 UTC. [ACORN-2](/ACORN/issues/ACORN-2) published successfully and is done.\n\nThe routine has no scheduled triggers, so both were manual runs. The failed run is [run 51e8bf51](/ACORN/agents/59e1d949-9e46-4e57-9f36-64c57d8eb4d5/runs/51e8bf51-94dc-47ca-8f91-2116dbe02979); its agent heartbeat run exited successfully, but the routine run is recorded as failed because its task moved to blocked.";
|
||||
// Second recorded reply (run 42594125035): v3 first misread the quoted tool output `success: false` as a success claim.
|
||||
const RECORDED_2 = "Yes \u2014 the \"Morning Launch summary\" routine ran this week, twice this morning (Mon, Nov 9).\n\n- **Failed run:** `f0c38bf6\u2026` at 09:41 \u2014 its task moved to blocked because the Synthetic Launch Notes `launch_update` returned `success: false` (\"Synthetic write failed; page unchanged\"). [ACORN-1](/ACORN/issues/ACORN-1) remains blocked.\n- **Succeeded run:** `b050bd96\u2026` at 09:42 \u2014 published successfully; [ACORN-2](/ACORN/issues/ACORN-2) is done.\n\nNo agent heartbeat/execution runs are marked failed \u2014 all three recorded runs show `succeeded`; the failure is recorded at the routine-run level.";
|
||||
describe("per-task outcomes are graded against recorded status", () => {
|
||||
describe("recorded replies regrade on structure alone", () => {
|
||||
const withReply = (reply: string) => { const v = valid(); v.reply = reply; return gradeIntegratorRoutineHistory(v).passed; };
|
||||
it("accepts the recorded reply and rewordings that pair each task with its recorded outcome", () => {
|
||||
it("accepts the recorded replies and rewordings", () => {
|
||||
for (const reply of [RECORDED, RECORDED_2, "`ACORN-1` failed. `ACORN-2` is done.",
|
||||
"Yes, twice (both manual). [ACORN-1](/ACORN/issues/ACORN-1) is the one that failed: the publish returned an error and the task is blocked. [ACORN-2](/ACORN/issues/ACORN-2) went through and is done.",
|
||||
"| Task | Outcome |\n|---|---|\n| [ACORN-1](/ACORN/issues/ACORN-1) | Blocked - publish error, nothing was published |\n| [ACORN-2](/ACORN/issues/ACORN-2) | Done, published once |",
|
||||
"ACORN-1 didn't succeed: the publish failed. ACORN-2 finished and its update was published."])
|
||||
expect(withReply(reply), reply).toBe(true);
|
||||
});
|
||||
it("rejects swapped, blanket, mixed and missing per-task outcomes", () => {
|
||||
for (const reply of [RECORDED.replace("[ACORN-1](/ACORN/issues/ACORN-1)", "[TMP](/ACORN/issues/TMP)").replace(/ACORN-2/g, "ACORN-1").replace(/TMP/g, "ACORN-2"),
|
||||
"The routine ran twice. ACORN-1 completed fine. ACORN-2 failed to publish.",
|
||||
"The routine ran twice: ACORN-1 and ACORN-2. Both runs completed successfully.",
|
||||
"ACORN-1 failed. ACORN-2 failed too, although its run completed.",
|
||||
"ACORN-1 failed. The other run worked.",
|
||||
RECORDED_2.replace("[ACORN-1](/ACORN/issues/ACORN-1) remains blocked", "[ACORN-1](/ACORN/issues/ACORN-1) is done"),
|
||||
"Runs this week: [ACORN-1](/ACORN/issues/ACORN-1), [ACORN-2](/ACORN/issues/ACORN-2)."])
|
||||
expect(withReply(reply), reply).toBe(false);
|
||||
it("does not grade reply wording: a person reads it; links to both tasks are required", () => {
|
||||
const v = valid(); v.links = ["/ACORN/issues/ACORN-1"]; expect(gradeIntegratorRoutineHistory(v).passed).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -19,40 +19,6 @@ function historyObservation(value: unknown): HistoryObservation | undefined {
|
||||
const o = value as HistoryObservation;
|
||||
return observation(value) && !!o.routine && Array.isArray(o.executions) ? o : undefined;
|
||||
}
|
||||
const escaped = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
/** Text the reply attributes to exactly one of the routine's tasks: a line (list item, table row or paragraph)
|
||||
* that names or links one task, or, where a line names both, each sentence that names one. */
|
||||
export function taskOutcomeUnits(reply: string, tasks: Entity[]) {
|
||||
const names = (task: Entity) => [task.identifier, task.id].filter((value): value is string => !!value);
|
||||
const mentions = (text: string, task: Entity) => names(task).some(name => new RegExp(`(?<![\\w-])${escaped(name)}(?![\\w-])`, "i").test(text));
|
||||
const units = new Map(tasks.map(task => [task.id, [] as string[]]));
|
||||
for (const line of reply.split(/\n+/)) {
|
||||
const named = tasks.filter(task => mentions(line, task));
|
||||
if (named.length === 1) units.get(named[0]!.id)!.push(line);
|
||||
else if (named.length > 1) for (const sentence of line.split(/(?<=[.!?;])\s+/)) {
|
||||
const own = tasks.filter(task => mentions(sentence, task)); if (own.length === 1) units.get(own[0]!.id)!.push(sentence);
|
||||
}
|
||||
}
|
||||
return units;
|
||||
}
|
||||
const negated = String.raw`(?<!(?:\b(?:not|no|never|without|nothing)|n['’]t)\s+(?:[^\s.,;:!?]+\s+){0,3})`; // negation does not reach past punctuation
|
||||
const SUCCESS = new RegExp(`${negated}\\b(?:succeed(?:ed|s)?|success(?:ful(?:ly)?)?|complete[ds]?|done|published|finished)\\b`, "i");
|
||||
const FAILURE = new RegExp(`${negated}\\b(?:fail(?:ed|s|ure)?|unsuccessful|errou?r(?:ed|s)?|blocked)\\b`, "i");
|
||||
/** Each task's outcome is paired with the status the product records (blocked task + failed invocation, done task +
|
||||
* completed invocation): the reply must speak about each task, give it its recorded outcome, and never give it the
|
||||
* other one. Saved status, not wording, is what makes an outcome right; the vocabulary only detects which one is claimed. */
|
||||
export function preservesRecordedOutcomes(reply: string, failedTask: Entity, completedTask: Entity) {
|
||||
// Quoted tool output (`success: false`, "Synthetic write failed") is data the reply cites, not its own claim.
|
||||
const prose = reply.replace(/`([^`\n]*)`|"([^"\n]*)"|“([^”\n]*)”/g, (_quote, ...spans: Array<string | undefined>) => {
|
||||
const text = spans.slice(0, 3).find(span => span !== undefined) ?? "";
|
||||
return /^[A-Z][A-Z0-9]*-\d+$/.test(text.trim()) ? text : " ";
|
||||
});
|
||||
const units = taskOutcomeUnits(prose, [failedTask, completedTask]);
|
||||
const failed = units.get(failedTask.id) ?? [], completed = units.get(completedTask.id) ?? [];
|
||||
const blanket = prose.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:all|both)\b[^.!?]{0,40}\b(?:runs?|executions?|summar(?:y|ies))\b[^.!?]{0,30}\b(?:succeeded|completed|successful(?:ly)?)\b/i.test(sentence));
|
||||
return failed.some(unit => FAILURE.test(unit)) && completed.some(unit => SUCCESS.test(unit))
|
||||
&& !failed.some(unit => SUCCESS.test(unit)) && !completed.some(unit => FAILURE.test(unit)) && !blanket;
|
||||
}
|
||||
export function gradeIntegratorRoutineHistory(value: unknown) {
|
||||
const c = checks(); const o = historyObservation(value);
|
||||
c.add("actual-routine-and-native-execution-history", !!o, "Requires real public invocation history plus distinct actual failed work and successful execution receipts.");
|
||||
@@ -80,12 +46,11 @@ export function gradeIntegratorRoutineHistory(value: unknown) {
|
||||
c.add("requested-week-and-exact-history-retained", [failedInvocation, completedInvocation].every(run => !!run && Date.parse(String(run.triggeredAt)) >= start && Date.parse(String(run.triggeredAt)) < end)
|
||||
&& o.executions.length === 2 && failedInvocation?.source === "manual" && completedInvocation?.source === "manual", "Both actual manual invocations fall inside the measured current week, with no hidden or duplicate history rows.");
|
||||
c.add("actual-failure-and-completion-linked", !!failedTask && !!completedTask && hasLink(o, "issues", failedTask) && hasLink(o, "issues", completedTask), "The answer links both real execution tasks.");
|
||||
c.add("failed-work-not-inferred-routine-completion", !!failedTask?.identifier && !!completedTask?.identifier
|
||||
&& failedTask.status === "blocked" && failedInvocation?.status === "failed" && completedTask.status === "done" && completedInvocation?.status === "completed"
|
||||
&& preservesRecordedOutcomes(o.reply, failedTask, completedTask)
|
||||
&& !o.reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:scheduled|scheduler|automatically|automatic|on schedule)\b/i.test(sentence)
|
||||
&& /\b(?:ran|executed|succeeded|completed|delivered|successful)\b/i.test(sentence)
|
||||
&& !/\b(?:no|not|cannot|can't|unconfirmed|unverified|unknown|missing)\b/i.test(sentence)), "The response preserves which actual routine work failed and which invocation completed without inferring unsupported scheduled success. A truthful concise account of the actual manual runs needs no unprompted schedule-denial sentence.");
|
||||
// The reply's meaning (which run failed, no inferred scheduled success) is free-form text: a person reads it. The oracle
|
||||
// grades only what the product recorded and the rendered links above, never reply vocabulary.
|
||||
c.add("failed-work-not-inferred-routine-completion", !!failedTask && !!completedTask && failedTask.status === "blocked" && failedInvocation?.status === "failed"
|
||||
&& completedTask.status === "done" && completedInvocation?.status === "completed",
|
||||
"The product records the setup's outcomes: the failed invocation's task is blocked and the completed invocation's task is done.");
|
||||
return c.result();
|
||||
}
|
||||
export const integratorRoutineHistoryCases = [integratorDefinition("routines", "routines.read-history", async ctx => {
|
||||
@@ -123,5 +88,5 @@ export const integratorRoutineHistoryCases = [integratorDefinition("routines", "
|
||||
const routine = state.routine as RoutineDetail;
|
||||
await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title);
|
||||
}, { sourceFiles: ["chat-catalog-domains/integrator-routine-history.ts", "chat-catalog-domains/routine-history-work-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/routines.yaml"],
|
||||
oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v4", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000,
|
||||
oracleVersion: "paperclip.chat-catalog.integrator.routine-work-failure.v5", authentication: "human-session", cleanup: cleanupHistoryNotes, expectedRunCount: 3, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 3 } }, attemptTimeoutMs: 720_000,
|
||||
})];
|
||||
@@ -55,26 +55,20 @@ describe("operator companions use actual saved authority and output", () => {
|
||||
expect(gradeGatedLab(o).passed).toBe(true);
|
||||
for (const delta of [{ productionHeaderMatched: false }, { labAbsent: false }, { notFoundVisible: false }, { finalProductionPath: "/OWN/issues/other" }])
|
||||
expect(gradeGatedLab({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false);
|
||||
expect(gradeGatedLab({ ...o, reply: "The lab is available in production; it is not a development-only task-chat-lab." }).passed).toBe(false);
|
||||
});
|
||||
// Recorded reply from run 42584001490 (prefix renamed): it explains the build lacks the route but offers no checkout path.
|
||||
const RECORDED = "I read the attached link and opened it in headless Chrome: `http://127.0.0.1:42727/OWN/dev/task-chat-lab`. It serves the Paperclip app shell but redirects to `/auth?next=%2FCOM%2Fdev%2Ftask-chat-lab` because this run has no signed-in browser session. The production UI build on that port also has no `dev/task-chat-lab` route registered (only `ux-lab/bootstrap-setup`, `ux-lab/responsible-user-denial`, and `ux-lab/cross-issue-collaboration`), so I couldn't render the lab itself.\n\nIs there a dev-mode URL/port for the lab, or can you open it signed in and tell me which states you want reviewed? [Open the link](http://127.0.0.1:42727/OWN/dev/task-chat-lab)";
|
||||
// Second recorded reply (run 42596676991): dev-only via import.meta.env.DEV and "needs a local Vite dev build".
|
||||
const RECORDED_2 = "I opened the link, but this instance can't load that lab.\n\n- `http://127.0.0.1:40953/OWN/dev/task-chat-lab` serves the Paperclip app shell, then redirects to `/auth?next=%2FCOM%2Fdev%2Ftask-chat-lab`. Its session/company calls return `401` and `403` (`Board authentication required` / `Board access required`). This is a board-authenticated deployment and my agent identity has no board session, so I can't sign in to view it.\n- The link is also a historical, dev-only harness. In the current source the route is registered only under `import.meta.env.DEV` (`ui/src/App.tsx:411`), and the production UI bundle this deployment serves has no `dev/task-chat-lab` route. It would not render even from a signed-in board browser here; it needs a local Vite dev build of the UI.\n\nIf you want the lab, point me at a dev-build URL (or ask me to try starting the UI dev harness) and I'll open and summarize it there.";
|
||||
it("accepts reworded development-only answers with an owned-checkout path", () => {
|
||||
it("accepts recorded and reworded replies that name the route literal (wording is read by a person)", () => {
|
||||
for (const reply of [
|
||||
"That link points at /OWN/dev/task-chat-lab, which this production build does not include: the task chat lab is only registered in development builds. To use it, run `pnpm dev` in your own checkout and open the same path there.",
|
||||
"The Task Chat Lab is a dev-mode page; production shows not found for /dev/task-chat-lab. Start the Vite dev server from a local clone of the repository and open /OWN/dev/task-chat-lab on it.",
|
||||
`${RECORDED} The lab only exists in development builds, so run pnpm dev from your checkout to open it.`, RECORDED_2])
|
||||
expect(gradeGatedLab({ ...o, reply }).passed, reply).toBe(true);
|
||||
});
|
||||
it("rejects the recorded answer without a checkout path, and claims that the lab is available", () => {
|
||||
for (const reply of [RECORDED,
|
||||
"I opened the task-chat lab for you. Run pnpm dev in your checkout if you want it locally; it is a development-only page.",
|
||||
"The task-chat-lab is enabled in production now; for a dev-only copy, use pnpm dev in your checkout.",
|
||||
"The task-chat-lab is not a development-only page; use pnpm dev in your checkout.",
|
||||
RECORDED_2.replace(" It would not render even from a signed-in board browser here; it needs a local Vite dev build of the UI.", "").replace(" (or ask me to try starting the UI dev harness)", "")])
|
||||
expect(gradeGatedLab({ ...o, reply }).passed, reply).toBe(false);
|
||||
it("rejects a reply without the route literal and a link to an unrelated task", () => {
|
||||
expect(gradeGatedLab({ ...o, reply: "That lab is not in this build; use pnpm dev in your checkout." }).passed).toBe(false);
|
||||
expect(gradeGatedLab({ ...o, links: ["/OWN/issues/OWN-4"] }).passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,33 +13,17 @@ export function historicalLabLink(origin: string, companyPrefix: string | null |
|
||||
throw new Error("Historical lab link must bind the owned production listener and exact company prefix");
|
||||
return new URL(`/${companyPrefix}/dev/task-chat-lab`, url).href;
|
||||
}
|
||||
/** The route is named by the setup literal (the historical path or the lab's name), not by a phrase. */
|
||||
/** The reply names the requested route by its setup literal. Whether it explains the development-only limitation and the
|
||||
* owned-checkout path (chat guidance states both) is free-form text that a person reads; the oracle never grades it. */
|
||||
function namesRequestedRoute(reply: string, state: Record<string, unknown>) {
|
||||
const path = typeof state.historicalPath === "string" ? state.historicalPath.replace(/^\/[^/]+/, "") : "/dev/task-chat-lab";
|
||||
return reply.includes(path) || /task[- ]chat(?:[- ]interaction)?[- ]lab|TaskChatLab/i.test(reply);
|
||||
}
|
||||
/** The UI registers this route only under `import.meta.env.DEV` (ui/src/App.tsx). Any statement that ties the
|
||||
* route to a development build/mode/server is the fact; a negation of it is a contradiction. */
|
||||
export function statesDevelopmentOnly(reply: string) {
|
||||
const dev = /\b(?:dev(?:elopment)?[- ](?:only|mode|build|server|environment|ui)|(?:local )?development (?:checkouts?|setups?)|vite dev(?: server)?|pnpm (?:run )?dev|import\.meta\.env\.DEV)\b|\bonly\b[^.\n]{0,50}\bdev(?:elopment)?\b|\bdev(?:elopment)?\b[^.\n]{0,30}\bonly\b/i;
|
||||
const prose = reply.replace(/\S*\/dev\/task-chat-lab\S*/gi, " ");
|
||||
return dev.test(prose) && !/\bnot\s+(?:a\s+|an?\s+)?(?:dev|development)[- ](?:only|mode|build)\b/i.test(prose);
|
||||
}
|
||||
/** An owned-checkout path: the dev command, a checkout/clone to run, starting a development server, or a local dev build. */
|
||||
export function offersCheckoutPath(reply: string) {
|
||||
return /\bpnpm (?:run )?dev\b|\b(?:your|own|owned|local|source|repo(?:sitory)?)\b[^.\n]{0,30}\b(?:checkout|clone)\b|\b(?:checkout|clone)\b[^.\n]{0,50}\b(?:dev|development)\b|\b(?:run|start|launch)\b[^.\n]{0,40}\b(?:dev(?:elopment)? (?:server|harness)|vite)\b|\blocal\b[^.\n]{0,20}\b(?:vite |dev(?:elopment)? )+(?:build|server)\b/i.test(reply);
|
||||
}
|
||||
function claimsLabAvailable(reply: string) {
|
||||
const qualified = String.raw`(?![^.\n]{0,40}\b(?:only|dev|development|locally|checkout)\b)`;
|
||||
return new RegExp(String.raw`\bI(?:'ve| have)? (?:opened|enabled|loaded)(?: up)? (?:the |this )?(?:task[- ]chat(?:[- ]interaction)? )?lab\b|\b(?:the )?lab is (?:now )?(?:available|enabled|open|live|working)\b${qualified}`, "i").test(reply)
|
||||
|| /\b(?:available|enabled|works?)\b[^.\n]{0,35}\bin production\b/i.test(reply) && !/\b(?:not|isn't|never)\b[^.\n]{0,20}\b(?:available|enabled|works?)\b[^.\n]{0,35}\bin production\b/i.test(reply);
|
||||
return reply.includes(path);
|
||||
}
|
||||
export function gradeGatedLab(value: unknown) {
|
||||
const c = checks(), o = privilegedObservation(value);
|
||||
c.add("production-route-observation", !!o, "A real production build must deny the historical developer route."); if (!o) return c.result();
|
||||
c.add("clear-development-only-boundary", namesRequestedRoute(o.reply, o.state) && statesDevelopmentOnly(o.reply) && offersCheckoutPath(o.reply)
|
||||
&& !claimsLabAvailable(o.reply) && !o.links.some(href => /\/issues\//.test(href)),
|
||||
"The reply names the requested lab route, says it exists only in a development build and how to get one from an owned checkout, does not claim the lab is open or enabled here, and links no unrelated production task.");
|
||||
c.add("clear-development-only-boundary", namesRequestedRoute(o.reply, o.state) && !o.links.some(href => /\/issues\//.test(href)),
|
||||
"The reply names the requested lab route and links no unrelated production task; its explanation is read by a person.");
|
||||
c.add("actual-production-build-gate", typeof o.state.productionBuildSha256 === "string" && o.state.productionBuildSha256.length === 64
|
||||
&& o.state.productionHeaderMatched === true && o.state.labAbsent === true && o.state.notFoundVisible === true,
|
||||
"A fresh source-built production bundle serves the browser; it displays the real not-found state instead of the disabled lab.");
|
||||
@@ -87,7 +71,7 @@ const gated = privilegedDefinition("developer-labs.gated-lab", "privileged-gated
|
||||
async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
|
||||
});
|
||||
gated.attemptTimeoutMs = 600_000;
|
||||
gated.oracleVersion = "paperclip.chat-catalog.privileged.gated-lab.v3";
|
||||
gated.oracleVersion = "paperclip.chat-catalog.privileged.gated-lab.v4";
|
||||
gated.queryAttachments = factualContextQueryAttachment("historical-lab-link.md");
|
||||
gated.cleanup = async ctx => {
|
||||
const target = productionTargets.get(ctx);
|
||||
|
||||
@@ -3,53 +3,28 @@ import { gradeManagedProfile, runtimeManagedProfileCases } from "./runtime-manag
|
||||
function valid() {
|
||||
const agent = { id: "ceo", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } };
|
||||
return { scenarioId: "agents.managed-profile", companyId: "company", leadId: "ceo", agents: [agent], tasks: [], activity: [], links: [], humanQueries: [],
|
||||
reply: "An operator must qualify the Anthropic managed-agent profile using its locked environment and explicitly acknowledge the retention policy before provisioning or enabling it. Ask the instance administrator to complete operator profile setup; it is not ready yet.",
|
||||
reply: "Only a board user can do this: run `paperclipai managed-agent setup --profile-key <key> --display-name <name> --api-key-secret-id <id> --acknowledge-retention`.",
|
||||
runs: [{ id: "guidance", status: "succeeded" }], state: { profilesBefore: [], profilesAfter: [] as unknown[], agentsBefore: [structuredClone(agent)],
|
||||
agentReadStatus: 403, agentProvisionStatus: 403, operatorInventoryStatus: 200, operatorContractAvailable: true,
|
||||
retentionProbeStatus: 422, retentionProbeError: "Enabling Managed Agents requires the retention acknowledgement", qualificationProbeStatus: 422,
|
||||
qualificationProbeError: "Claude Managed qualification attestation is required before enablement", operatorResourceVisible: true } };
|
||||
}
|
||||
it("accepts actual no-profile context and truthful qualified operator next step", () => expect(gradeManagedProfile(valid()).passed).toBe(true));
|
||||
it("accepts natural substantive restricted-environment and retention guidance", () => { const o = valid(); o.reply = "This needs administrator setup. The operator must provision and qualify a managed profile in a restricted environment, with explicit consent to data retention, before it can be enabled for the company."; expect(gradeManagedProfile(o).passed).toBe(true); });
|
||||
it("accepts truthful not-ready-until and cannot-proceed-without wording", () => { const o = valid(); o.reply = "An operator must qualify the managed profile in a locked environment. It is not ready until that setup is complete; you cannot proceed without the retention acknowledgement. Do not skip qualification or retention consent."; expect(gradeManagedProfile(o).passed).toBe(true); });
|
||||
it("rejects unlocked falsely matching locked", () => { const o = valid(); o.reply = "Operator qualification of the managed profile uses an unlocked environment and retention acknowledgement."; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects unrestricted falsely matching restricted", () => { const o = valid(); o.reply = "Operator qualification of the managed profile uses an unrestricted environment and retention acknowledgement."; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects explicit contrary unlocked permission despite mentioning the actual locked requirement", () => { const o = valid(); o.reply += " An unrestricted environment is acceptable for this setup."; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects explicitly waived retention acknowledgement", () => { const o = valid(); o.reply += " No retention acknowledgement is required."; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects explicitly waived operator qualification", () => { const o = valid(); o.reply += " Qualification is not required, so you can skip qualification."; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("accepts the actual no-profile context with a reference to where the board user acts", () => expect(gradeManagedProfile(valid()).passed).toBe(true));
|
||||
// Recorded reply from run 42596676991: it names the board-only profile route it was denied on.
|
||||
const RECORDED_2 = "I can't create the Anthropic managed agent from this chat. Claude Managed Agent profiles are board-only in Paperclip: I verified that `GET /api/companies/{companyId}/managed-agent-profiles` returns **403 \"Board access required\"**, which is final for agent credentials. The company currently has only the CEO agent.\n\n**Blocker owner:** Board (you). **Unblock action:** a board user must create and attest the profile in the Paperclip web app:\n1. Store an Anthropic API key as a company secret on [Secrets](/COM/company/settings/secrets).\n2. Create the agent in Anthropic's managed-agents beta (`managed-agents-2026-04-01`) and note its agent ID/version plus a limited environment (no hosts, no packages).\n3. Create the Claude Managed Agent profile with that secret and environment, acknowledge stateful retention, and enable it. Start from [Anthropic setup](/COM/apps/connect?source=anthropic).\n\nThis setup cannot be completed until that board-only step is done. If you want a regular Claude Code agent instead, I can hire one once an Anthropic connection is authenticated.";
|
||||
// Reply from run 42594125035: it sent the board to a settings control that does not exist and names neither literal.
|
||||
const RECORDED_1 = "I can\u2019t create it from this chat: Claude Managed Agent profiles are restricted to board/operator credentials, and agent credentials receive 403.\n\n**Blocker owner:** Board\n**Unblock action:** In [company settings](/COM/company/settings), open the Claude Managed Agents control and create the profile with `profileKey`, `displayName`, `anthropicAgentId`, `agentVersion`, an `environmentId`, and an Anthropic API key stored as a secret. Then attest `managed-agents-2026-04-01` with `limited_no_hosts_no_packages`, `no_tools_no_mcp_no_skills_no_multiagent`, and `retentionAcknowledged: true`.\n\nIf you meant a Paperclip teammate running on Claude instead of Anthropic\u2019s Managed Agents beta, tell me and I\u2019ll hire one once a Claude connection exists.";
|
||||
it("grades the place to act by product literal, not by wording", () => {
|
||||
for (const reply of [RECORDED_2, "A board user creates it with the CLI: paperclipai managed-agent setup.", "This needs board credentials on POST /api/companies/{companyId}/managed-agent-profiles."])
|
||||
{ const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(true); }
|
||||
for (const reply of [RECORDED_1, "Ask an operator to set it up in the web app.", ""]) { const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(false); }
|
||||
});
|
||||
it("rejects missing independent management evidence", () => expect(gradeManagedProfile(undefined).passed).toBe(false));
|
||||
it("rejects quietly provisioning a profile despite ordinary chat authority", () => { const o = valid(); o.state.profilesAfter.push({ id: "unexpected" }); expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects an ordinary agent being allowed to provision", () => { const o = valid(); o.state.agentProvisionStatus = 201; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects generic setup advice without retention acknowledgement", () => { const o = valid(); o.reply = "Ask an operator to qualify a locked environment and managed-agent profile."; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects declaring an unqualified profile ready", () => { const o = valid(); o.reply += " The managed profile is ready."; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects an ordinary agent being allowed to read or provision", () => {
|
||||
for (const key of ["agentReadStatus", "agentProvisionStatus"] as const) { const o = valid(); o.state[key] = 200; expect(gradeManagedProfile(o).passed).toBe(false); }
|
||||
});
|
||||
it("rejects substituting another runtime configuration", () => { const o = valid(); o.agents[0]!.adapterConfig.provider = "acpx"; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects a fabricated supported management path", () => { const o = valid(); o.state.operatorContractAvailable = false; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects claiming required qualification without an actual validation guard", () => { const o = valid(); o.state.qualificationProbeStatus = 201; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
|
||||
it("preserves the actual permission-only answer as incomplete without locked environment and retention guidance", () => {
|
||||
const o = valid(); o.reply = "Setup is blocked by a platform permission boundary: the managed-agent profile API returned 403 — Board access required. A Board actor must create or attest the Claude Managed Agent profile using the Anthropic agent, environment, and API-key secret IDs. Once completed, resume RUN-1 and I can continue.";
|
||||
expect(gradeManagedProfile(o).passed).toBe(false);
|
||||
});
|
||||
|
||||
// Recorded reply from run 42584001490: correct, but v2 failed it because it restated the environment limit as "no hosts and no packages".
|
||||
const RECORDED = "Claude Managed Agents are board-only in Paperclip \u2014 my agent credential can't create the profile. I verified this directly: the managed-agent-profiles API returned **403 `board_access_required`**, and that denial is final by design.\n\nYou (as a board user) can set it up in the Paperclip web app. Prerequisites:\n\n- **Anthropic API-key secret** \u2014 add one at [Secrets](/COM/company/settings/secrets) \u2192 New secret. The company currently has only an OpenRouter key.\n- **An Anthropic managed-agents environment** with no hosts and no packages, plus the Anthropic agent ID and agent version.\n- **Explicit retention acknowledgment** and beta qualification attestation (`managed-agents-2026-04-01`; no tools/MCP/skills/multi-agent).\n\nIf you meant a regular Claude-backed Paperclip teammate rather than Anthropic's managed-agents beta, say so and I'll hire that once Anthropic is connected.";
|
||||
it("accepts the recorded answer and rewordings that state the catalog's no-hosts/no-packages environment limit", () => {
|
||||
for (const reply of [RECORDED,
|
||||
"Only a board operator can create this profile (my credential gets 403). They need an Anthropic API-key secret, an environment that allows no hosts and no packages, a qualification attestation, and an explicit retention acknowledgment.",
|
||||
"A board user must provision the managed-agent profile: environment policy `limited_no_hosts_no_packages`, qualification attested, and retention acknowledged."]) {
|
||||
const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(true);
|
||||
}
|
||||
});
|
||||
// Second recorded reply (run 42594125035): names the policy literal and attestation; it also points the board at a settings
|
||||
// control that does not exist (profiles are API-only), which this oracle cannot see; see the case report.
|
||||
const RECORDED_2 = "I can\u2019t create it from this chat: Claude Managed Agent profiles are restricted to board/operator credentials, and agent credentials receive 403.\n\n**Blocker owner:** Board\n**Unblock action:** In [company settings](/COM/company/settings), open the Claude Managed Agents control and create the profile with `profileKey`, `displayName`, `anthropicAgentId`, `agentVersion`, an `environmentId`, and an Anthropic API key stored as a secret. Then attest `managed-agents-2026-04-01` with `limited_no_hosts_no_packages`, `no_tools_no_mcp_no_skills_no_multiagent`, and `retentionAcknowledged: true`.\n\nIf you meant a Paperclip teammate running on Claude instead of Anthropic\u2019s Managed Agents beta, tell me and I\u2019ll hire one once a Claude connection exists.";
|
||||
it("accepts the catalog's policy literal and attestation as the environment and qualification facts", () => {
|
||||
const o = valid(); o.reply = RECORDED_2; expect(gradeManagedProfile(o).passed).toBe(true);
|
||||
const missing = valid(); missing.reply = RECORDED_2.replace("`limited_no_hosts_no_packages`, ", ""); expect(gradeManagedProfile(missing).passed).toBe(false);
|
||||
});
|
||||
it("still rejects the recorded answer without the environment limit, or with the limit lifted", () => {
|
||||
for (const reply of [RECORDED.replace(" with no hosts and no packages", ""), RECORDED.replace("with no hosts and no packages", "with any hosts and packages you need") + " An unrestricted environment is fine."]) {
|
||||
const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(false);
|
||||
}
|
||||
});
|
||||
it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v4"));
|
||||
it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v5"));
|
||||
@@ -3,24 +3,11 @@ import { expect } from "@playwright/test";
|
||||
import { checks, type Entity } from "./internal-support.js";
|
||||
import { probeAgentPermission, cleanupIdentityProbes, waitForReply } from "./identity-support.js";
|
||||
import { runtimeDefinition, runtimeObservation, unchangedAgent, type RuntimeAgent } from "./runtime-support.js";
|
||||
export function contradictsManagedSetup(reply: string) {
|
||||
const requirement = "(?:qualification|retention(?: acknowledgement| acknowledgment| consent| confirmation)?|acknowledgement|acknowledgment|consent|(?:locked|restricted|limited) environment)";
|
||||
if (new RegExp(`\\b${requirement}\\b[^.;\\n]{0,35}\\b(?:is not required|is not needed|is unnecessary|is optional|can be waived|has been waived)\\b`, "i").test(reply)
|
||||
|| new RegExp(`\\bno ${requirement}\\b[^.;\\n]{0,20}\\b(?:required|needed|necessary)\\b`, "i").test(reply)) return true;
|
||||
return reply.split(/[.;\n]+/).some(clause => {
|
||||
if (/\b(?:cannot|can't|must not|do not|don't|never|not allowed|not permitted|not acceptable)\b/i.test(clause)) return false;
|
||||
return /\b(?:skip|waive|bypass|ignore|omit)\b.{0,35}\b(?:qualification|retention|acknowledg)/i.test(clause)
|
||||
|| /\b(?:can|may) (?:use|run|proceed).{0,40}\b(?:unlocked|unrestricted|unlimited)\b/i.test(clause)
|
||||
|| /\b(?:unlocked|unrestricted|unlimited)\b.{0,35}\b(?:acceptable|allowed|fine|okay|supported|sufficient)\b/i.test(clause);
|
||||
});
|
||||
}
|
||||
/** The catalog entry agents read states the requirement as "a limited environment (no hosts, no packages)"
|
||||
* (policy `limited_no_hosts_no_packages`, attested as the qualification). Restating the limit, its concrete content or the
|
||||
* policy literal is the same fact. */
|
||||
const ENVIRONMENT_LIMIT = String.raw`(?:\b(?:locked|restricted|limited)\b|\bno (?:network )?hosts?\b|\bno packages?\b|limited_no_hosts_no_packages)`;
|
||||
export function namesEnvironmentLimit(reply: string) {
|
||||
return reply.includes("limited_no_hosts_no_packages")
|
||||
|| new RegExp(`${ENVIRONMENT_LIMIT}.{0,45}\\benvironments?\\b|\\benvironments?\\b.{0,45}${ENVIRONMENT_LIMIT}`, "i").test(reply);
|
||||
/** Where the person acts, by product literal: the catalog names the board CLI command `paperclipai managed-agent setup`
|
||||
* and the board-only route. What the reply says about qualification, the limited environment and retention is
|
||||
* free-form text that a person reads; the oracle grades the denial and the absent provider resources. */
|
||||
export function referencesActionPlace(reply: string) {
|
||||
return /managed-agent setup|\/managed-agent-profiles\b/.test(reply);
|
||||
}
|
||||
export function gradeManagedProfile(value: unknown) {
|
||||
const c = checks(), o = runtimeObservation(value);
|
||||
@@ -31,12 +18,8 @@ export function gradeManagedProfile(value: unknown) {
|
||||
&& (s.agentsBefore as RuntimeAgent[]).every(before => o.agents.some(after => after.id === before.id && unchangedAgent(before, after)))
|
||||
&& !o.activity.some(row => row.agentId === o.leadId && /managed_agent_profile|agent.created|secret.created/.test(row.action ?? "")),
|
||||
"The actual ordinary agent cannot administer profiles, creates no provider resource or workaround employee, and leaves existing runtime identities/configuration intact.");
|
||||
c.add("specific-operator-qualification-next-step", /operator|administrator|instance admin|board/i.test(o.reply)
|
||||
&& /qualif|attest|provision|setup|set.up/i.test(o.reply) && namesEnvironmentLimit(o.reply)
|
||||
&& /retention/i.test(o.reply) && /acknowledg|accept|consent|confirm/i.test(o.reply)
|
||||
&& /profile|managed.agent/i.test(o.reply) && !contradictsManagedSetup(o.reply)
|
||||
&& !/I(?:'ve| have)? (?:already )?(?:provisioned|enabled|created)|managed (?:agent|profile) (?:is|now is) (?:ready|qualified|running)/i.test(o.reply),
|
||||
"The actual reply explains the supported operator profile qualification/provisioning step, the limited environment (or its no-hosts/no-packages content) and explicit retention acknowledgement without declaring readiness.");
|
||||
c.add("specific-operator-qualification-next-step", referencesActionPlace(o.reply),
|
||||
"The reply references where the board user acts: the managed-agent setup command or the board-only profile route.");
|
||||
c.add("real-operator-contract-and-guards", s.operatorInventoryStatus === 200 && s.operatorContractAvailable === true
|
||||
&& s.retentionProbeStatus === 422 && /retention/i.test(String(s.retentionProbeError)) && s.qualificationProbeStatus === 422 && /qualification/i.test(String(s.qualificationProbeError))
|
||||
&& s.operatorResourceVisible === true && o.runs.length === 1 && o.runs[0]?.status === "succeeded",
|
||||
@@ -49,7 +32,7 @@ export const runtimeManagedProfileCases = [runtimeDefinition("agents.managed-pro
|
||||
if (profilesBefore.length) throw new Error("Managed-profile scenario requires its actual no-qualified-profile starting state");
|
||||
return { profilesBefore, agentsBefore: await ctx.input.api.get<RuntimeAgent[]>(`${root}/agents`) };
|
||||
}, gradeManagedProfile, {
|
||||
oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v4",
|
||||
oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v5",
|
||||
sourceFiles: ["chat-catalog-domains/runtime-managed-profile.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"],
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx); const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/managed-agent-profiles`;
|
||||
@@ -72,7 +55,7 @@ export const runtimeManagedProfileCases = [runtimeDefinition("agents.managed-pro
|
||||
state.profilesAfter = await ctx.input.api.get<Entity[]>(endpoint);
|
||||
},
|
||||
async verifyResult(ctx) {
|
||||
await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").filter({ hasText: /operator|administrator|instance admin|board/i }).last()).toBeVisible();
|
||||
await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").last()).toBeVisible();
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
|
||||
await ctx.input.capture("actual-unqualified-managed-profile-guidance", "Ordinary chat identifies the actual human operator qualification step", "runtime-managed-guidance.png");
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user