PaperClipAI/tests/runner-e2e/chat-catalog-domains/external-gateway-error.test.ts

import { expect, it } from "vitest";
import {
  gradeGatewayError,
  gatewayErrorAnswer,
  observedMorganReceiptRead,
} from "./external-gateway-error.js";
function valid() {
  const decision = (runId: string, agentId: string, issueId: string) => ({
    runId,
    decisions: [
      {
        invocation: {
          id: `inv-${runId}`,
          runId,
          agentId,
          issueId,
          companyId: "company",
          connectionId: "notes",
          catalogEntryId: "write",
          status: "failed",
          errorCode: "tool_error",
        },
        auditEvents: [
          { invocationId: `inv-${runId}`, runId, outcome: "failure" },
        ],
      },
    ],
  });
  return {
    scenarioId: "gateways-profiles.inspect-gateway-error",
    companyId: "company",
    leadId: "ceo",
    reply:
      "Morgan's task receipt reports a provider error; the page remained unchanged. I cannot inspect company-wide history; an authorized human can open the activity audit for the broader picture.",
    phaseReplies: [],
    connections: [],
    profiles: [],
    secrets: [],
    endpoints: [],
    interactions: [],
    activity: [],
    links: [],
    humanQueries: [],
    runs: [
      {
        id: "morgan-run",
        agentId: "morgan",
        status: "succeeded",
        runtimeMode: "native",
      },
      {
        id: "own-run",
        agentId: "ceo",
        status: "succeeded",
        runtimeMode: "native",
      },
    ],
    state: {
      companyId: "company",
      connectionId: "notes",
      writeEntryId: "write",
      morganId: "morgan",
      morganTaskId: "morgan-task",
      ownTaskId: "own-task",
      morganRunId: "morgan-run",
      ownRunId: "own-run",
      decisions: [
        decision("morgan-run", "morgan", "morgan-task"),
        decision("own-run", "ceo", "own-task"),
      ],
      morganReceiptNativeBound: true,
      ownReceiptNativeBound: true,
      providerWrites: 2,
      providerAllAuthorized: true,
      written: null as string | null,
      agentAuditStatus: 403,
      agentDecisionStatus: 403,
      canonicalReadMorganReceipt: true,
      scopedAuditLink: false,
    },
  };
}
it("accepts actual Morgan and own failure receipts with a permitted read or honest audit handoff", () => {
  expect(gradeGatewayError(valid()).passed).toBe(true);
  const o = valid();
  o.state.canonicalReadMorganReceipt = false;
  o.state.scopedAuditLink = true;
  o.reply =
    "I cannot inspect Morgan's company-wide tool history. An authorized human can review the activity audit to determine the cause.";
  expect(gradeGatewayError(o).passed).toBe(true);
});
it("rejects wrong identity/cause/authority and missing provider/task/result evidence", () => {
  for (const [index, mutate] of [
    (o: ReturnType<typeof valid>) => {
      o.state.decisions[0]!.decisions[0]!.invocation.agentId = "ceo";
    },
    (o: ReturnType<typeof valid>) => {
      o.state.decisions[0]!.decisions[0]!.invocation.issueId = "own-task";
    },
    (o: ReturnType<typeof valid>) => {
      o.state.decisions[0]!.decisions[0]!.invocation.errorCode =
        "deny_policy_block";
    },
    (o: ReturnType<typeof valid>) => {
      o.state.decisions[0]!.decisions[0]!.auditEvents = [];
    },
    (o: ReturnType<typeof valid>) => {
      o.state.morganReceiptNativeBound = false;
    },
    (o: ReturnType<typeof valid>) => {
      o.state.providerWrites = 0;
    },
    (o: ReturnType<typeof valid>) => {
      o.state.written = "unexpected mutation";
    },
    (o: ReturnType<typeof valid>) => {
      o.state.agentAuditStatus = 200;
    },
    (o: ReturnType<typeof valid>) => {
      o.state.canonicalReadMorganReceipt = false;
    },
    (o: ReturnType<typeof valid>) => {
      o.reply =
        "Morgan's notes tool failed because of policy denial. The provider returned an error.";
    },
    (o: ReturnType<typeof valid>) => {
      o.reply =
        "I've reviewed all gateway activity. Morgan had a provider error.";
    },
    (o: ReturnType<typeof valid>) => {
      o.reply = "My receipt proves Morgan had a provider error.";
    },
  ].entries()) {
    const o = valid();
    mutate(o);
    expect(gradeGatewayError(o).passed, String(index)).toBe(false);
  }
  expect(gradeGatewayError(undefined).passed).toBe(false);
  expect(
    gatewayErrorAnswer(
      "My own receipt does not represent Morgan's history. Morgan's task receipt shows a provider error.",
    ),
  ).toBe(true);
});
it("requires a trusted successful model tool result containing the exact separately persisted Morgan report", () => {
  const report = {
    id: "comment",
    issueId: "morgan-task",
    authorAgentId: "morgan",
    createdByRunId: "morgan-run",
    body: "The provider returned an error; the page was unchanged.",
  };
  const event = (seq: number, eventType: string, payload: unknown) => ({
    seq,
    eventType,
    protocolSchemaVersion: 2,
    payload: {
      prpEvent: {
        schema: "paperclip.prp.event.v2",
        schemaVersion: 2,
        sourceKind: "runner",
        payload,
      },
    },
  });
  const events = [
    event(1, "item.started", {
      item: {
        type: "tool_use",
        id: "call",
        input: {
          operationId: "GET /api/issues/{id}/comments",
          pathParams: { id: "morgan-task" },
        },
      },
    }),
    event(2, "item.completed", {
      item: { type: "tool_result", id: "call", result: { data: [report] } },
    }),
    event(3, "tool.execution.completed", {
      executionId: "call",
      status: "completed",
    }),
  ];
  expect(observedMorganReceiptRead(events, report)).toBe(true);
  for (const broken of [
    events.slice(1),
    events.slice(0, 2),
    [
      events[0]!,
      event(2, "item.completed", {
        item: {
          type: "tool_result",
          id: "call",
          result: { data: [{ ...report, createdByRunId: "own-run" }] },
        },
      }),
      events[2]!,
    ],
    events.map((row) => ({ ...row, protocolSchemaVersion: 1 })),
  ])
    expect(observedMorganReceiptRead(broken, report)).toBe(false);
});