PaperClipAI/tests/runner-e2e/everyday.test.ts

import { describe, it, expect } from "vitest";
import { runnerMatrix, runnerSuites } from "./catalog.js";
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
import { everydayTasks, productionStoryProfile } from "./everyday-cases.js";
import {
  isStoryWorkspaceDeferral,
  storyLifecycleChecks,
  storyRepliesConsumed,
  storyParentFinishedAfterChildren,
  type StoryRun,
} from "./everyday-observations.js";

describe("manual everyday workflow catalog", () => {
  it("is discoverable and explicitly selected without changing scheduled --all", () => {
    const all = selectRunnerExecutions(parseRunnerSelectors(["--all"]));
    expect(all.some((e) => e.suite.id === "everyday-workflows")).toBe(false);
    const selected = selectRunnerExecutions(
      parseRunnerSelectors(["--suite", "everyday-workflows"]),
    );
    expect(selected).toHaveLength(35);
    expect(selected.every((e) => e.profile.generation === "native")).toBe(true);
    expect(
      selectRunnerExecutions(
        parseRunnerSelectors(["--profile", "runner-codex"]),
      ).some((e) => e.suite.id === "everyday-workflows"),
    ).toBe(false);
    const listed = selectRunnerExecutions(parseRunnerSelectors(["--list"]));
    expect(listed.some((e) => e.suite.id === "everyday-workflows")).toBe(true);
  });
  it("does not schedule arbitrary runner-crash probes as model evals", () => {
    const selected = selectRunnerExecutions(
      parseRunnerSelectors(["--suite", "everyday-workflows"]),
    );
    expect(selected.some((e) => e.task.id.startsWith("recover-runner"))).toBe(false);
    for (const id of ["recover-runner", "recover-runner-safe", "recover-runner-uncertain"])
      expect(() => selectRunnerExecutions(parseRunnerSelectors([
        "--id", `everyday-workflows.runner-codex.local.${id}`,
      ]))).toThrow();
    expect(selected.some((e) => e.task.id === "recover-controller")).toBe(true);
    expect(selected.some((e) => e.task.id === "stop-redirect")).toBe(true);
  });
  it("keeps ordinary prompts free of completion/API instructions", () => {
    for (const task of everydayTasks)
      expect(task.buildPrompt("sample")).not.toMatch(
        /finish_task|paperclip_finish|PATCH|mark .*done|idempotencyKey/i,
      );
    const profile = runnerSuites.find((s) => s.id === "everyday-workflows")!
      .profiles[0]!;
    const value = productionStoryProfile(profile).buildAgent({
      environmentId: "env",
      environmentFixtureId: "local",
      workspacePath: "/tmp/test",
      secretRefs: {
        OPENAI_API_KEY: {
          type: "secret_ref",
          secretId: "secret",
          version: "latest",
        },
      },
      executionId: "case",
    });
    expect(JSON.stringify(value.instructionsBundle)).not.toMatch(
      /fixture|mark .*done|finish_task|api\/issues/i,
    );
  });
  it("does not claim unsupported remote crash/hiring coverage", () => {
    const remote = runnerMatrix.filter(
      (e) =>
        e.suite.id === "everyday-workflows" && e.environment.id === "daytona",
    );
    expect(remote).toHaveLength(8);
    expect(new Set(remote.map((e) => e.task.id))).toEqual(
      new Set(["build-revise", "delegate-feedback", "recover-controller", "create-skill-studio"]),
    );
  });
});

describe("lifecycle oracle calibrated failures", () => {
  const parent = {
    id: "parent",
    companyId: "company",
    title: "Project",
    status: "done",
    assigneeAgentId: "lead",
  };
  const run: StoryRun = {
    id: "run",
    companyId: "company",
    agentId: "lead",
    status: "succeeded",
    runtimeMode: "native",
    runnerInstanceId: "runner",
    contextSnapshot: { issueId: "parent" },
  };
  const score = (
    runs: StoryRun[],
    issues = [parent],
    allowedInterruptedRuns: string[] = [],
  ) =>
    storyLifecycleChecks({
      issues,
      runs,
      parentId: parent.id,
      leadId: "lead",
      allowedInterruptedRuns,
    });
  it("accepts successful owned native work", () =>
    expect(score([run]).every((c) => c.passed)).toBe(true));
  it.each([
    ["legacy execution", { ...run, runtimeMode: "legacy" }, "native-runtime"],
    [
      "no runner identity",
      { ...run, runnerInstanceId: null },
      "native-runtime",
    ],
    ["worker on parent", { ...run, agentId: "worker" }, "parent-owned-by-lead"],
    ["crash without recovery", { ...run, status: "failed" }, "successful-runs"],
    ["unsettled run", { ...run, status: "running" }, "settled"],
  ] as const)("rejects %s", (_label, bad, id) =>
    expect(score([bad]).find((c) => c.id === id)?.passed).toBe(false),
  );
  it("does not exempt unrelated failures because another run was intentionally stopped", () => {
    const checks = score(
      [
        { ...run, status: "cancelled" },
        { ...run, id: "other", status: "failed" },
      ],
      [parent],
      ["run"],
    );
    expect(checks.find((c) => c.id === "successful-runs")?.passed).toBe(false);
  });
  it("excludes a proven pre-dispatch workspace deferral without hiding executed failures", () => {
    const deferred: StoryRun = {
      id: "deferred",
      companyId: "company",
      agentId: "lead",
      status: "cancelled",
      errorCode: "workspace_busy",
      resultJson: {
        executionRecovery: {
          kind: "workspace_wait",
          providerWorkStarted: false,
        },
      },
    };
    expect(score([run, deferred]).every((c) => c.passed)).toBe(true);
    expect(
      score([run, { ...deferred, processPid: 123 }]).every((c) => c.passed),
    ).toBe(false);
    expect(
      score([run, { ...deferred, resultJson: {} }]).every((c) => c.passed),
    ).toBe(false);
  });
  it("rejects a finished answer left in review", () =>
    expect(
      score([run], [{ ...parent, status: "in_review" }]).find(
        (c) => c.id === "tasks-done",
      )?.passed,
    ).toBe(false));
});

describe("reply completion boundary", () => {
  const run: StoryRun = {
    id: "old",
    companyId: "company",
    agentId: "lead",
    status: "succeeded",
    runnerProfileJson: {
      nativeExecutionInput: { task: { prompt: "Original task" } },
    },
  };
  it("does not accept Done from the first run while a later request is still queued", () => {
    expect(storyRepliesConsumed([run], ["later-comment"])).toBe(false);
    const next = {
      ...run,
      id: "new",
      runnerProfileJson: {
        nativeExecutionInput: { task: { prompt: "User reply later-comment" } },
      },
    };
    expect(
      storyRepliesConsumed(
        [run, { ...next, status: "running" }],
        ["later-comment"],
      ),
    ).toBe(false);
    expect(storyRepliesConsumed([run, next], ["later-comment"])).toBe(true);
  });
});

describe("delegation completion order", () => {
  it("rejects a parent closed before its worker finishes", () => {
    const base: StoryRun = {
      id: "lead-run",
      companyId: "company",
      agentId: "lead",
      status: "succeeded",
      nativeIssueId: "parent",
      finishedAt: "2026-09-14T12:00:00Z",
    };
    const child = {
      ...base,
      id: "worker-run",
      agentId: "worker",
      nativeIssueId: "child",
      finishedAt: "2026-09-14T12:01:00Z",
    };
    expect(
      storyParentFinishedAfterChildren([base, child], "parent", "lead", [
        "child",
      ]),
    ).toBe(false);
    expect(
      storyParentFinishedAfterChildren(
        [{ ...base, finishedAt: "2026-09-14T12:02:00Z" }, child],
        "parent",
        "lead",
        ["child"],
      ),
    ).toBe(true);
    expect(
      storyParentFinishedAfterChildren([base], "parent", "lead", ["child"]),
    ).toBe(false);
  });
});

describe("cancelled queued workspace retry", () => {
  const queued: StoryRun = {
    id: "retry",
    companyId: "company",
    agentId: "agent",
    status: "cancelled",
    runtimeMode: "legacy",
    runtimeModeResolvedAt: null,
    startedAt: null,
    retryOfRunId: "workspace-deferral",
    scheduledRetryReason: "workspace_busy",
    errorCode: "cancelled",
    lastOutputSeq: 0,
    resultJson: {
      startupCancellation: {
        requestedAt: "2026-09-14T19:10:23.057Z",
        beforeNativeSelection: false,
      },
    },
  };
  it("does not mistake an unstarted cancelled retry for provider execution", () => {
    expect(isStoryWorkspaceDeferral(queued)).toBe(true);
  });
  it.each([
    { startedAt: "2026-09-14T19:10:22Z" },
    { processPid: 42 },
    { runnerInstanceId: "runner" },
    { nativeSessionId: "session" },
    { lastOutputSeq: 1 },
    { usageJson: { outputTokens: 1 } },
    { runtimeModeResolvedAt: "2026-09-14T19:10:22Z" },
    { scheduledRetryReason: "other" },
    { resultJson: null },
  ])("retains a contradictory or unproven cancellation %j", (change) => {
    expect(isStoryWorkspaceDeferral({ ...queued, ...change })).toBe(false);
  });
});