mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
Clarify native completion tools and qualify independently on production defaults
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
59c07ede72
commit
b8a6bf950a
35 files changed
+1483
-52
No files matched your search
@@ -0,0 +1,58 @@
|
||||
# Native completion guidance qualification
|
||||
|
||||
This explicit-only suite independently qualifies PR #14961 on master
|
||||
`59c07ede72dc08b8aba149a01cc11e0b7a204621`. It does not include PR #14948's
|
||||
reduced manuals, shared prompts or operational-skill changes. The earlier
|
||||
comparison on reduced defaults remains historical evidence, not admission for
|
||||
this context.
|
||||
|
||||
The six local cells pair the original `assigned-skill-explicit-invocation`
|
||||
journey and `native-blocked-report` across `runner-codex`,
|
||||
`runner-acpx-claude` and `runner-opencode`. Completion keeps the original task,
|
||||
installed output skill, explicit invocation and independently saved issue
|
||||
document checks. Blocking checks the durable whole-task blocker, exact external
|
||||
owner/action, explanatory visible reply, unchanged workspace and absence of
|
||||
extra tasks, agents, documents or interactions.
|
||||
|
||||
Both use the actual production default hire: no custom QA instructions bundle.
|
||||
Before execution, the public bundle receipt must hash to the unchanged master
|
||||
AGENTS.md, and company/agent budgets must each be 1,000 cents. Profile model,
|
||||
effort, credentials, skills, permissions and provider configuration are inherited.
|
||||
|
||||
The native oracle requires exactly one succeeded native run attributed to this
|
||||
issue, bound complete public events, one runner semantic proposal and authoritative
|
||||
control-plane acceptance/terminal. A matched finishing tool start/result precedes
|
||||
the actual provider final; no new invocation follows the proposal. The final
|
||||
must be persisted as the run's agent reply. Acceptance may precede or follow the
|
||||
final. This proves observable ordering and durable disposition; it does not prove
|
||||
the provider consumed feedback or establish general coding-quality equivalence.
|
||||
|
||||
`single_attempt` is enforced by the launcher even when hosted invocation only
|
||||
supplies `--id`. Every first failure, usage and cleanup receipt is retained; no
|
||||
automatic infrastructure or behavioral retry occurs. The bounded comparison is
|
||||
six cells per variant, twelve expected provider turns total, with the existing
|
||||
per-cell deadlines (skill: original deadline; blocker: five minutes).
|
||||
|
||||
Candidate and baseline use identical executable fixtures and public oracles.
|
||||
Only the five native description/fingerprint source files and their six
|
||||
variant-specific unit tests may differ. The historical variant restores those
|
||||
eleven files from master, including v13; candidate uses the archived native change
|
||||
and v14. Source-contract hashes reject mixed revisions and changes to master
|
||||
defaults, auth/runtime context or shared instructions. Derived capability metadata
|
||||
is independently checked; it currently requires no regeneration in either variant.
|
||||
|
||||
Credential-free admission runs before local environment/credential loading:
|
||||
|
||||
```sh
|
||||
node tests/runner-e2e/native-completion-checks.mjs --output-dir=/absolute/report/path
|
||||
node tests/runner-e2e/native-completion-checks.mjs --verify=/absolute/report/path/preflight.json
|
||||
pnpm test:e2e:runner -- --list --suite native-completion
|
||||
```
|
||||
|
||||
Admission requires a committed clean exact source, variant-appropriate schema,
|
||||
tool bridge/catalog and resume tests, independent oracle/default/retry calibrations,
|
||||
original context-integrity calibrations, typecheck, manifest checks and exact
|
||||
six-cell discovery. Generic stock-suite gates remain unchanged. Paid hosted runs
|
||||
must dispatch the trusted default-branch workflow to distinct immutable target
|
||||
branches using the exact six IDs, preserving authorization and secret boundaries.
|
||||
Live results are pending until retained cell evidence is published.
|
||||
@@ -422,6 +422,8 @@ pnpm test:e2e:runner -- --list --suite agent-chat-hardening
|
||||
pnpm test:e2e:runner -- --id agent-chat-hardening.runner-codex.local.stop-startup-new-resume
|
||||
```
|
||||
|
||||
The independent, explicit-only `native-completion` suite qualifies native finish/block descriptions on unchanged master defaults. It preserves the original assigned-skill document journey and pairs it with whole-task blocking across three native profiles, with enforced single attempts. See [NATIVE-COMPLETION.md](NATIVE-COMPLETION.md) for admission, provenance and limits.
|
||||
|
||||
`context-integrity` is an explicit-only local suite with two bounded cases across
|
||||
ten listed legacy/native profiles (20 cells). Six cells are pending-prerequisite
|
||||
profiles and are listed for discovery but rejected before provider credentials are
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js";
|
||||
import type { FailureClass } from "./types.js";
|
||||
|
||||
const retryableClasses = ["transient_infrastructure", "provider_variance"] as const;
|
||||
const allClasses: FailureClass[] = [...retryableClasses, "candidate_failure", "permanent_infrastructure", "secret_leak", "cleanup_failure"];
|
||||
const options = { maxAutomaticRetries: 1, ui: false, debug: false };
|
||||
|
||||
describe("explicit single-attempt fixture policy", () => {
|
||||
for (const failureClass of allClasses) it(`retains the first ${failureClass} failure without a second attempt`, async () => {
|
||||
const observedAttempts: number[] = [], retryNotifications: unknown[] = [];
|
||||
const first = { status: "failed" as const, failureClass, attempt: 1, runIds: ["first-run"], cost: 3.25 };
|
||||
const result = await executeWithAutomaticRetry<typeof first>({ task: { automaticRetryPolicy: "single_attempt" }, options,
|
||||
qualificationCandidate: false, cancelled: () => false, onRetry: result => retryNotifications.push(result),
|
||||
runAttempt: async attempt => { observedAttempts.push(attempt); return first; } });
|
||||
expect(observedAttempts).toEqual([1]);
|
||||
expect(retryNotifications).toEqual([]);
|
||||
expect(result).toBe(first);
|
||||
expect(result.runIds).toEqual(["first-run"]);
|
||||
expect(result.cost).toBe(3.25);
|
||||
});
|
||||
it("overrides any requested automatic retry count only for opted-in fixtures", () => {
|
||||
expect(effectiveAutomaticRetryLimit({ automaticRetryPolicy: "single_attempt" }, 99)).toBe(0);
|
||||
expect(effectiveAutomaticRetryLimit({}, 1)).toBe(1);
|
||||
expect(effectiveAutomaticRetryLimit({}, 0)).toBe(0);
|
||||
});
|
||||
for (const failureClass of retryableClasses) it(`preserves the ordinary ${failureClass} retry`, async () => {
|
||||
const attempts: number[] = [], notifications: unknown[] = [];
|
||||
const failed = { status: "failed" as const, failureClass, attempt: 1 };
|
||||
const passed = { status: "passed" as const, attempt: 2 };
|
||||
const result = await executeWithAutomaticRetry({ task: {}, options, qualificationCandidate: false,
|
||||
cancelled: () => false, onRetry: result => notifications.push(result),
|
||||
runAttempt: async attempt => { attempts.push(attempt); return attempt === 1 ? failed : passed; } });
|
||||
expect(attempts).toEqual([1, 2]);
|
||||
expect(notifications).toEqual([failed]);
|
||||
expect(result).toBe(passed);
|
||||
});
|
||||
for (const restriction of ["qualificationCandidate", "ui", "debug", "zeroRetries"] as const)
|
||||
it(`preserves the existing ${restriction} restriction`, async () => {
|
||||
const attempts: number[] = [];
|
||||
const first = { status: "failed" as const, failureClass: "provider_variance" as const };
|
||||
const result = await executeWithAutomaticRetry({ task: {}, options: { ...options,
|
||||
ui: restriction === "ui", debug: restriction === "debug", maxAutomaticRetries: restriction === "zeroRetries" ? 0 : 1 },
|
||||
qualificationCandidate: restriction === "qualificationCandidate", cancelled: () => false,
|
||||
onRetry: () => { throw new Error("Unexpected retry notification"); },
|
||||
runAttempt: async attempt => { attempts.push(attempt); return first; } });
|
||||
expect(attempts).toEqual([1]); expect(result).toBe(first);
|
||||
});
|
||||
it("checks cancellation before an otherwise admitted second attempt", async () => {
|
||||
const attempts: number[] = [];
|
||||
await expect(executeWithAutomaticRetry({ task: {}, options, qualificationCandidate: false,
|
||||
cancelled: () => true, onRetry: () => { throw new Error("Unexpected retry notification"); },
|
||||
runAttempt: async attempt => { attempts.push(attempt); return { status: "failed" as const, failureClass: "provider_variance" as const }; },
|
||||
})).rejects.toThrow("campaign cancelled");
|
||||
expect(attempts).toEqual([1]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { shouldRetryFailure } from "./failure-classifier.js";
|
||||
import type { FailureClass, RunnerTaskFixture } from "./types.js";
|
||||
|
||||
type RetryTask = Pick<RunnerTaskFixture, "automaticRetryPolicy">;
|
||||
interface RetryableAttempt { status: "passed" | "failed"; failureClass?: FailureClass }
|
||||
interface RetryOptions { maxAutomaticRetries: number; ui: boolean; debug: boolean }
|
||||
|
||||
export function effectiveAutomaticRetryLimit(task: RetryTask, configuredLimit: number): number {
|
||||
return task.automaticRetryPolicy === "single_attempt" ? 0 : configuredLimit;
|
||||
}
|
||||
|
||||
/** Keep the original failed result when the fixture admits exactly one attempt. */
|
||||
export async function executeWithAutomaticRetry<Result extends RetryableAttempt>(input: {
|
||||
task: RetryTask;
|
||||
options: RetryOptions;
|
||||
qualificationCandidate: boolean;
|
||||
runAttempt(attempt: 1 | 2): Promise<Result>;
|
||||
cancelled(): boolean;
|
||||
onRetry(result: Result): void;
|
||||
}): Promise<Result> {
|
||||
const firstResult = await input.runAttempt(1);
|
||||
if (input.qualificationCandidate || input.options.ui || input.options.debug ||
|
||||
firstResult.status !== "failed" || !firstResult.failureClass ||
|
||||
!shouldRetryFailure(firstResult.failureClass,
|
||||
effectiveAutomaticRetryLimit(input.task, input.options.maxAutomaticRetries))) return firstResult;
|
||||
if (input.cancelled()) throw new Error("Runner E2E campaign cancelled");
|
||||
input.onRetry(firstResult);
|
||||
return input.runAttempt(2);
|
||||
}
|
||||
@@ -136,10 +136,10 @@ describe("runner E2E catalog", () => {
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 20, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1,
|
||||
6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 6, 20, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(438);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(438);
|
||||
expect(validateRunnerCatalog()).toHaveLength(444);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(444);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(48);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { nativeCompletionTasks, nativeCompletionDefinitionDigest } from "./native-completion-cases.js";
|
||||
import { nativeCompletionProfile, NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js";
|
||||
import { chatConfirmationTasks } from "./chat-cases.js";
|
||||
import { hiringTemplateTasks, hiringTemplateProfile, hiringTemplateDefinitionDigest } from "./hiring-template-cases.js";
|
||||
import { instructionPersistenceTask } from "./instruction-persistence.js";
|
||||
@@ -1193,6 +1195,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
.map(task => `everyday-workflows.${profile.id}.daytona.${task.id}`))],
|
||||
definitionMetadata: { version: 4, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only" },
|
||||
},
|
||||
{
|
||||
id: "native-completion", label: "Native completion guidance", manualOnly: true,
|
||||
description: "Production-default assigned-skill document completion and concrete whole-task blocking, with independent native result/final ordering.",
|
||||
groups: ["native", "local"],
|
||||
profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude", "runner-opencode"].includes(profile.id)).map(nativeCompletionProfile),
|
||||
environments: [localEnvironment], tasks: nativeCompletionTasks, expectedMatrixSize: 6,
|
||||
definitionMetadata: { version: 1, fixtureDigest: nativeCompletionDefinitionDigest(), instructions: "unchanged-master-production-default",
|
||||
providerTurns: 6, maximumAttemptsPerCell: 1, automaticRetryPolicy: "single_attempt", budgetMonthlyCents: NATIVE_COMPLETION_BUDGET_CENTS,
|
||||
grading: "original-assigned-skill-durable-document-plus-native-disposition-and-observable-final-order", scheduling: "explicit-only" },
|
||||
},
|
||||
{
|
||||
id: "context-integrity",
|
||||
label: "Context Integrity",
|
||||
|
||||
+30
-31
@@ -21,7 +21,8 @@ import {
|
||||
import { isImmutableDaytonaImage, runnerMatrix } from "./catalog.js";
|
||||
import { renderRunnerE2EDashboard } from "./dashboard.js";
|
||||
import { packageEvidence } from "./evidence.js";
|
||||
import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js";
|
||||
import { classifyFailure } from "./failure-classifier.js";
|
||||
import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js";
|
||||
import { buildRunnerCampaign } from "./history.js";
|
||||
import {
|
||||
buildRunnerE2EProcessEnvironment,
|
||||
@@ -50,6 +51,7 @@ import {
|
||||
type RunnerE2EResult,
|
||||
} from "./types.js";
|
||||
import { assertRunnerE2EPrerequisites } from "./prerequisites.js";
|
||||
import { assertNativeCompletionSelection, prepareNativeCompletionPreflight, NATIVE_COMPLETION_PREFLIGHT_ENV } from "./native-completion-admission.js";
|
||||
import {
|
||||
reapNewDetachedDarwinSharedMemory,
|
||||
snapshotDarwinSharedMemory,
|
||||
@@ -1014,36 +1016,19 @@ async function runExecutionWithRetry(input: {
|
||||
options: ReturnType<typeof parseRunnerSelectors>;
|
||||
}): Promise<RunnerE2EResult> {
|
||||
const { execution, campaignId, options } = input;
|
||||
const [firstResult] = await runAttempt({
|
||||
executions: [execution],
|
||||
attempt: 1,
|
||||
campaignId,
|
||||
options,
|
||||
return executeWithAutomaticRetry({
|
||||
task: execution.task, options,
|
||||
qualificationCandidate: execution.profile.qualificationCandidate !== undefined,
|
||||
cancelled: () => cancelled,
|
||||
onRetry: result => console.warn(
|
||||
`Retrying ${execution.id} in a fresh isolated harness after ${result.failureClass!.replaceAll("_", " ")}`,
|
||||
),
|
||||
runAttempt: async attempt => {
|
||||
const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options });
|
||||
if (!result) throw new Error(`No result produced for ${execution.id} attempt ${attempt}`);
|
||||
return result;
|
||||
},
|
||||
});
|
||||
if (!firstResult) throw new Error(`No result produced for ${execution.id}`);
|
||||
if (
|
||||
execution.profile.qualificationCandidate !== undefined ||
|
||||
options.ui ||
|
||||
options.debug ||
|
||||
firstResult.status !== "failed" ||
|
||||
!firstResult.failureClass ||
|
||||
!shouldRetryFailure(firstResult.failureClass, options.maxAutomaticRetries)
|
||||
) {
|
||||
return firstResult;
|
||||
}
|
||||
if (cancelled) throw new Error("Runner E2E campaign cancelled");
|
||||
console.warn(
|
||||
`Retrying ${execution.id} in a fresh isolated harness after ${firstResult.failureClass.replaceAll("_", " ")}`,
|
||||
);
|
||||
const [retryResult] = await runAttempt({
|
||||
executions: [execution],
|
||||
attempt: 2,
|
||||
campaignId,
|
||||
options,
|
||||
});
|
||||
if (!retryResult)
|
||||
throw new Error(`No retry result produced for ${execution.id}`);
|
||||
return retryResult;
|
||||
}
|
||||
|
||||
async function runWithConcurrency<T, R>(
|
||||
@@ -1098,6 +1083,15 @@ async function main() {
|
||||
// Keep admission before local-env loading and credential checks. Pending
|
||||
// profiles remain discoverable, but cannot reach a provider.
|
||||
assertRunnerE2EPrerequisites(executions);
|
||||
assertNativeCompletionSelection(executions);
|
||||
let nativeCampaignId: string | undefined;
|
||||
if (executions.some(execution => execution.suite.id === "native-completion")) {
|
||||
nativeCampaignId = cleanId(process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??
|
||||
`local-${new Date().toISOString().replace(/[:.]/g, "-")}`);
|
||||
const nativeCampaignDirectory = path.join(resultsRoot, nativeCampaignId);
|
||||
await mkdir(nativeCampaignDirectory, { recursive: true });
|
||||
process.env[NATIVE_COMPLETION_PREFLIGHT_ENV] = prepareNativeCompletionPreflight(nativeCampaignDirectory);
|
||||
}
|
||||
|
||||
await loadLocalEnvironment(process.env);
|
||||
const missingCredentials = [
|
||||
@@ -1119,7 +1113,7 @@ async function main() {
|
||||
);
|
||||
}
|
||||
|
||||
const campaignId = cleanId(
|
||||
const campaignId = nativeCampaignId ?? cleanId(
|
||||
process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??
|
||||
`local-${new Date().toISOString().replace(/[:.]/g, "-")}`,
|
||||
);
|
||||
@@ -1132,6 +1126,11 @@ async function main() {
|
||||
version: 1,
|
||||
maxAutomaticRetries: options.maxAutomaticRetries,
|
||||
retryClasses: ["transient_infrastructure", "provider_variance"],
|
||||
executions: executions.map(execution => ({
|
||||
executionId: execution.id,
|
||||
automaticRetryPolicy: execution.task.automaticRetryPolicy ?? "default",
|
||||
maxAutomaticRetries: effectiveAutomaticRetryLimit(execution.task, options.maxAutomaticRetries),
|
||||
})),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js";
|
||||
import path from "node:path";
|
||||
import { isManagedHiringCase } from "./chat-cases.js";
|
||||
import { FixtureRegistry } from "./fixture-registry.js";
|
||||
@@ -135,7 +136,7 @@ export async function setupLiveFixtures(input: {
|
||||
return api.post<CompanyRecord>("/api/companies", {
|
||||
name: `Runner E2E ${execution.id} ${input.executionNonce}`,
|
||||
description: "Ephemeral paid full-stack runner acceptance fixture",
|
||||
budgetMonthlyCents: execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS : 0,
|
||||
budgetMonthlyCents: execution.suite.id === "native-completion" ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS : 0,
|
||||
});
|
||||
},
|
||||
async teardown() {
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { assertNativeBlockerReply } from "./native-blocker-visible.js";
|
||||
|
||||
test("accepts a visible blocker explanation rather than requiring a marker-only reply", async ({ page }) => {
|
||||
await page.setContent('<div data-testid="reply">Deployment is <strong>blocked</strong>.<br>Release Owner must Grant deployment access. BLOCKED_probe</div>');
|
||||
await assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100);
|
||||
});
|
||||
|
||||
test("accepts a future access condition while the task remains blocked", async ({ page }) => {
|
||||
await page.setContent('<div data-testid="reply">Deployment remains blocked until access is granted. Release Owner must Grant deployment access. BLOCKED_probe</div>');
|
||||
await assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100);
|
||||
});
|
||||
|
||||
for (const [name, body] of [
|
||||
["marker only", "BLOCKED_probe"],
|
||||
["missing owner", "Blocked. Someone must Grant deployment access. BLOCKED_probe"],
|
||||
["missing action", "Blocked. Release Owner must take action. BLOCKED_probe"],
|
||||
["missing reason", "Release Owner: Grant deployment access. BLOCKED_probe"],
|
||||
["contradictory disposition", "Deployment is not blocked. Release Owner completed Grant deployment access. BLOCKED_probe"],
|
||||
["resolved blocker", "Deployment is no longer blocked. Release Owner must Grant deployment access. BLOCKED_probe"],
|
||||
["granted access", "Blocked. Access already granted. Release Owner must Grant deployment access. BLOCKED_probe"],
|
||||
["duplicate replies", "Blocked. Release Owner must Grant deployment access. BLOCKED_probe</div><div data-testid=reply>BLOCKED_probe"],
|
||||
]) {
|
||||
test(`rejects ${name}`, async ({ page }) => {
|
||||
await page.setContent(`<div data-testid="reply">${body}</div>`);
|
||||
await expect(assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100)).rejects.toThrow();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { expect, type Locator } from "@playwright/test";
|
||||
|
||||
/** Blocker replies include an explanation; ordinary marker-only replies do not. */
|
||||
export async function assertNativeBlockerReply(agentReplies: Locator, marker: string, timeout = 30_000): Promise<void> {
|
||||
const reply = agentReplies.filter({ hasText: marker });
|
||||
await expect(reply).toHaveCount(1, { timeout });
|
||||
await expect(reply).toBeVisible({ timeout });
|
||||
await expect(reply).toContainText("Release Owner", { timeout });
|
||||
await expect(reply).toContainText("Grant deployment access", { timeout });
|
||||
await expect(reply).not.toContainText(
|
||||
/\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\b/i,
|
||||
{ timeout },
|
||||
);
|
||||
await expect(reply).toContainText(
|
||||
/\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\b/i,
|
||||
{ timeout },
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { assertNativeCompletionSelection, prepareNativeCompletionPreflight, nativeCompletionPreflightEnvironment, verifyNativeCompletionPreflight } from "./native-completion-admission.js";
|
||||
import { CREDENTIAL_NAMES, type MatrixExecution } from "./types.js";
|
||||
|
||||
vi.mock("node:child_process", () => ({ spawnSync: vi.fn() }));
|
||||
const spawn = vi.mocked(spawnSync);
|
||||
const campaignDirectory = "/fixture/results/gha-1-1-native-completion.fixture";
|
||||
beforeEach(() => { vi.resetAllMocks(); vi.unstubAllEnvs(); });
|
||||
|
||||
describe("native completion credential-free admission", () => {
|
||||
it("allows toolchain paths and excludes every present or future credential", () => {
|
||||
const source = { PATH: "/bin", HOME: "/home/fixture", CARGO_HOME: "/cargo", CI: "true",
|
||||
GH_TOKEN: "secret", FUTURE_PROVIDER_API_KEY: "secret", NODE_OPTIONS: "ambient override",
|
||||
...Object.fromEntries(CREDENTIAL_NAMES.map(name => [name, "secret"])) };
|
||||
expect(nativeCompletionPreflightEnvironment(source)).toEqual({ PATH: "/bin", HOME: "/home/fixture", CARGO_HOME: "/cargo", CI: "true" });
|
||||
});
|
||||
it("rejects missing receipts without invoking a process", () => {
|
||||
expect(() => verifyNativeCompletionPreflight(undefined)).toThrow("no prerequisite receipt");
|
||||
expect(spawn).not.toHaveBeenCalled();
|
||||
});
|
||||
it("fails before provider execution when the prerequisite subprocess fails", () => {
|
||||
spawn.mockReturnValue({ status: 1 } as ReturnType<typeof spawnSync>);
|
||||
expect(() => prepareNativeCompletionPreflight(campaignDirectory)).toThrow("before provider execution");
|
||||
expect(spawn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it("retains credential-free prerequisites inside the exact campaign root and verifies them", () => {
|
||||
vi.stubEnv("OPENAI_API_KEY", "secret"); vi.stubEnv("UNLISTED_PROVIDER_CREDENTIAL", "secret");
|
||||
spawn.mockReturnValueOnce({ status: 0 } as ReturnType<typeof spawnSync>);
|
||||
spawn.mockReturnValueOnce({ status: 0, stdout: '{"passed":true}' } as ReturnType<typeof spawnSync>);
|
||||
const receipt = prepareNativeCompletionPreflight(campaignDirectory);
|
||||
expect(receipt).toMatch(/^\/fixture\/results\/gha-1-1-native-completion\.fixture\/native-completion-prerequisites\/[^/]+\/preflight.json$/);
|
||||
expect(spawn.mock.calls[1]?.[1]).toContain(`--verify=${receipt}`);
|
||||
for (const call of spawn.mock.calls) {
|
||||
expect(call[2]?.env).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(call[2]?.env).not.toHaveProperty("UNLISTED_PROVIDER_CREDENTIAL");
|
||||
}
|
||||
});
|
||||
it("rejects a workflow source override that differs from the exact receipt SHA", () => {
|
||||
vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_SHA", "a".repeat(40));
|
||||
spawn.mockReturnValue({ status: 0, stdout: JSON.stringify({ passed: true, sourceSha: "b".repeat(40) }) } as ReturnType<typeof spawnSync>);
|
||||
expect(() => verifyNativeCompletionPreflight("/fixture/preflight.json")).toThrow("source SHA differs");
|
||||
spawn.mockReturnValue({ status: 0, stdout: JSON.stringify({ passed: true, sourceSha: "a".repeat(40) }) } as ReturnType<typeof spawnSync>);
|
||||
expect(verifyNativeCompletionPreflight("/fixture/preflight.json").sourceSha).toBe("a".repeat(40));
|
||||
});
|
||||
it("refuses failed or unreadable receipt verification", () => {
|
||||
spawn.mockReturnValue({ status: 1, stderr: "stale source" } as ReturnType<typeof spawnSync>);
|
||||
expect(() => verifyNativeCompletionPreflight("/fixture/preflight.json")).toThrow("stale source");
|
||||
spawn.mockReturnValue({ status: 0, stdout: "malformed" } as ReturnType<typeof spawnSync>);
|
||||
expect(() => verifyNativeCompletionPreflight("/fixture/preflight.json")).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
function selectedFixture() {
|
||||
const tasks = ["assigned-skill-explicit-invocation", "native-blocked-report"].map(id =>
|
||||
({ id, expectedRunCount: 1, automaticRetryPolicy: "single_attempt" }));
|
||||
return { id: "native-completion.runner-codex.local.assigned-skill-explicit-invocation",
|
||||
profile: { id: "runner-codex", generation: "native" }, environment: { id: "local" }, task: tasks[0],
|
||||
suite: { id: "native-completion", manualOnly: true, expectedMatrixSize: 6, tasks } };
|
||||
}
|
||||
describe("native completion selected-fixture policy", () => {
|
||||
it("admits an exact subset of the six declared one-turn single-attempt cells", () => {
|
||||
const base = selectedFixture();
|
||||
const executions = ["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(id => base.suite.tasks.map(task =>
|
||||
({ ...structuredClone(base), profile: { id, generation: "native" }, task,
|
||||
id: `native-completion.${id}.local.${task.id}` })));
|
||||
expect(() => assertNativeCompletionSelection(executions as unknown as MatrixExecution[])).not.toThrow();
|
||||
expect(() => assertNativeCompletionSelection([executions[0]!] as unknown as MatrixExecution[])).not.toThrow();
|
||||
});
|
||||
const invalid: Array<[string, (fixture: ReturnType<typeof selectedFixture>) => void]> = [
|
||||
["a legacy generation", f => { f.profile.generation = "legacy"; }],
|
||||
["an unlisted profile", f => { f.profile.id = "runner-other"; }],
|
||||
["a remote environment", f => { f.environment.id = "daytona"; }],
|
||||
["an unexpected case", f => { f.task!.id = "other-task"; }],
|
||||
["an extra expected turn", f => { f.task!.expectedRunCount = 2; }],
|
||||
["a missing single-attempt policy", f => { f.task!.automaticRetryPolicy = "default"; }],
|
||||
["a forged execution ID", f => { f.id = "other-cell"; }],
|
||||
["an automatic suite", f => { f.suite.manualOnly = false; }],
|
||||
["an expanded matrix", f => { f.suite.expectedMatrixSize = 7; }],
|
||||
["a missing original completion case", f => { f.suite.tasks.shift(); }],
|
||||
["an extra task", f => { f.suite.tasks.push({ ...f.suite.tasks[1]!, id: "extra-task" }); }],
|
||||
];
|
||||
for (const [label, mutate] of invalid) it(`rejects ${label} before credentials`, () => {
|
||||
const f = selectedFixture(); mutate(f);
|
||||
expect(() => assertNativeCompletionSelection([f] as unknown as MatrixExecution[])).toThrow("before provider execution");
|
||||
});
|
||||
it("leaves ordinary suites and their retry policies unchanged", () => {
|
||||
const f = selectedFixture(); f.suite.id = "ordinary-suite"; f.task!.automaticRetryPolicy = "default";
|
||||
expect(() => assertNativeCompletionSelection([f] as unknown as MatrixExecution[])).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
|
||||
const root = path.resolve(import.meta.dirname, "../..");
|
||||
export const NATIVE_COMPLETION_PREFLIGHT_ENV = "PAPERCLIP_RUNNER_E2E_NATIVE_COMPLETION_PREFLIGHT";
|
||||
|
||||
/** Explicit fixture policy is enforced even when the workflow supplies only --id. */
|
||||
export function assertNativeCompletionSelection(executions: readonly MatrixExecution[]): void {
|
||||
const profiles = ["runner-codex", "runner-acpx-claude", "runner-opencode"];
|
||||
const tasks = ["assigned-skill-explicit-invocation", "native-blocked-report"];
|
||||
for (const execution of executions.filter(execution => execution.suite.id === "native-completion")) {
|
||||
const suite = execution.suite;
|
||||
if (!profiles.includes(execution.profile.id) || execution.profile.generation !== "native" ||
|
||||
execution.profile.qualificationCandidate !== undefined || execution.environment.id !== "local" ||
|
||||
!tasks.includes(execution.task.id) || execution.task.expectedRunCount !== 1 ||
|
||||
(execution.task.minimumExpectedRunCount !== undefined && execution.task.minimumExpectedRunCount !== 1) ||
|
||||
execution.task.automaticRetryPolicy !== "single_attempt" ||
|
||||
execution.id !== `native-completion.${execution.profile.id}.local.${execution.task.id}` ||
|
||||
suite.manualOnly !== true || suite.expectedMatrixSize !== 6 || suite.tasks.length !== 2 ||
|
||||
!tasks.every(id => suite.tasks.filter(task => task.id === id && task.expectedRunCount === 1 &&
|
||||
task.automaticRetryPolicy === "single_attempt").length === 1)) {
|
||||
throw new Error("Native completion requires the declared local native fixtures, one turn and one attempt before provider execution.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Future credentials and ambient auth overrides stay outside prerequisite children. */
|
||||
export function nativeCompletionPreflightEnvironment(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
||||
return Object.fromEntries([
|
||||
"PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL",
|
||||
"CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS",
|
||||
].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]]));
|
||||
}
|
||||
|
||||
export function verifyNativeCompletionPreflight(receiptPath: string | undefined) {
|
||||
if (!receiptPath) throw new Error("Native completion has no prerequisite receipt; use the live launcher.");
|
||||
const run = spawnSync(process.execPath, [path.join(root, "tests/runner-e2e/native-completion-checks.mjs"), `--verify=${receiptPath}`], {
|
||||
cwd: root, env: nativeCompletionPreflightEnvironment(process.env), encoding: "utf8", timeout: 90_000,
|
||||
});
|
||||
if (run.status !== 0) throw new Error(`Native completion prerequisite verification failed: ${run.stderr || run.error?.message || run.status}`);
|
||||
const receipt = JSON.parse(run.stdout) as Record<string, unknown>;
|
||||
const expectedSourceSha = process.env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA?.trim();
|
||||
if (expectedSourceSha && receipt.sourceSha !== expectedSourceSha)
|
||||
throw new Error("Native completion prerequisite source SHA differs from the requested immutable source.");
|
||||
return receipt;
|
||||
}
|
||||
|
||||
export function prepareNativeCompletionPreflight(campaignDirectory: string) {
|
||||
const output = path.join(campaignDirectory, "native-completion-prerequisites", randomUUID());
|
||||
const receipt = path.join(output, "preflight.json");
|
||||
const run = spawnSync(process.execPath, [path.join(root, "tests/runner-e2e/native-completion-checks.mjs"), `--output-dir=${output}`], {
|
||||
cwd: root, env: nativeCompletionPreflightEnvironment(process.env), stdio: "inherit", timeout: 35 * 60_000,
|
||||
});
|
||||
if (run.status !== 0) throw new Error(`Native completion prerequisites failed before provider execution. Retained receipt: ${receipt}`);
|
||||
verifyNativeCompletionPreflight(receipt);
|
||||
return receipt;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import type { RunnerTaskFixture } from "./types.js";
|
||||
import { contextIntegrityTasks } from "./context-integrity-cases.js";
|
||||
|
||||
export const nativeBlockedReportTask: RunnerTaskFixture = {
|
||||
id: "native-blocked-report", label: "Native concrete blocker report", groups: ["native-completion"],
|
||||
workMode: "standard", flow: "single_turn", expectedRunCount: 1, automaticRetryPolicy: "single_attempt",
|
||||
attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 },
|
||||
expectedTerminalState: { issue: "blocked", run: "succeeded" },
|
||||
buildTitle: nonce => `Native blocker report ${nonce}`,
|
||||
buildVisibleMarker: nonce => `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}`,
|
||||
buildPrompt: nonce => [
|
||||
"Deployment cannot proceed because required release access has not been granted.",
|
||||
"Release Owner is the external owner of that grant. Do not attempt deployment, run shell/process commands, change files, invoke external services, create documents, agents, interactions or subtasks, or claim the access exists.",
|
||||
'Record the whole task as blocked and identify the owner as Release Owner. The exact unblock action is "Grant deployment access" (without quotation marks or trailing punctuation).',
|
||||
`Include BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")} exactly once in your final explanation.`,
|
||||
].join("\n"),
|
||||
buildMatchers: nonce => [
|
||||
{ kind: "issue_status", expected: "blocked" }, { kind: "run_status", expected: "succeeded" },
|
||||
{ kind: "runtime_mode", expected: "native" },
|
||||
{ kind: "message_contains", expected: `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}` },
|
||||
{ kind: "message_contains", expected: "Release Owner" },
|
||||
{ kind: "message_contains", expected: "Grant deployment access" },
|
||||
{ kind: "message_regex", pattern: "^(?![\\s\\S]*\\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\\b)[\\s\\S]*\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.reportedWorkDisposition", expected: "blocked" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.blocker.owner.name", expected: "Release Owner" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.blocker.unblockAction", expected: "Grant deployment access" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.blocker.scope", expected: "task_wide" },
|
||||
],
|
||||
};
|
||||
|
||||
const assignedSkill = contextIntegrityTasks.find(task => task.id === "assigned-skill-explicit-invocation");
|
||||
if (!assignedSkill) throw new Error("Missing original assigned-skill completion journey");
|
||||
// Preserve the original task prompt, skill, durable-document oracle and deadline.
|
||||
export const nativeCompletionTasks: readonly RunnerTaskFixture[] = [
|
||||
{ ...assignedSkill, automaticRetryPolicy: "single_attempt" }, nativeBlockedReportTask,
|
||||
];
|
||||
export function nativeCompletionDefinitionDigest() {
|
||||
const hash = createHash("sha256");
|
||||
for (const file of ["native-completion-cases.ts", "native-completion-scoring.ts", "native-completion-defaults.ts",
|
||||
"native-blocker-visible.ts", "native-completion-admission.ts", "native-completion-checks.mjs",
|
||||
"native-completion-source-contract.mjs", "automatic-retry.ts", "context-integrity-cases.ts", "context-integrity-flow.ts", "context-integrity-scoring.ts", "types.ts", "catalog.ts", "live-fixtures.ts", "runner.spec.ts", "launch.ts"]) {
|
||||
hash.update(file); hash.update(readFileSync(new URL(file, import.meta.url)));
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Independent, credential-free admission for the native completion comparison.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import {
|
||||
NATIVE_COMPLETION_SOURCE_CONTRACT, NATIVE_COMPLETION_SOURCE_FILES,
|
||||
nativeCompletionSourceFingerprint, nativeSourceSha256,
|
||||
} from "./native-completion-source-contract.mjs";
|
||||
|
||||
const root = resolve(import.meta.dirname, "../..");
|
||||
export const NATIVE_COMPLETION_PREFLIGHT_SCHEMA = "paperclip.native-completion-preflight.v1";
|
||||
export const NATIVE_COMPLETION_CELL_IDS = ["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile =>
|
||||
["assigned-skill-explicit-invocation", "native-blocked-report"].map(task => `native-completion.${profile}.local.${task}`));
|
||||
export function nativeCompletionGates(variant) {
|
||||
if (!["candidate", "historical"].includes(variant)) throw new Error("Unknown native completion source variant");
|
||||
return [
|
||||
{ id: "NC-schemas", name: "Variant-native completion schemas", cwd: "packages/paperclip-runner",
|
||||
files: ["src/contracts/completion-result.test.ts"],
|
||||
required: ["distinguishes user-facing answer content", "propagates answer and wait descriptions", "requires a reason code for verification that was not run"] },
|
||||
{ id: "NC-tools", name: "Native tool delivery and retained provider catalogs", cwd: "packages/paperclip-runner", files: [
|
||||
"src/drivers/codex/codex-app-server-driver.lifecycle.test.ts", "src/drivers/runner-tool-bridge.test.ts",
|
||||
"src/drivers/opencode/mcp-bridge.test.ts", "src/live/runnerd-codex-transport.test.ts"],
|
||||
testPattern: "binds to loopback|preserves stock Codex instructions on|includes ACPX terminal tools|preserves answer and internal wait descriptions",
|
||||
required: ["runner semantic MCP bridge binds to loopback", "OpenCode MCP bridge binds to loopback",
|
||||
"preserves stock Codex instructions on task recovery", "preserves stock Codex instructions on prepared recovery",
|
||||
"preserves stock Codex instructions on direct recovery", "includes ACPX terminal tools in the authenticated bridge catalog",
|
||||
...(variant === "candidate" ? ["codex", "opencode", "claude_managed", "aws_agentcore", "acpx"].map(provider =>
|
||||
`preserves answer and internal wait descriptions in the serialized native ${provider} tool catalog`) : [])] },
|
||||
{ id: "NC-resume", name: "Variant-native fingerprint and checkpoint refresh", cwd: ".",
|
||||
files: ["server/src/services/native-runtime/native-session-resume.test.ts"], testPattern: "refreshes retained",
|
||||
required: ["refreshes retained completion descriptions", "refreshes retained task-bound human-input description",
|
||||
...(variant === "candidate" ? ["native completion tool guidance"] : [])] },
|
||||
{ id: "NC-eval", name: "Independent native oracle, defaults, admission and attempt policy", cwd: ".",
|
||||
config: "tests/runner-e2e/vitest.config.ts", files: [
|
||||
"tests/runner-e2e/native-completion-scoring.test.ts", "tests/runner-e2e/native-completion-defaults.test.ts",
|
||||
"tests/runner-e2e/native-completion-admission.test.ts", "tests/runner-e2e/automatic-retry.test.ts",
|
||||
"tests/runner-e2e/context-integrity.test.ts", "tests/runner-e2e/context-integrity-evidence.test.ts",
|
||||
"tests/runner-e2e/context-integrity-flow-logs.test.ts"],
|
||||
required: ["allows toolchain paths and excludes every present or future credential",
|
||||
"retains credential-free prerequisites inside the exact campaign root",
|
||||
"retains the first transient_infrastructure failure without a second attempt",
|
||||
"retains the first provider_variance failure without a second attempt"] },
|
||||
];
|
||||
}
|
||||
export const NATIVE_COMPLETION_COMMAND_GATE_IDS = ["NC-node", "NC-typecheck", "NC-manifest", "NC-discovery"];
|
||||
export function gradeNativeCompletionGate(gate, report, exitCode) {
|
||||
const assertions = (report?.testResults ?? []).flatMap(file => file.assertionResults ?? []);
|
||||
const requirements = gate.required.map(name => ({ name, passed: assertions.some(assertion =>
|
||||
assertion.fullName?.includes(name) && assertion.status === "passed") }));
|
||||
const files = gate.files.map(file => ({ file, passed: (report?.testResults ?? []).some(result =>
|
||||
result.name?.endsWith(file) && result.status === "passed" && result.assertionResults?.some(assertion => assertion.status === "passed") &&
|
||||
result.assertionResults.every(assertion => assertion.status === "passed" || (gate.testPattern && assertion.status === "skipped" &&
|
||||
!gate.required.some(name => assertion.fullName?.includes(name))))) }));
|
||||
return { id: gate.id, name: gate.name, passed: exitCode === 0 && requirements.every(row => row.passed) && files.every(row => row.passed),
|
||||
exitCode, files, requirements, total: report?.numTotalTests ?? 0, passedTests: report?.numPassedTests ?? 0,
|
||||
failedTests: report?.numFailedTests ?? 0, pendingTests: report?.numPendingTests ?? 0 };
|
||||
}
|
||||
|
||||
export function gradeNativeCompletionDiscovery(output, exitCode) {
|
||||
const lines = output.trim().split(/\r?\n/).filter(Boolean);
|
||||
const rows = lines.slice(1).map(line => line.split("\t"));
|
||||
const ids = rows.map(row => row[0]);
|
||||
return { passed: exitCode === 0 && lines[0] === "ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS" &&
|
||||
rows.length === 6 && new Set(ids).size === 6 && NATIVE_COMPLETION_CELL_IDS.every(id => ids.includes(id)) &&
|
||||
rows.every(row => row.length === 6 && row[1] === "native-completion" && row[2] === "native" && row[4]?.trim() && row[5]?.trim()),
|
||||
executionIds: ids, expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1 };
|
||||
}
|
||||
export function nativeCompletionPrerequisiteEnvironment(source) {
|
||||
return Object.fromEntries(["PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL",
|
||||
"CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS"].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]]));
|
||||
}
|
||||
function git(args) { return spawnSync("git", args, { cwd: root, encoding: "utf8" }); }
|
||||
function currentSource() {
|
||||
const source = nativeCompletionSourceFingerprint();
|
||||
const head = git(["rev-parse", "HEAD"]), ancestor = git(["merge-base", "--is-ancestor", source.baseSha, "HEAD"]);
|
||||
const tracked = git(["ls-files", "--error-unmatch", "--", ...NATIVE_COMPLETION_SOURCE_FILES]);
|
||||
const clean = git(["diff", "--quiet", "HEAD"]);
|
||||
const status = git(["status", "--porcelain", "--untracked-files=normal"]);
|
||||
return { ...source, sha: head.status === 0 ? head.stdout.trim() : null,
|
||||
layering: ancestor.status === 0, immutable: tracked.status === 0 && clean.status === 0 && status.status === 0 && status.stdout.trim() === "" };
|
||||
}
|
||||
function buildOutputFingerprint() {
|
||||
const hash = createHash("sha256");
|
||||
function visit(relative) {
|
||||
for (const entry of readdirSync(join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const file = join(relative, entry.name);
|
||||
if (entry.isDirectory()) visit(file);
|
||||
else if (entry.isFile()) { const bytes = readFileSync(join(root, file)); hash.update(JSON.stringify([file, bytes.length])).update(bytes); }
|
||||
else throw new Error(`Unexpected build output: ${file}`);
|
||||
}
|
||||
}
|
||||
for (const directory of ["packages/shared/dist", "packages/plugins/sdk/dist", "packages/paperclip-runner/dist"]) {
|
||||
if (!existsSync(join(root, directory, "index.js"))) throw new Error(`Missing prerequisite build output: ${directory}`);
|
||||
visit(directory);
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
export function assertNativeCompletionPreflightReceipt(report, current) {
|
||||
const expected = [...nativeCompletionGates(current.variant).map(gate => gate.id), ...NATIVE_COMPLETION_COMMAND_GATE_IDS];
|
||||
if (report?.schema !== NATIVE_COMPLETION_PREFLIGHT_SCHEMA || report.passed !== true || report.providerCalls !== 0 ||
|
||||
report.live !== "not_run" || report.sourceSha !== current.sha || !/^[a-f0-9]{40}$/.test(current.sha ?? "") ||
|
||||
![current.fingerprint, current.fixtureFingerprint, current.manifestFingerprint, current.buildOutputFingerprint]
|
||||
.every(value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value)) ||
|
||||
report.sourceFingerprint !== current.fingerprint || report.fixtureFingerprint !== current.fixtureFingerprint ||
|
||||
report.manifestFingerprint !== current.manifestFingerprint || report.variant !== current.variant ||
|
||||
report.baseSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha || report.archiveSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha ||
|
||||
report.layering !== true || current.layering !== true || report.immutable !== true || current.immutable !== true ||
|
||||
!Array.isArray(report.sourceErrors) || report.sourceErrors.length !== 0 || current.sourceErrors?.length !== 0 ||
|
||||
report.setup?.passed !== true || report.setup.sdkExitCode !== 0 || report.setup.runnerTypeScriptExitCode !== 0 ||
|
||||
report.setup.buildOutputFingerprint !== current.buildOutputFingerprint ||
|
||||
!Array.isArray(report.setup.evidence) || report.setup.evidence.length !== 2 ||
|
||||
["setup-sdk.txt", "setup-runner-typescript.txt"].some(file => report.setup.evidence.filter(row =>
|
||||
row.file === file && /^[a-f0-9]{64}$/.test(row.sha256 ?? "")).length !== 1) ||
|
||||
!Array.isArray(report.gates) || report.gates.length !== expected.length ||
|
||||
expected.some(id => report.gates.filter(gate => gate.id === id && gate.passed === true && gate.exitCode === 0).length !== 1) ||
|
||||
report.expectedCells !== 6 || report.expectedTurns !== 6 || report.maximumAttemptsPerCell !== 1) {
|
||||
throw new Error("Native completion requires passing prerequisites for this exact immutable source SHA, variant and fingerprint.");
|
||||
}
|
||||
return report;
|
||||
}
|
||||
function runCommand(command, args, env, timeout = 10 * 60_000, cwd = root) {
|
||||
return spawnSync(command, args, { cwd, env, encoding: "utf8", timeout });
|
||||
}
|
||||
function capture(output, name, run) {
|
||||
const file = `${name}.txt`, bytes = `${run?.stdout ?? ""}\n${run?.stderr ?? ""}`;
|
||||
writeFileSync(join(output, file), bytes);
|
||||
return { file, sha256: nativeSourceSha256(bytes) };
|
||||
}
|
||||
export function assertRetainedNativeCompletionEvidence(output, evidence) {
|
||||
if (!evidence || typeof evidence.file !== "string" || !/^[A-Za-z0-9_.-]+$/.test(evidence.file) ||
|
||||
nativeSourceSha256(readFileSync(join(output, evidence.file))) !== evidence.sha256)
|
||||
throw new Error("Native completion retained prerequisite evidence changed or is unavailable.");
|
||||
return readFileSync(join(output, evidence.file), "utf8");
|
||||
}
|
||||
function manifestCommands(env) {
|
||||
return ["generate-capability-contract.mjs", "check-capability-inventory.mjs"].map(file => runCommand(process.execPath,
|
||||
[join(root, "packages/paperclip-runner/scripts", file), ...(file.startsWith("generate-") ? ["--check"] : [])], env));
|
||||
}
|
||||
|
||||
export function main(args = process.argv.slice(2)) {
|
||||
if (args.includes("--list")) {
|
||||
console.log(JSON.stringify({ variants: ["candidate", "historical"], gates: nativeCompletionGates("candidate"),
|
||||
commands: NATIVE_COMPLETION_COMMAND_GATE_IDS, cells: NATIVE_COMPLETION_CELL_IDS, providerCalls: 0 }, null, 2)); return;
|
||||
}
|
||||
if (args.some(arg => !arg.startsWith("--output-dir=") && !arg.startsWith("--verify=")))
|
||||
throw new Error("Use --list, --output-dir=<path> or --verify=<receipt>; never paid providers.");
|
||||
const source = currentSource(), env = nativeCompletionPrerequisiteEnvironment(process.env);
|
||||
const verify = args.find(arg => arg.startsWith("--verify="))?.slice("--verify=".length);
|
||||
if (verify) {
|
||||
const report = assertNativeCompletionPreflightReceipt(JSON.parse(readFileSync(verify, "utf8")), {
|
||||
...source, buildOutputFingerprint: buildOutputFingerprint(),
|
||||
});
|
||||
const output = resolve(verify, "..");
|
||||
for (const evidence of report.setup.evidence) assertRetainedNativeCompletionEvidence(output, evidence);
|
||||
for (const gate of nativeCompletionGates(source.variant)) {
|
||||
const retained = report.gates.find(row => row.id === gate.id);
|
||||
const grade = gradeNativeCompletionGate(gate, JSON.parse(assertRetainedNativeCompletionEvidence(output, retained.evidence)), 0);
|
||||
if (!grade.passed) throw new Error(`Missing or failed retained native prerequisite assertions: ${gate.id}`);
|
||||
}
|
||||
for (const id of NATIVE_COMPLETION_COMMAND_GATE_IDS) {
|
||||
const retained = report.gates.find(row => row.id === id), text = assertRetainedNativeCompletionEvidence(output, retained.evidence);
|
||||
if (id === "NC-node" && (!/# fail 0\b/.test(text) || !/# tests [1-9]\d*\b/.test(text)))
|
||||
throw new Error("Missing passing native admission calibrations.");
|
||||
if (id === "NC-discovery" && !gradeNativeCompletionDiscovery(text.split("\n\n")[0], 0).passed)
|
||||
throw new Error("Native completion six-cell discovery changed.");
|
||||
if (id === "NC-manifest" && manifestCommands(env).some(run => run.status !== 0))
|
||||
throw new Error("Generated native capability manifests are stale.");
|
||||
}
|
||||
console.log(JSON.stringify(report)); return report;
|
||||
}
|
||||
const output = resolve(args.find(arg => arg.startsWith("--output-dir="))?.slice("--output-dir=".length) ??
|
||||
join(root, "tests/runner-e2e/results", `native-completion-preflight-${new Date().toISOString().replaceAll(":", "-")}`));
|
||||
mkdirSync(output, { recursive: true });
|
||||
const report = { schema: NATIVE_COMPLETION_PREFLIGHT_SCHEMA, sourceSha: source.sha, sourceFingerprint: source.fingerprint,
|
||||
fixtureFingerprint: source.fixtureFingerprint, manifestFingerprint: source.manifestFingerprint,
|
||||
variant: source.variant, baseSha: source.baseSha, archiveSha: source.archiveSha,
|
||||
sourceErrors: source.sourceErrors, immutable: source.immutable, layering: source.layering,
|
||||
measuredAt: new Date().toISOString(), providerCalls: 0, live: "not_run", expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1,
|
||||
setup: { passed: false, sdkExitCode: null, runnerTypeScriptExitCode: null, evidence: [] }, gates: [], passed: false };
|
||||
if (!source.sha || source.sourceErrors.length || !source.layering || !source.immutable) {
|
||||
writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n");
|
||||
console.error("Native completion source admission failed; commit the exact native-only source and fixtures before qualification.");
|
||||
process.exitCode = 1; return report;
|
||||
}
|
||||
const sdk = runCommand(process.execPath, [join(root, "scripts/ensure-plugin-build-deps.mjs")], env, 5 * 60_000);
|
||||
report.setup.sdkExitCode = sdk.status; report.setup.evidence.push(capture(output, "setup-sdk", sdk));
|
||||
const runner = sdk.status === 0 ? runCommand("pnpm", ["--filter", "@paperclipai/paperclip-runner", "build:typescript"], env) : null;
|
||||
report.setup.runnerTypeScriptExitCode = runner?.status ?? null;
|
||||
report.setup.evidence.push(capture(output, "setup-runner-typescript", runner));
|
||||
report.setup.passed = sdk.status === 0 && runner?.status === 0;
|
||||
if (report.setup.passed) {
|
||||
report.setup.buildOutputFingerprint = buildOutputFingerprint();
|
||||
for (const gate of nativeCompletionGates(source.variant)) {
|
||||
console.log(`Checking ${gate.id}: ${gate.name}`);
|
||||
const file = `${gate.id}.json`, run = runCommand(process.execPath, [join(root, "node_modules/vitest/vitest.mjs"), "run", ...gate.files,
|
||||
...(gate.config ? ["--config", gate.config] : []), ...(gate.testPattern ? ["--testNamePattern", gate.testPattern] : []),
|
||||
"--reporter=default", "--reporter=json", `--outputFile.json=${join(output, file)}`], env, 10 * 60_000, resolve(root, gate.cwd));
|
||||
capture(output, gate.id, run);
|
||||
let result; try { result = JSON.parse(readFileSync(join(output, file), "utf8")); } catch { /* Missing discovery fails closed. */ }
|
||||
report.gates.push({ ...gradeNativeCompletionGate(gate, result, run.status),
|
||||
evidence: existsSync(join(output, file)) ? { file, sha256: nativeSourceSha256(readFileSync(join(output, file))) } : null });
|
||||
}
|
||||
const commands = [
|
||||
{ id: "NC-node", command: process.execPath, args: ["--test", "--test-reporter=tap", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"] },
|
||||
{ id: "NC-typecheck", command: process.execPath, args: ["node_modules/typescript/bin/tsc", "-p", "tests/runner-e2e/tsconfig.json"] },
|
||||
{ id: "NC-discovery", command: process.execPath, args: ["--import", "./server/node_modules/tsx/dist/loader.mjs", "tests/runner-e2e/launch.ts", "--list", "--suite", "native-completion"] },
|
||||
];
|
||||
for (const command of commands) {
|
||||
console.log(`Checking ${command.id}`);
|
||||
const run = runCommand(command.command, command.args, env);
|
||||
report.gates.push({ id: command.id, passed: run.status === 0 &&
|
||||
(command.id !== "NC-discovery" || gradeNativeCompletionDiscovery(run.stdout, run.status).passed) &&
|
||||
(command.id !== "NC-node" || /# fail 0\b/.test(run.stdout) && /# tests [1-9]\d*\b/.test(run.stdout)),
|
||||
exitCode: run.status, evidence: capture(output, command.id, run) });
|
||||
}
|
||||
const manifest = manifestCommands(env), combined = { stdout: manifest.map(run => run.stdout ?? "").join("\n"), stderr: manifest.map(run => run.stderr ?? "").join("\n") };
|
||||
report.gates.push({ id: "NC-manifest", passed: manifest.every(run => run.status === 0),
|
||||
exitCode: manifest.find(run => run.status !== 0)?.status ?? 0, evidence: capture(output, "NC-manifest", combined) });
|
||||
const after = currentSource();
|
||||
report.passed = report.gates.every(gate => gate.passed) && after.immutable && after.layering &&
|
||||
after.sha === source.sha && after.fingerprint === source.fingerprint && after.sourceErrors.length === 0 &&
|
||||
buildOutputFingerprint() === report.setup.buildOutputFingerprint;
|
||||
}
|
||||
writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n");
|
||||
console.log(`Native completion prerequisite evidence: ${join(output, "preflight.json")}`);
|
||||
if (!report.passed) process.exitCode = 1;
|
||||
return report;
|
||||
}
|
||||
if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) main();
|
||||
@@ -0,0 +1,118 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import {
|
||||
nativeCompletionGates, gradeNativeCompletionGate, gradeNativeCompletionDiscovery,
|
||||
assertNativeCompletionPreflightReceipt, assertRetainedNativeCompletionEvidence, nativeCompletionPrerequisiteEnvironment,
|
||||
NATIVE_COMPLETION_PREFLIGHT_SCHEMA, NATIVE_COMPLETION_CELL_IDS, NATIVE_COMPLETION_COMMAND_GATE_IDS,
|
||||
} from "./native-completion-checks.mjs";
|
||||
import { nativeSourceSha256, NATIVE_COMPLETION_SOURCE_CONTRACT } from "./native-completion-source-contract.mjs";
|
||||
|
||||
const gate = { id: "NC-test", name: "Boundary", files: ["boundary.test.ts"], required: ["native assertion"] };
|
||||
const report = () => ({ testResults: [{ name: "/repo/boundary.test.ts", status: "passed", assertionResults: [
|
||||
{ fullName: "Boundary native assertion", status: "passed" }] }], numTotalTests: 1, numPassedTests: 1 });
|
||||
test("native completion prerequisite gates exclude reduced manual and shared legacy assertions", () => {
|
||||
for (const variant of ["candidate", "historical"]) {
|
||||
const gates = nativeCompletionGates(variant);
|
||||
assert.deepEqual(gates.map(gate => gate.id), ["NC-schemas", "NC-tools", "NC-resume", "NC-eval"]);
|
||||
assert.ok(gates.every(gate => gate.files.length && gate.required.length));
|
||||
assert.ok(!gates.flatMap(gate => gate.files).some(file => /agent-skills-routes|onboarding-first-task-assets|server-utils\.test|prompt-sections/.test(file)));
|
||||
}
|
||||
assert.ok(nativeCompletionGates("candidate")[1].required.some(name => name.includes("serialized native codex")));
|
||||
assert.ok(!nativeCompletionGates("historical")[1].required.some(name => name.includes("serialized native codex")));
|
||||
assert.ok(!nativeCompletionGates("historical")[2].required.includes("native completion tool guidance"));
|
||||
assert.throws(() => nativeCompletionGates("unknown"));
|
||||
});
|
||||
test("native completion prerequisite admits an executed passing boundary", () => assert.equal(gradeNativeCompletionGate(gate, report(), 0).passed, true));
|
||||
for (const status of ["pending", "skipped", "failed", "todo"]) test(`native completion prerequisite rejects a ${status} required assertion`, () => {
|
||||
const observation = report(); observation.testResults[0].assertionResults[0].status = status;
|
||||
assert.equal(gradeNativeCompletionGate(gate, observation, 0).passed, false);
|
||||
});
|
||||
test("native completion prerequisite rejects missing discovery, unrelated assertions and failed commands", () => {
|
||||
assert.equal(gradeNativeCompletionGate(gate, undefined, 0).passed, false);
|
||||
const different = report(); different.testResults[0].assertionResults[0].fullName = "Other";
|
||||
assert.equal(gradeNativeCompletionGate(gate, different, 0).passed, false);
|
||||
assert.equal(gradeNativeCompletionGate(gate, report(), 1).passed, false);
|
||||
assert.equal(gradeNativeCompletionGate({ ...gate, files: [...gate.files, "undiscovered.test.ts"] }, report(), 0).passed, false);
|
||||
});
|
||||
test("native completion prerequisite allows filtering unrelated assertions but never the required assertion", () => {
|
||||
const observation = report(); observation.testResults[0].assertionResults.push({ fullName: "Unrelated", status: "skipped" });
|
||||
assert.equal(gradeNativeCompletionGate({ ...gate, testPattern: "native assertion" }, observation, 0).passed, true);
|
||||
assert.equal(gradeNativeCompletionGate(gate, observation, 0).passed, false);
|
||||
observation.testResults[0].assertionResults[0].status = "skipped";
|
||||
assert.equal(gradeNativeCompletionGate({ ...gate, testPattern: "native assertion" }, observation, 0).passed, false);
|
||||
});
|
||||
const discovery = () => ["ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS", ...NATIVE_COMPLETION_CELL_IDS.map(id =>
|
||||
`${id}\tnative-completion\tnative\tprovider\tmodel\tAPI_KEY`)].join("\n");
|
||||
test("native completion prerequisite discovers exactly six native single-turn cells", () => {
|
||||
assert.deepEqual(gradeNativeCompletionDiscovery(discovery(), 0), { passed: true, executionIds: NATIVE_COMPLETION_CELL_IDS,
|
||||
expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1 });
|
||||
});
|
||||
for (const kind of ["missing", "duplicate", "legacy", "extra", "failed", "missing-model", "wrong-task"]) test(`native completion prerequisite rejects ${kind} discovery`, () => {
|
||||
let output = discovery();
|
||||
if (kind === "missing") output = output.split("\n").slice(0, -1).join("\n");
|
||||
if (kind === "duplicate") output = output.replace(NATIVE_COMPLETION_CELL_IDS[1], NATIVE_COMPLETION_CELL_IDS[0]);
|
||||
if (kind === "legacy") output = output.replace("\tnative\t", "\tlegacy\t");
|
||||
if (kind === "extra") output += `\n${output.split("\n")[1]}`;
|
||||
if (kind === "missing-model") output = output.replace("\tmodel\t", "\t\t");
|
||||
if (kind === "wrong-task") output = output.replace("assigned-skill-explicit-invocation", "other-case");
|
||||
assert.equal(gradeNativeCompletionDiscovery(output, kind === "failed" ? 1 : 0).passed, false);
|
||||
});
|
||||
test("native completion prerequisite excludes future credentials and auth overrides from every child", () => {
|
||||
assert.deepEqual(nativeCompletionPrerequisiteEnvironment({ PATH: "/bin", HOME: "/fixture", CI: "true",
|
||||
OPENAI_API_KEY: "secret", ANTHROPIC_API_KEY: "secret", FUTURE_TOKEN: "secret", NODE_OPTIONS: "secret", GH_TOKEN: "secret" }),
|
||||
{ PATH: "/bin", HOME: "/fixture", CI: "true" });
|
||||
});
|
||||
const current = variant => ({ sha: "a".repeat(40), fingerprint: "b".repeat(64), fixtureFingerprint: "c".repeat(64),
|
||||
manifestFingerprint: "d".repeat(64), buildOutputFingerprint: "e".repeat(64), variant, immutable: true, layering: true, sourceErrors: [] });
|
||||
const receipt = variant => ({ schema: NATIVE_COMPLETION_PREFLIGHT_SCHEMA, passed: true, providerCalls: 0, live: "not_run",
|
||||
sourceSha: current(variant).sha, sourceFingerprint: current(variant).fingerprint, fixtureFingerprint: current(variant).fixtureFingerprint,
|
||||
manifestFingerprint: current(variant).manifestFingerprint, variant,
|
||||
baseSha: NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha, archiveSha: NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha, immutable: true, layering: true, sourceErrors: [],
|
||||
expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1,
|
||||
setup: { passed: true, sdkExitCode: 0, runnerTypeScriptExitCode: 0, buildOutputFingerprint: current(variant).buildOutputFingerprint,
|
||||
evidence: ["setup-sdk.txt", "setup-runner-typescript.txt"].map(file => ({ file, sha256: "f".repeat(64) })) },
|
||||
gates: [...nativeCompletionGates(variant).map(gate => gate.id), ...NATIVE_COMPLETION_COMMAND_GATE_IDS].map(id => ({ id, passed: true, exitCode: 0 })) });
|
||||
for (const variant of ["candidate", "historical"]) test(`native completion prerequisite admits only the exact immutable ${variant} receipt`, () => {
|
||||
assert.equal(assertNativeCompletionPreflightReceipt(receipt(variant), current(variant)).passed, true);
|
||||
});
|
||||
const negatives = [
|
||||
["old SHA", r => { r.sourceSha = "f".repeat(40); }], ["changed source", r => { r.sourceFingerprint = "f".repeat(64); }],
|
||||
["changed fixtures", r => { r.fixtureFingerprint = "f".repeat(64); }], ["stale manifests", r => { r.manifestFingerprint = "f".repeat(64); }],
|
||||
["wrong variant", r => { r.variant = "historical"; }], ["wrong base", r => { r.baseSha = "f".repeat(40); }],
|
||||
["wrong archive", r => { r.archiveSha = "f".repeat(40); }], ["dirty source", r => { r.immutable = false; }],
|
||||
["wrong master layering", r => { r.layering = false; }], ["source errors", r => { r.sourceErrors.push("missing.ts"); }],
|
||||
["paid provider calls", r => { r.providerCalls = 1; }], ["paid live run", r => { r.live = "run"; }],
|
||||
["missing gate", r => { r.gates.pop(); }], ["duplicate gate", r => { r.gates[1] = r.gates[0]; }],
|
||||
["failed gate", r => { r.gates[0].passed = false; }], ["nonzero exit", r => { r.gates[0].exitCode = 1; }],
|
||||
["failed SDK build", r => { r.setup.sdkExitCode = 1; }], ["missing Runner build", r => { delete r.setup.runnerTypeScriptExitCode; }],
|
||||
["changed build output", r => { r.setup.buildOutputFingerprint = "f".repeat(64); }],
|
||||
["extra cells", r => { r.expectedCells = 7; }], ["extra turns", r => { r.expectedTurns = 7; }],
|
||||
["additional attempts", r => { r.maximumAttemptsPerCell = 2; }],
|
||||
["missing setup evidence", r => { r.setup.evidence.pop(); }],
|
||||
["duplicate setup evidence", r => { r.setup.evidence[1] = r.setup.evidence[0]; }],
|
||||
];
|
||||
for (const [name, mutate] of negatives) test(`native completion prerequisite rejects ${name} before providers`, () => {
|
||||
const r = receipt("candidate"); mutate(r);
|
||||
assert.throws(() => assertNativeCompletionPreflightReceipt(r, current("candidate")), /exact immutable source SHA, variant and fingerprint/);
|
||||
});
|
||||
test("native completion prerequisite rejects a current dirty source and unavailable digests", () => {
|
||||
assert.throws(() => assertNativeCompletionPreflightReceipt(receipt("candidate"), { ...current("candidate"), immutable: false }));
|
||||
assert.throws(() => assertNativeCompletionPreflightReceipt(receipt("candidate"), { ...current("candidate"), fingerprint: null }));
|
||||
});
|
||||
|
||||
test("native completion prerequisite rejects changed or missing retained evidence and unsafe paths", () => {
|
||||
const output = mkdtempSync(join(tmpdir(), "native-completion-retained-"));
|
||||
try {
|
||||
const bytes = '{"passing":"native assertions"}', evidence = { file: "NC-tools.json", sha256: nativeSourceSha256(bytes) };
|
||||
writeFileSync(join(output, evidence.file), bytes);
|
||||
assert.equal(assertRetainedNativeCompletionEvidence(output, evidence), bytes);
|
||||
writeFileSync(join(output, evidence.file), "Changed retained assertions");
|
||||
assert.throws(() => assertRetainedNativeCompletionEvidence(output, evidence), /changed or is unavailable/);
|
||||
assert.throws(() => assertRetainedNativeCompletionEvidence(output, { file: "missing.json", sha256: evidence.sha256 }));
|
||||
assert.throws(() => assertRetainedNativeCompletionEvidence(output, { file: "../elsewhere.json", sha256: evidence.sha256 }));
|
||||
assert.throws(() => assertRetainedNativeCompletionEvidence(output, null));
|
||||
} finally { rmSync(output, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { captureNativeDefault, gradeNativeDefault, nativeCompletionProfile, nativeCompletionWorkspaceDigest, NATIVE_MASTER_DEFAULT_SHA256, type NativeDefaultReceipt } from "./native-completion-defaults.js";
|
||||
import { mkdtemp, mkdir, writeFile, symlink, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { RunnerProfileFixture } from "./types.js";
|
||||
import { runnerSuites } from "./catalog.js";
|
||||
import { contextIntegrityTasks } from "./context-integrity-cases.js";
|
||||
|
||||
const receipt = (): NativeDefaultReceipt => ({ schema: "paperclip.native-completion-default.v1", agentId: "agent", companyId: "company",
|
||||
entryFile: "AGENTS.md", files: [{ path: "AGENTS.md", sha256: NATIVE_MASTER_DEFAULT_SHA256, bytes: 100 }], budgets: { company: 1000, agent: 1000 } });
|
||||
describe("native production defaults", () => {
|
||||
it("hashes scoped file and symlink identity without reading an outside target", async () => {
|
||||
const temporary = await mkdtemp(path.join(tmpdir(), "native-workspace-calibration-"));
|
||||
try {
|
||||
const root = path.join(temporary, "workspace"), outside = path.join(temporary, "outside");
|
||||
await mkdir(root); await mkdir(outside);
|
||||
await writeFile(path.join(root, "file"), "initial"); await writeFile(path.join(outside, "private"), "first");
|
||||
await symlink(outside, path.join(root, "skill-link"));
|
||||
const first = await nativeCompletionWorkspaceDigest(root);
|
||||
await writeFile(path.join(outside, "private"), "changed outside");
|
||||
expect(await nativeCompletionWorkspaceDigest(root)).toBe(first);
|
||||
await writeFile(path.join(root, "file"), "changed fixture");
|
||||
expect(await nativeCompletionWorkspaceDigest(root)).not.toBe(first);
|
||||
} finally { await rm(temporary, { recursive: true, force: true }); }
|
||||
});
|
||||
it("preserves the original assigned-skill prompt flow and oracle in exactly six explicit cells", () => {
|
||||
const suite = runnerSuites.find(value => value.id === "native-completion")!;
|
||||
const original = contextIntegrityTasks.find(value => value.id === "assigned-skill-explicit-invocation")!;
|
||||
const completion = suite.tasks.find(value => value.id === original.id)!;
|
||||
expect({ ...completion, automaticRetryPolicy: undefined }).toEqual({ ...original, automaticRetryPolicy: undefined });
|
||||
expect(suite.manualOnly).toBe(true);
|
||||
expect(suite.expectedMatrixSize).toBe(6);
|
||||
expect(suite.tasks).toHaveLength(2);
|
||||
expect(suite.tasks.every(task => task.expectedRunCount === 1 && task.automaticRetryPolicy === "single_attempt")).toBe(true);
|
||||
expect(suite.environments.map(value => value.id)).toEqual(["local"]);
|
||||
expect(suite.profiles.map(value => value.id).sort()).toEqual(["runner-acpx-claude", "runner-codex", "runner-opencode"]);
|
||||
});
|
||||
it("accepts only the unchanged master default and bounded budgets", () => expect(gradeNativeDefault(receipt()).passed).toBe(true));
|
||||
it.each(["qa", "reduced", "extra-file", "wrong-entry", "company-unbounded", "agent-unbounded"])("rejects %s override", name => {
|
||||
const value = receipt();
|
||||
if (name === "qa" || name === "reduced") value.files[0]!.sha256 = name;
|
||||
if (name === "extra-file") value.files.push({ path: "HEARTBEAT.md", sha256: "x", bytes: 1 });
|
||||
if (name === "wrong-entry") value.entryFile = "OTHER.md";
|
||||
if (name === "company-unbounded") value.budgets.company = 0;
|
||||
if (name === "agent-unbounded") value.budgets.agent = 0;
|
||||
expect(gradeNativeDefault(value).passed).toBe(false);
|
||||
});
|
||||
it("strips only the fixture bundle and preserves auth skills permissions and model", () => {
|
||||
const original = { instructionsBundle: { entryFile: "AGENTS.md", files: { "AGENTS.md": "QA" } },
|
||||
adapterConfig: { model: "same", env: { KEY: { type: "secret_ref", secretId: "id" } } },
|
||||
runtimeConfig: { permissionMode: "same" }, skills: ["same"], budgetMonthlyCents: 0 };
|
||||
const profile = { generation: "native", buildAgent: () => original } as unknown as RunnerProfileFixture;
|
||||
const wrapped = nativeCompletionProfile(profile).buildAgent({} as never);
|
||||
expect(wrapped).toEqual({ ...original, instructionsBundle: undefined, budgetMonthlyCents: 1000 });
|
||||
expect(Object.hasOwn(wrapped, "instructionsBundle")).toBe(false);
|
||||
expect(original.instructionsBundle).toBeDefined();
|
||||
expect(() => nativeCompletionProfile({ ...profile, generation: "legacy" })).toThrow();
|
||||
});
|
||||
it("hashes actual public bundle bytes before execution without publishing the body", async () => {
|
||||
const content = readFileSync(new URL("../../server/src/onboarding-assets/default/AGENTS.md", import.meta.url), "utf8");
|
||||
const called: string[] = [];
|
||||
const api = { async get<T>(path: string): Promise<T> {
|
||||
called.push(path);
|
||||
return (path.endsWith("instructions-bundle") ? { entryFile: "AGENTS.md", files: [{ path: "AGENTS.md" }] }
|
||||
: path.includes("instructions-bundle/file") ? { content } : { budgetMonthlyCents: 1000 }) as T;
|
||||
} };
|
||||
const actual = await captureNativeDefault({ api, agentId: "agent", companyId: "company" });
|
||||
expect(gradeNativeDefault(actual).passed).toBe(true);
|
||||
expect(called).toContain("/api/agents/agent/instructions-bundle/file?path=AGENTS.md");
|
||||
expect(JSON.stringify(actual)).not.toContain(content);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { lstat, readFile, readdir, readlink } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { RunnerProfileFixture } from "./types.js";
|
||||
|
||||
export const NATIVE_COMPLETION_BUDGET_CENTS = 1_000;
|
||||
// Independent, immutable current-master default; not the reduced #14948 manual.
|
||||
export const NATIVE_MASTER_DEFAULT_SHA256 = "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f";
|
||||
|
||||
/** Scoped blocker proof: hash symlink identity without following any target. */
|
||||
export async function nativeCompletionWorkspaceDigest(root: string): Promise<string> {
|
||||
const hash = createHash("sha256");
|
||||
async function visit(relative: string) {
|
||||
const entries = await readdir(path.join(root, relative), { withFileTypes: true });
|
||||
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const file = path.join(relative, entry.name), absolute = path.join(root, file), stat = await lstat(absolute);
|
||||
hash.update(JSON.stringify([file, stat.isSymbolicLink() ? "symlink" : stat.isDirectory() ? "directory" : stat.isFile() ? "file" : "other"]));
|
||||
if (stat.isSymbolicLink()) hash.update(await readlink(absolute));
|
||||
else if (stat.isDirectory()) await visit(file);
|
||||
else if (stat.isFile()) hash.update(await readFile(absolute));
|
||||
}
|
||||
}
|
||||
await visit(""); return hash.digest("hex");
|
||||
}
|
||||
|
||||
export function nativeCompletionProfile(profile: RunnerProfileFixture): RunnerProfileFixture {
|
||||
if (profile.generation !== "native") throw new Error("Native completion requires a native profile");
|
||||
return { ...profile, buildAgent(input) {
|
||||
const { instructionsBundle: _fixtureBundle, ...agent } = profile.buildAgent(input);
|
||||
return { ...agent, budgetMonthlyCents: NATIVE_COMPLETION_BUDGET_CENTS };
|
||||
} };
|
||||
}
|
||||
|
||||
export interface NativeDefaultReceipt {
|
||||
schema: "paperclip.native-completion-default.v1";
|
||||
agentId: string;
|
||||
companyId: string;
|
||||
entryFile: unknown;
|
||||
files: Array<{ path: string; sha256: string; bytes: number }>;
|
||||
budgets: { company: unknown; agent: unknown };
|
||||
}
|
||||
|
||||
export function gradeNativeDefault(receipt: NativeDefaultReceipt) {
|
||||
const checks = [
|
||||
{ id: "production-default-bundle", passed: receipt.entryFile === "AGENTS.md"
|
||||
&& receipt.files.length === 1 && receipt.files[0]?.path === "AGENTS.md"
|
||||
&& receipt.files[0]?.sha256 === NATIVE_MASTER_DEFAULT_SHA256,
|
||||
detail: "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." },
|
||||
{ id: "bounded-company-and-agent", passed: receipt.budgets.company === NATIVE_COMPLETION_BUDGET_CENTS
|
||||
&& receipt.budgets.agent === NATIVE_COMPLETION_BUDGET_CENTS,
|
||||
detail: "Both public budgets enforce the declared 1,000-cent hard stop." },
|
||||
];
|
||||
return { passed: checks.every(check => check.passed), checks };
|
||||
}
|
||||
|
||||
export async function captureNativeDefault(input: {
|
||||
api: { get<T>(path: string): Promise<T> }; agentId: string; companyId: string;
|
||||
}): Promise<NativeDefaultReceipt> {
|
||||
const [bundle, company, agent] = await Promise.all([
|
||||
input.api.get<{ entryFile: unknown; files: Array<{ path: string }> }>(`/api/agents/${input.agentId}/instructions-bundle`),
|
||||
input.api.get<{ budgetMonthlyCents: unknown }>(`/api/companies/${input.companyId}`),
|
||||
input.api.get<{ budgetMonthlyCents: unknown }>(`/api/agents/${input.agentId}`),
|
||||
]);
|
||||
const files = await Promise.all(bundle.files.map(async file => {
|
||||
const detail = await input.api.get<{ content: string }>(`/api/agents/${input.agentId}/instructions-bundle/file?path=${encodeURIComponent(file.path)}`);
|
||||
return { path: file.path, sha256: createHash("sha256").update(detail.content).digest("hex"), bytes: Buffer.byteLength(detail.content) };
|
||||
}));
|
||||
return { schema: "paperclip.native-completion-default.v1", agentId: input.agentId, companyId: input.companyId,
|
||||
entryFile: bundle.entryFile, files, budgets: { company: company.budgetMonthlyCents, agent: agent.budgetMonthlyCents } };
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { gradeNativeCompletion, type NativeCompletionObservation } from "./native-completion-scoring.js";
|
||||
type Row = Record<string, unknown>;
|
||||
function sample(blocked = true, compatibility = false): NativeCompletionObservation {
|
||||
const body = blocked ? "Deployment remains blocked until access is granted. Release Owner must Grant deployment access. BLOCKED_probe" : "Saved the requested document.";
|
||||
const result = { reportedWorkDisposition: blocked ? "blocked" : "done", ...(blocked ? { blocker: { owner: { name: "Release Owner" }, unblockAction: "Grant deployment access", scope: "task_wide" } } : {}) };
|
||||
const event = (seq: number, eventType: string, payload: Row, sourceKind = "runner"): Row => ({ seq, eventType, protocolSchemaVersion: 1,
|
||||
sourceEventId: `event${seq}`, sourceInstanceId: sourceKind, sourceSeq: seq,
|
||||
payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", eventType,
|
||||
sourceKind, sourceEventId: `event${seq}`, sourceInstanceId: sourceKind, sourceSeq: seq, payload } } });
|
||||
const tool = blocked ? "paperclip_block" : "paperclip_finish";
|
||||
return { caseId: blocked ? "native-blocked-report" : "assigned-skill-explicit-invocation", companyId: "company", agentId: "agent", marker: "BLOCKED_probe",
|
||||
issue: { id: "issue", companyId: "company", assigneeAgentId: "agent", status: result.reportedWorkDisposition },
|
||||
runs: [{ id: "run", nativeIssueId: "issue", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "native", resultJson: { nativeResult: result } }],
|
||||
comments: [{ createdByRunId: "run", authorAgentId: "agent", body }],
|
||||
initial: { issueIds: [], agentIds: ["agent"] }, state: { issueIds: ["issue"], agentIds: ["agent"], documentCount: blocked ? 0 : 1, interactionCount: 0 }, workspaceChanged: false,
|
||||
events: [compatibility ? event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "tool" } }) : event(1, "tool.execution.started", { name: tool, executionId: "tool" }), event(2, "run.result.proposed", result),
|
||||
compatibility ? event(3, "item.completed", { kind: "tool_result", item: { type: "tool_result", status: "completed", id: "tool" } }) : event(3, "tool.execution.completed", { name: tool, status: "completed", executionId: "tool" }),
|
||||
event(4, "item.completed", { kind: "agentMessage", channel: "final", item: { type: "agentMessage", phase: "final_answer", text: body, channel: "final" } }),
|
||||
event(5, "run.result.accepted", { result }, "control_plane"), event(6, "run.terminal", { runTerminalState: "succeeded", turnTerminalState: "completed" }, "control_plane")],
|
||||
};
|
||||
}
|
||||
function payload(input: NativeCompletionObservation, index: number): Row { return ((input.events[index]!.payload as Row).prpEvent as Row).payload as Row; }
|
||||
describe("native completion independent oracle", () => {
|
||||
it.each([[true, false], [true, true], [false, false], [false, true]])("accepts disposition %s compatibility %s with final before late control-plane acceptance", (blocked, compatibility) => expect(gradeNativeCompletion(sample(blocked, compatibility)).passed).toBe(true));
|
||||
it("accepts authoritative acceptance before the provider final", () => {
|
||||
const value = sample();
|
||||
const events = [...value.events];
|
||||
value.events = [events[0]!, events[1]!, events[2]!, events[4]!, events[3]!, events[5]!].map((event, index) => ({ ...event, seq: index + 1 }));
|
||||
expect(gradeNativeCompletion(value).passed).toBe(true);
|
||||
});
|
||||
it("accepts complete public streams containing non-PRP lifecycle rows", () => {
|
||||
const value = sample();
|
||||
value.events = [{ seq: 1, eventType: "lifecycle" }, ...value.events.map(event => ({ ...event, seq: Number(event.seq) + 1 }))];
|
||||
expect(gradeNativeCompletion(value).passed).toBe(true);
|
||||
});
|
||||
it.each(["canonical", "compatibility"])("rejects an unmatched %s terminal result", kind => {
|
||||
const value = sample(true, kind === "compatibility");
|
||||
if (kind === "canonical") payload(value, 2).executionId = "unmatched";
|
||||
else (payload(value, 2).item as Row).id = "unmatched";
|
||||
expect(gradeNativeCompletion(value).passed).toBe(false);
|
||||
});
|
||||
it.each(["extra-run", "retry", "continuation", "wrong-account", "wrong-agent", "wrong-disposition", "punctuated-action", "wrong-scope", "missing-final", "summary-fallback", "pre-tool-final", "post-admission-call", "missing-acceptance", "runner-acceptance", "failed-terminal", "event-gap", "binding-mismatch", "extra-task", "extra-agent", "extra-document", "interaction", "changed-workspace", "process-call", "marker-only", "contradiction", "wrong-reply-run"])("rejects %s", name => {
|
||||
const value = sample(); const run = value.runs[0]!;
|
||||
if (name === "extra-run") value.runs = [run, { ...run, id: "extra" }];
|
||||
if (name === "retry") run.retryOfRunId = "prior";
|
||||
if (name === "continuation") run.continuationAttempt = 1;
|
||||
if (name === "wrong-account") run.companyId = "other";
|
||||
if (name === "wrong-agent") run.agentId = "other";
|
||||
if (name === "wrong-disposition") value.issue.status = "done";
|
||||
if (name === "punctuated-action") ((run.resultJson as Row).nativeResult as {blocker: {unblockAction: string}}).blocker.unblockAction = "Grant deployment access.";
|
||||
if (name === "wrong-scope") ((run.resultJson as Row).nativeResult as {blocker: {scope: string}}).blocker.scope = "step";
|
||||
if (name === "missing-final" || name === "summary-fallback") payload(value, 3).kind = "summary", (payload(value, 3).item as Row).type = "summary";
|
||||
if (name === "pre-tool-final") (payload(value, 2).name = "other");
|
||||
if (name === "post-admission-call") value.events[3]!.eventType = "tool.execution.started";
|
||||
if (name === "missing-acceptance") value.events[4]!.eventType = "other";
|
||||
if (name === "runner-acceptance") ((value.events[4]!.payload as Row).prpEvent as Row).sourceKind = "runner";
|
||||
if (name === "failed-terminal") payload(value, 5).runTerminalState = "failed";
|
||||
if (name === "event-gap") value.events[1]!.seq = 20;
|
||||
if (name === "binding-mismatch") ((value.events[1]!.payload as Row).prpEvent as Row).runId = "other";
|
||||
if (name === "extra-task") value.state.issueIds.push("extra");
|
||||
if (name === "extra-agent") value.state.agentIds.push("extra");
|
||||
if (name === "extra-document") value.state.documentCount = 1;
|
||||
if (name === "interaction") value.state.interactionCount = 1;
|
||||
if (name === "changed-workspace") value.workspaceChanged = true;
|
||||
if (name === "process-call") payload(value, 0).transport = "process";
|
||||
if (name === "marker-only" || name === "contradiction") {
|
||||
const text = name === "marker-only" ? "BLOCKED_probe" : "Deployment is not blocked. Release Owner completed Grant deployment access. BLOCKED_probe";
|
||||
(payload(value, 3).item as Row).text = text; value.comments[0]!.body = text;
|
||||
}
|
||||
if (name === "wrong-reply-run") value.comments[0]!.createdByRunId = "other";
|
||||
expect(gradeNativeCompletion(value).passed).toBe(false);
|
||||
});
|
||||
it.each([undefined, "other-issue"])("rejects missing or wrong native issue attribution %s", nativeIssueId => {
|
||||
const value = sample(); value.runs[0]!.nativeIssueId = nativeIssueId;
|
||||
expect(gradeNativeCompletion(value).passed).toBe(false);
|
||||
});
|
||||
it.each(["missing-action-envelope", "wrong-action-source", "missing-source-identity", "extra-wrong-source-result"])("rejects %s", name => {
|
||||
const value = sample();
|
||||
if (name === "missing-action-envelope") value.events[0]!.payload = {};
|
||||
if (name === "wrong-action-source") ((value.events[0]!.payload as Row).prpEvent as Row).sourceKind = "control_plane";
|
||||
if (name === "missing-source-identity") {
|
||||
delete value.events[0]!.sourceEventId;
|
||||
delete ((value.events[0]!.payload as Row).prpEvent as Row).sourceEventId;
|
||||
}
|
||||
if (name === "extra-wrong-source-result") {
|
||||
const extra = structuredClone(value.events[4]!);
|
||||
((extra.payload as Row).prpEvent as Row).sourceKind = "runner";
|
||||
value.events = [...value.events, { ...extra, seq: 7 }];
|
||||
}
|
||||
expect(gradeNativeCompletion(value).passed).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
|
||||
const forbiddenKinds = new Set(["commandExecution", "process", "fileChange", "file_change", "apply_patch"]);
|
||||
const toolKinds = new Set(["dynamicToolCall", "mcpToolCall", "commandExecution", "tool_call", "tool_use"]);
|
||||
const terminalName = (value: unknown, expected: string) => typeof value === "string" && (value === expected || value.endsWith(`.${expected}`));
|
||||
|
||||
export interface NativeCompletionObservation {
|
||||
caseId: "assigned-skill-explicit-invocation" | "native-blocked-report";
|
||||
companyId: string; agentId: string; issue: Row; runs: readonly Row[];
|
||||
comments: readonly Row[]; events: readonly Row[];
|
||||
state: { issueIds: string[]; agentIds: string[]; documentCount: number; interactionCount: number };
|
||||
initial: { issueIds: string[]; agentIds: string[] };
|
||||
workspaceChanged: boolean;
|
||||
marker: string;
|
||||
}
|
||||
|
||||
/** Observable tool/result/final sequence, not a claim that the provider consumed feedback. */
|
||||
export function gradeNativeCompletion(input: NativeCompletionObservation) {
|
||||
const checks: Array<{ id: string; passed: boolean; detail: string }> = [];
|
||||
const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail });
|
||||
const blocked = input.caseId === "native-blocked-report";
|
||||
const run = input.runs[0] ?? {};
|
||||
const expectedDisposition = blocked ? "blocked" : "done";
|
||||
const expectedTool = blocked ? "paperclip_block" : "paperclip_finish";
|
||||
const nativeResult = row(row(run.resultJson).nativeResult);
|
||||
check("one-native-attempt", input.runs.length === 1 && run.status === "succeeded" && run.runtimeMode === "native"
|
||||
&& typeof run.id === "string" && run.id.length > 0 && typeof input.issue.id === "string" && input.issue.id.length > 0
|
||||
&& run.nativeIssueId === input.issue.id
|
||||
&& run.companyId === input.companyId && run.agentId === input.agentId
|
||||
&& !run.retryOfRunId && !run.continuationAttempt && !input.issue.scheduledRetry,
|
||||
"Exactly one succeeded native run, with no continuation/retry or uncounted extra work.");
|
||||
check("durable-disposition", input.issue.companyId === input.companyId && input.issue.assigneeAgentId === input.agentId
|
||||
&& input.issue.status === expectedDisposition && nativeResult.reportedWorkDisposition === expectedDisposition,
|
||||
"Public durable issue status and the native semantic result agree with the requested disposition.");
|
||||
if (blocked) {
|
||||
const blocker = row(nativeResult.blocker);
|
||||
check("exact-blocker", row(blocker.owner).name === "Release Owner" && blocker.unblockAction === "Grant deployment access"
|
||||
&& blocker.scope === "task_wide", "The structured owner, exact action and whole-task scope are independently checked.");
|
||||
}
|
||||
|
||||
const events = input.events.map(event => ({ event, envelope: row(row(event.payload).prpEvent) }));
|
||||
const bound = events.length > 0 && events.every(({ event, envelope }, index) => {
|
||||
const action = /^(?:tool\.execution\.|item\.)/.test(String(event.eventType));
|
||||
const required = action || ["run.result.proposed", "run.result.accepted", "run.terminal"].includes(String(event.eventType));
|
||||
return event.seq === index + 1
|
||||
&& (Object.keys(envelope).length === 0 ? !required : isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion as number | undefined)
|
||||
&& (!action || envelope.sourceKind === "runner")
|
||||
&& typeof envelope.sourceEventId === "string" && envelope.sourceEventId.length > 0
|
||||
&& typeof envelope.sourceInstanceId === "string" && envelope.sourceInstanceId.length > 0
|
||||
&& Number.isSafeInteger(envelope.sourceSeq) && Number(envelope.sourceSeq) > 0
|
||||
&& envelope.runId === run.id && envelope.eventType === event.eventType
|
||||
&& envelope.sourceEventId === event.sourceEventId && envelope.sourceInstanceId === event.sourceInstanceId
|
||||
&& envelope.sourceSeq === event.sourceSeq);
|
||||
});
|
||||
check("complete-bound-events", bound, "A complete contiguous public PRP stream is bound to this run and persisted source identity.");
|
||||
const selected = (type: string) => events.filter(({ event }) => event.eventType === type);
|
||||
const proposed = selected("run.result.proposed");
|
||||
const accepted = selected("run.result.accepted");
|
||||
const terminal = selected("run.terminal");
|
||||
const acceptedResult = row(row(accepted[0]?.envelope.payload).result);
|
||||
const terminalResult = row(terminal[0]?.envelope.payload);
|
||||
check("authoritative-native-result", proposed.length === 1 && accepted.length === 1 && terminal.length === 1
|
||||
&& proposed[0]?.envelope.sourceKind === "runner" && accepted[0]?.envelope.sourceKind === "control_plane" && terminal[0]?.envelope.sourceKind === "control_plane"
|
||||
&& row(proposed[0]?.envelope.payload).reportedWorkDisposition === expectedDisposition
|
||||
&& acceptedResult.reportedWorkDisposition === expectedDisposition
|
||||
&& terminalResult.runTerminalState === "succeeded" && terminalResult.turnTerminalState === "completed",
|
||||
"One runner proposal is independently accepted and terminated by the control plane.");
|
||||
const proposalSeq = Number(proposed[0]?.event.seq ?? -1);
|
||||
const finishes = events.filter(({ event, envelope }) => {
|
||||
const payload = row(envelope.payload), item = row(payload.item);
|
||||
const completion = envelope.sourceKind === "runner" && Number(event.seq) > proposalSeq && (
|
||||
event.eventType === "tool.execution.completed" && terminalName(payload.name, expectedTool) && payload.status === "completed"
|
||||
|| event.eventType === "item.completed" && (payload.kind === "tool_result" || item.type === "tool_result") && item.status === "completed"
|
||||
);
|
||||
if (!completion) return false;
|
||||
return events.some(({ event: start, envelope: startEnvelope }) => {
|
||||
const started = row(startEnvelope.payload), startedItem = row(started.item);
|
||||
return startEnvelope.sourceKind === "runner" && Number(start.seq) < proposalSeq && (
|
||||
event.eventType === "tool.execution.completed" && start.eventType === "tool.execution.started"
|
||||
&& typeof payload.executionId === "string" && payload.executionId.length > 0 && started.executionId === payload.executionId && terminalName(started.name, expectedTool)
|
||||
|| event.eventType === "item.completed" && start.eventType === "item.started"
|
||||
&& (started.kind === "tool_call" || startedItem.type === "tool_call")
|
||||
&& typeof item.id === "string" && item.id.length > 0 && item.id === startedItem.id
|
||||
);
|
||||
});
|
||||
});
|
||||
const finals = events.filter(({ event, envelope }) => {
|
||||
const payload = row(envelope.payload), item = row(payload.item);
|
||||
return envelope.sourceKind === "runner" && event.eventType === "item.completed"
|
||||
&& (payload.kind === "agentMessage" || item.type === "agentMessage")
|
||||
&& (payload.channel === "final" || item.channel === "final") && item.phase === "final_answer"
|
||||
&& typeof item.text === "string" && item.text.trim().length > 0;
|
||||
});
|
||||
const finishSeq = Number(finishes.at(-1)?.event.seq ?? -1);
|
||||
const finalSeq = Number(finals[0]?.event.seq ?? -1);
|
||||
const starts = events.filter(({ event, envelope }) => envelope.sourceKind === "runner" && (
|
||||
event.eventType === "tool.execution.started" || event.eventType === "item.started"
|
||||
&& toolKinds.has(String(row(envelope.payload).kind ?? row(row(envelope.payload).item).type))));
|
||||
check("final-after-tool-result", proposed.length === 1 && finishes.length > 0 && finals.length === 1
|
||||
&& proposalSeq < finishSeq && finishSeq < finalSeq
|
||||
&& proposalSeq < Number(accepted[0]?.event.seq ?? -1) && Number(accepted[0]?.event.seq ?? -1) < Number(terminal[0]?.event.seq ?? -1)
|
||||
&& !starts.some(({ event }) => Number(event.seq) > proposalSeq),
|
||||
"The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently.");
|
||||
const finalText = String(row(row(finals[0]?.envelope.payload).item).text ?? "").trim();
|
||||
const replies = input.comments.filter(comment => comment.createdByRunId === run.id && comment.authorAgentId === input.agentId
|
||||
&& typeof comment.body === "string" && comment.body.trim() === finalText);
|
||||
check("persisted-provider-final", finalText.length > 0 && replies.length === 1,
|
||||
"A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient.");
|
||||
if (blocked) check("visible-blocker-content", finalText.split(input.marker).length === 2
|
||||
&& finalText.includes("Release Owner") && finalText.includes("Grant deployment access")
|
||||
&& /\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\b/i.test(finalText)
|
||||
&& !/\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\b/i.test(finalText),
|
||||
"The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker.");
|
||||
const same = (a: string[], b: string[]) => a.length === b.length && a.every(id => b.includes(id)) && new Set(a).size === a.length;
|
||||
check("bounded-durable-work", same(input.state.issueIds, [...input.initial.issueIds, String(input.issue.id)])
|
||||
&& same(input.state.agentIds, input.initial.agentIds) && input.state.documentCount === (blocked ? 0 : 1)
|
||||
&& input.state.interactionCount === 0,
|
||||
"No additional tasks, agents or interactions; completion permits its one required document and blocker permits none.");
|
||||
if (blocked) {
|
||||
const forbidden = events.some(({ event, envelope }) => {
|
||||
const payload = row(envelope.payload), item = row(payload.item);
|
||||
return envelope.sourceKind === "runner" && (forbiddenKinds.has(String(payload.kind ?? item.type))
|
||||
|| String(event.eventType).startsWith("tool.execution.") && payload.transport === "process"
|
||||
|| /(?:write_document|create_task|hire_agent|register_deliverable|apply_patch|exec_command)/.test(String(payload.name ?? item.name ?? "")));
|
||||
});
|
||||
check("no-deployment-or-file-work", !input.workspaceChanged && !forbidden,
|
||||
"The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed.");
|
||||
}
|
||||
return { schema: "paperclip.native-completion-observation.v1", passed: checks.every(value => value.passed), checks,
|
||||
limitations: ["Exact provider feedback identity/consumption is not measured by the public sequence."] };
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// Exact native-only comparison boundary: executable fixtures stay byte-identical.
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
export const NATIVE_COMPLETION_SOURCE_CONTRACT = {
|
||||
"schema": "paperclip.native-completion-source-contract.v1",
|
||||
"baseSha": "59c07ede72dc08b8aba149a01cc11e0b7a204621",
|
||||
"archiveSha": "9138f570c341c251a5727c32d6615ce238bc8e03",
|
||||
"variants": {
|
||||
"candidate": {
|
||||
"packages/paperclip-runner/src/contracts/completion-result.ts": "bd4bb79d2be5c4ee3765df67dfb4da98ef95bc532cac872be40cb41a6bff6cda",
|
||||
"packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts": "aff3b3d2d626fbaf504a7e46adb27cb8127d0afbc62a8ed58111938f1293de7d",
|
||||
"packages/paperclip-runner/src/drivers/runner-tool-bridge.ts": "4cca0d7837e0f5a1f709abb0ba3982227288a01abd16a08107337416fde4a9e2",
|
||||
"packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "ec1cd94c17be22d0556d4586fa6407b28e2309ed7c2437d95a32beeb0755cc4f",
|
||||
"server/src/services/native-runtime/native-session-resume.ts": "5b27465e1a328412770fc27c4bc637e88b62022adc6a67709bb74187eadc465b",
|
||||
"packages/paperclip-runner/src/contracts/completion-result.test.ts": "42bd18105505da7fd7963311bf862002df7158f8c0cf2a81b927dff4c47ac500",
|
||||
"packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts": "2b15425514d8e3316baec48b8ce118394acf67d97b76cbab2a99916fc4e8e1d2",
|
||||
"packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts": "097d4973ac4ca200e82af31aece0f9d5531e1756ffe739d4422bd8f65d479232",
|
||||
"packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts": "533dd12ea00c61e012f5d596e03080fa1aaf63375ad7602c0de4e0a6c8370400",
|
||||
"packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts": "8546181e5b3a63a194ee9ea0fc99c2bd97562f0c105fffde298fa6d74b22e83b",
|
||||
"server/src/services/native-runtime/native-session-resume.test.ts": "57b498d76bc89801edc81d700cd87660a16074ae8bb8a43aa566205cc377d7b4"
|
||||
},
|
||||
"historical": {
|
||||
"packages/paperclip-runner/src/contracts/completion-result.ts": "00a684f9aea87f68eabf0974681c52461bc308631a1252a89c4ed087d7158b28",
|
||||
"packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts": "87929dd81b6fdc44fb46e5b14ac775ca7244354587028b3bd356a617dcf31907",
|
||||
"packages/paperclip-runner/src/drivers/runner-tool-bridge.ts": "966696d39eb9554e5f40bd1d89d31efc426c3c01d2ce883b3b6e968db8480088",
|
||||
"packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "7080ae2896fe7a81fdf1a1c4f7fcd1773611cbd3336896b5f9e861c5b0aa0ed4",
|
||||
"server/src/services/native-runtime/native-session-resume.ts": "72b4f2e19c01a83ccd52e21102d2c5422265eec84b589ea75a7cffe40d1d0b8d",
|
||||
"packages/paperclip-runner/src/contracts/completion-result.test.ts": "7c615dc36ad07b7659c3605f94fec89ae592727ce078a62fa45eb1bb549f26d5",
|
||||
"packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts": "0b7e8b7a552d6a3d74d761e0e68b720ed02d621ded41a1c2076ca3fda0f96d0c",
|
||||
"packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts": "1ce59108e0cfef78fa71f0da9b46af70fd7bf3887d4a662556afc072d2351653",
|
||||
"packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts": "bb1b0447cca30f3a2984e9e832949108c43cefa35d765440d07ee8098be4ef0c",
|
||||
"packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts": "624bbac680e1dfc87fff9fab71501e8208e0bdbc6aefcb45a008b8e6c9030a27",
|
||||
"server/src/services/native-runtime/native-session-resume.test.ts": "ddd60db81cdb8d17a2b31ea934779687708b927f877f3fb6d0c249ceb805be62"
|
||||
}
|
||||
},
|
||||
"fixedContext": {
|
||||
"server/src/onboarding-assets/default/AGENTS.md": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f",
|
||||
"packages/adapter-utils/src/server-utils.ts": "916e879f70a90d31c89bc1486d5f84807c4ecafeab62961f67d84d1ebec7a173",
|
||||
"skills/paperclip/SKILL.md": "19b4e17bc6588b087efce63673c3f2839ac09dba9c072f56b8e90924a66856de",
|
||||
"server/src/services/onboarding-first-task-assets.ts": "c7fd1db7ed97850bd67e48a4d8c1470c8ff801f895f4f665b84690978f19b47e",
|
||||
"server/src/services/default-agent-instructions.ts": "8bd3b59f900c132db5ed89b96524f35ba7c8dafb9d58db1dac888430f08c4d63",
|
||||
"packages/paperclip-runner/src/contracts/runtime-context.ts": "a17e30a69f3cb38fba00c37e1a06753d723651a004e6050c9e77ae202a61264b",
|
||||
"packages/paperclip-runner/src/backends/runtime-context.ts": "e7c46e81cc9c93f9a4f805b5091ef08c59ea70c62f208cfda70d6edeed363898",
|
||||
"server/src/services/native-runtime/native-execution-input.ts": "e346bf3f163dfc5374c3c90fb515c1db6847bbab69fb59a49d73227478cef70d",
|
||||
"server/src/services/native-runtime/completion-contracts.ts": "b1f4f7a9b04fea902c23313adb296e67f202ee5878424f85a7d0cebcb775ab69",
|
||||
"server/src/services/native-runtime/runtime-context.ts": "13f71f749ad6a3a0612e3e268fdc2bc58ee26f70399dd120b1341abf00d1847b",
|
||||
"packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts": "1a3b80e9b155a1e7e1f72313ad41bb0a397d6d42cb6aedf23704c1cf9121c0b8"
|
||||
}
|
||||
};
|
||||
export const NATIVE_COMPLETION_FIXTURE_FILES = [
|
||||
"tests/runner-e2e/native-completion-cases.ts",
|
||||
"tests/runner-e2e/native-completion-scoring.ts",
|
||||
"tests/runner-e2e/native-completion-scoring.test.ts",
|
||||
"tests/runner-e2e/native-completion-defaults.ts",
|
||||
"tests/runner-e2e/native-completion-defaults.test.ts",
|
||||
"tests/runner-e2e/native-blocker-visible.ts",
|
||||
"tests/runner-e2e/native-blocker-visible.spec.ts",
|
||||
"tests/runner-e2e/native-completion-admission.ts",
|
||||
"tests/runner-e2e/native-completion-admission.test.ts",
|
||||
"tests/runner-e2e/native-completion-checks.mjs",
|
||||
"tests/runner-e2e/native-completion-checks.test.mjs",
|
||||
"tests/runner-e2e/native-completion-source-contract.mjs",
|
||||
"tests/runner-e2e/native-completion-source-contract.test.mjs",
|
||||
"tests/runner-e2e/automatic-retry.ts",
|
||||
"tests/runner-e2e/automatic-retry.test.ts",
|
||||
"tests/runner-e2e/launch.ts",
|
||||
"tests/runner-e2e/types.ts",
|
||||
"tests/runner-e2e/catalog.ts",
|
||||
"tests/runner-e2e/catalog.test.ts",
|
||||
"tests/runner-e2e/runner.spec.ts",
|
||||
"tests/runner-e2e/playwright-support.config.ts",
|
||||
"tests/runner-e2e/context-integrity-cases.ts",
|
||||
"tests/runner-e2e/context-integrity.test.ts",
|
||||
"tests/runner-e2e/context-integrity-evidence.test.ts",
|
||||
"tests/runner-e2e/context-integrity-flow-logs.test.ts",
|
||||
"tests/runner-e2e/live-fixtures.ts",
|
||||
"tests/runner-e2e/context-integrity-flow.ts",
|
||||
"tests/runner-e2e/context-integrity-scoring.ts",
|
||||
"tests/runner-e2e/selectors.ts",
|
||||
"tests/runner-e2e/harness-env.ts",
|
||||
"tests/runner-e2e/tsconfig.json",
|
||||
"tests/runner-e2e/vitest.config.ts"
|
||||
];
|
||||
export const NATIVE_COMPLETION_MANIFEST_FILES = [
|
||||
"packages/paperclip-runner/scripts/generate-capability-contract.mjs",
|
||||
"packages/paperclip-runner/scripts/check-capability-inventory.mjs",
|
||||
"packages/paperclip-runner/scripts/lib/capability-inventory.mjs",
|
||||
"packages/paperclip-runner/spec/capability/source-contract.json",
|
||||
"packages/paperclip-runner/spec/capability/capabilities.yaml",
|
||||
"packages/paperclip-runner/spec/capability/eval-traceability.yaml",
|
||||
"packages/paperclip-runner/spec/capability/mcp-tool-map.yaml",
|
||||
"packages/paperclip-runner/spec/capability/inventory.schema.json",
|
||||
"packages/paperclip-runner/src/generated/capability-contract.ts",
|
||||
"packages/paperclip-runner/docs/capability-contract.md",
|
||||
...["capabilities.yaml", "mcp-tool-map.yaml", "eval-traceability.yaml", "capability-contract.md", "downstream-handoff.md"]
|
||||
.map(file => `packages/paperclip-runner/generated/capability/${file}`),
|
||||
];
|
||||
export const NATIVE_COMPLETION_SOURCE_FILES = [...new Set([
|
||||
...Object.keys(NATIVE_COMPLETION_SOURCE_CONTRACT.variants.candidate),
|
||||
...Object.keys(NATIVE_COMPLETION_SOURCE_CONTRACT.fixedContext),
|
||||
...NATIVE_COMPLETION_FIXTURE_FILES, ...NATIVE_COMPLETION_MANIFEST_FILES,
|
||||
"scripts/ensure-plugin-build-deps.mjs",
|
||||
"packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs",
|
||||
"packages/paperclip-runner/src/live/runnerd-codex-transport.ts",
|
||||
"packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts",
|
||||
"packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts",
|
||||
"packages/adapters/codex-local/src/index.ts", "packages/adapters/claude-local/src/index.ts",
|
||||
"packages/paperclip-runner/package.json", "tsconfig.base.json",
|
||||
])].sort();
|
||||
|
||||
export const nativeSourceSha256 = bytes => createHash("sha256").update(bytes).digest("hex");
|
||||
function fingerprint(paths, read) {
|
||||
const hash = createHash("sha256");
|
||||
for (const file of [...paths].sort()) {
|
||||
const bytes = read(file);
|
||||
hash.update(JSON.stringify([file, bytes.length])).update(bytes);
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
/** Infer a variant only from all eleven matching production/assertion files. */
|
||||
export function nativeCompletionSourceFingerprint(read = file => readFileSync(join(
|
||||
resolve(import.meta.dirname, "../.."), file)), contract = NATIVE_COMPLETION_SOURCE_CONTRACT) {
|
||||
const observations = new Map(), sourceErrors = [];
|
||||
for (const file of NATIVE_COMPLETION_SOURCE_FILES) {
|
||||
try { observations.set(file, read(file)); } catch { sourceErrors.push(file); }
|
||||
}
|
||||
const matches = Object.entries(contract.variants).filter(([, expected]) => Object.entries(expected)
|
||||
.every(([file, digest]) => observations.has(file) && nativeSourceSha256(observations.get(file)) === digest));
|
||||
const variant = matches.length === 1 ? matches[0][0] : null;
|
||||
if (!variant) sourceErrors.push("native-completion: mixed or unknown production/assertion variant");
|
||||
for (const [file, digest] of Object.entries(contract.fixedContext))
|
||||
if (!observations.has(file) || nativeSourceSha256(observations.get(file)) !== digest)
|
||||
sourceErrors.push(`native-completion: changed master context ${file}`);
|
||||
const fromObservation = file => observations.get(file);
|
||||
const complete = NATIVE_COMPLETION_SOURCE_FILES.every(file => observations.has(file));
|
||||
return { variant, sourceErrors,
|
||||
fingerprint: complete ? fingerprint(NATIVE_COMPLETION_SOURCE_FILES, fromObservation) : null,
|
||||
fixtureFingerprint: complete ? fingerprint(NATIVE_COMPLETION_FIXTURE_FILES, fromObservation) : null,
|
||||
manifestFingerprint: complete ? fingerprint(NATIVE_COMPLETION_MANIFEST_FILES, fromObservation) : null,
|
||||
baseSha: contract.baseSha, archiveSha: contract.archiveSha,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
NATIVE_COMPLETION_SOURCE_CONTRACT as original, NATIVE_COMPLETION_SOURCE_FILES,
|
||||
nativeCompletionSourceFingerprint, nativeSourceSha256,
|
||||
} from "./native-completion-source-contract.mjs";
|
||||
|
||||
function fixture(variant = "candidate") {
|
||||
const bytes = new Map(NATIVE_COMPLETION_SOURCE_FILES.map(file => [file, Buffer.from(`common:${file}`)]));
|
||||
const variantFiles = Object.keys(original.variants.candidate), fixed = Object.keys(original.fixedContext);
|
||||
const contract = { ...original, variants: Object.fromEntries(["candidate", "historical"].map(name => [name,
|
||||
Object.fromEntries(variantFiles.map(file => [file, nativeSourceSha256(Buffer.from(`${name}:${file}`))]))])),
|
||||
fixedContext: Object.fromEntries(fixed.map(file => [file, nativeSourceSha256(bytes.get(file))])) };
|
||||
for (const file of variantFiles) bytes.set(file, Buffer.from(`${variant}:${file}`));
|
||||
const read = file => { if (!bytes.has(file)) throw new Error("Missing source"); return bytes.get(file); };
|
||||
return { bytes, contract, read };
|
||||
}
|
||||
for (const variant of ["candidate", "historical"]) test(`native source contract accepts the complete ${variant} source map`, () => {
|
||||
const f = fixture(variant), result = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
assert.equal(result.variant, variant); assert.deepEqual(result.sourceErrors, []);
|
||||
for (const key of ["fingerprint", "fixtureFingerprint", "manifestFingerprint"]) assert.match(result[key], /^[a-f0-9]{64}$/);
|
||||
});
|
||||
test("native source contract keeps fixtures identical across candidate and historical variants", () => {
|
||||
const a = fixture("candidate"), b = fixture("historical");
|
||||
const candidate = nativeCompletionSourceFingerprint(a.read, a.contract), historical = nativeCompletionSourceFingerprint(b.read, b.contract);
|
||||
assert.equal(candidate.fixtureFingerprint, historical.fixtureFingerprint);
|
||||
assert.equal(candidate.manifestFingerprint, historical.manifestFingerprint);
|
||||
assert.notEqual(candidate.fingerprint, historical.fingerprint);
|
||||
});
|
||||
for (const file of Object.keys(original.variants.candidate)) test(`native source contract rejects a mixed production/assertion source: ${file}`, () => {
|
||||
const f = fixture(); f.bytes.set(file, Buffer.from(`historical:${file}`));
|
||||
const result = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
assert.equal(result.variant, null); assert.ok(result.sourceErrors.some(error => error.includes("mixed or unknown")));
|
||||
});
|
||||
for (const file of Object.keys(original.fixedContext)) test(`native source contract rejects changed master context: ${file}`, () => {
|
||||
const f = fixture(); f.bytes.set(file, Buffer.from("Reduced or replaced context"));
|
||||
const result = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
assert.ok(result.sourceErrors.includes(`native-completion: changed master context ${file}`));
|
||||
});
|
||||
test("native source contract rejects missing files and changes its digest when a fixture changes", () => {
|
||||
const f = fixture(), before = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
f.bytes.set("tests/runner-e2e/native-completion-cases.ts", Buffer.from("Changed behavior"));
|
||||
const changed = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
assert.notEqual(before.fixtureFingerprint, changed.fixtureFingerprint);
|
||||
f.bytes.delete("tests/runner-e2e/native-completion-cases.ts");
|
||||
const missing = nativeCompletionSourceFingerprint(f.read, f.contract);
|
||||
assert.equal(missing.fingerprint, null); assert.ok(missing.sourceErrors.includes("tests/runner-e2e/native-completion-cases.ts"));
|
||||
});
|
||||
test("native source contract rejects an ambiguous identical variant map", () => {
|
||||
const f = fixture(); f.contract.variants.historical = f.contract.variants.candidate;
|
||||
assert.equal(nativeCompletionSourceFingerprint(f.read, f.contract).variant, null);
|
||||
});
|
||||
test("native source contract binds exactly five production and six variant assertion files", () => {
|
||||
const files = Object.keys(original.variants.candidate);
|
||||
assert.equal(files.length, 11); assert.equal(files.filter(file => file.endsWith(".test.ts")).length, 6);
|
||||
assert.deepEqual(files, Object.keys(original.variants.historical));
|
||||
assert.ok(files.every(file => original.variants.candidate[file] !== original.variants.historical[file]));
|
||||
assert.equal(original.baseSha, "59c07ede72dc08b8aba149a01cc11e0b7a204621");
|
||||
assert.equal(original.archiveSha, "9138f570c341c251a5727c32d6615ce238bc8e03");
|
||||
assert.ok(!NATIVE_COMPLETION_SOURCE_FILES.some(file => file.includes("stock-harness") || file.endsWith("issue-documents.md")));
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test";
|
||||
/** Browser-only harness regressions: no Paperclip instance or provider credentials. */
|
||||
export default defineConfig({
|
||||
testDir: ".",
|
||||
testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts"],
|
||||
testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts"],
|
||||
workers: 1,
|
||||
retries: 0,
|
||||
timeout: 10_000,
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js";
|
||||
import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js";
|
||||
import { gradeNativeCompletion } from "./native-completion-scoring.js";
|
||||
import { assertNativeBlockerReply } from "./native-blocker-visible.js";
|
||||
import { warmManagedFileEvidence } from "./warm-managed-files.js";
|
||||
import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js";
|
||||
import { runsCompletionUpdateProbe, completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js";
|
||||
@@ -552,6 +556,10 @@ for (const execution of executions) {
|
||||
let apiResponseSourceId: string | undefined;
|
||||
let titleCreation: { issue: unknown; submitted: unknown } | undefined;
|
||||
let lifecycleBlockerId: string | null = null;
|
||||
if (execution.suite.id === "native-completion") {
|
||||
assertNativeCompletionSelection([execution]);
|
||||
verifyNativeCompletionPreflight(process.env[NATIVE_COMPLETION_PREFLIGHT_ENV]);
|
||||
}
|
||||
const credentials = credentialValues();
|
||||
const secrets = normalizedSecrets(Object.values(credentials));
|
||||
const api = new RunnerApi(request);
|
||||
@@ -560,6 +568,9 @@ for (const execution of executions) {
|
||||
const networkDiagnostics: Array<Record<string, unknown>> = [];
|
||||
const pageLifecycleDiagnostics: Array<Record<string, unknown>> = [];
|
||||
const browserBootstrap = observeBrowserBootstrap(page);
|
||||
let nativeInitial: { issueIds: string[]; agentIds: string[]; workspaceDigest: string | null } | undefined;
|
||||
const nativeWorkspaceDigest = () => execution.task.id === "native-blocked-report"
|
||||
? nativeCompletionWorkspaceDigest(workspacePath!) : Promise.resolve(null);
|
||||
let fixtures: LiveFixtureValues | undefined;
|
||||
let reviewProvider: Awaited<ReturnType<typeof setupConnectionReview>> | undefined;
|
||||
let issue: IssueRecord | undefined;
|
||||
@@ -842,6 +853,18 @@ for (const execution of executions) {
|
||||
daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE,
|
||||
});
|
||||
|
||||
if (execution.suite.id === "native-completion") {
|
||||
const receipt = await captureNativeDefault({ api, agentId: fixtures.agent.id, companyId: fixtures.company.id });
|
||||
const grade = gradeNativeDefault(receipt);
|
||||
await writeSanitizedJson(snapshotsDir, "native-default-before-execution.json", { receipt, grade }, secrets);
|
||||
if (!grade.passed) throw new Error("Native production-default hire admission failed before execution");
|
||||
const [issues, agents, workspaceDigest] = await Promise.all([
|
||||
api.get<Array<{ id: string }>>(`/api/companies/${fixtures.company.id}/issues?limit=100`),
|
||||
api.get<Array<{ id: string }>>(`/api/companies/${fixtures.company.id}/agents`), nativeWorkspaceDigest(),
|
||||
]);
|
||||
nativeInitial = { issueIds: issues.map(value => value.id), agentIds: agents.map(value => value.id), workspaceDigest };
|
||||
}
|
||||
|
||||
if (execution.suite.id === "api-response-reading") {
|
||||
const source = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/issues`, {
|
||||
title: `Synthetic diagnostic evidence ${nonce}`,
|
||||
@@ -2574,6 +2597,8 @@ for (const execution of executions) {
|
||||
await expect(proofLink).toHaveAttribute("href", `/api/attachments/${downloadedResponseProof!.attachmentId}/content`);
|
||||
} else if (execution.suite.id === "task-titles") {
|
||||
await expect(visibleAgentReplies.filter({ hasText: marker }).last()).toBeVisible({ timeout: 30_000 });
|
||||
} else if (execution.suite.id === "native-completion" && execution.task.id === "native-blocked-report") {
|
||||
await assertNativeBlockerReply(visibleAgentReplies, marker);
|
||||
} else if (execution.task.flow === "warm_three_turn") {
|
||||
// Prove the persisted user-facing response is visible, independently
|
||||
// of the byte-for-byte workspace checks and lease continuity checks.
|
||||
@@ -2633,6 +2658,31 @@ for (const execution of executions) {
|
||||
);
|
||||
}
|
||||
}
|
||||
if (execution.suite.id === "native-completion") {
|
||||
if (!issue || !nativeInitial) throw new Error("Missing native qualification issue or pre-execution receipt");
|
||||
const [currentIssue, companyRuns, issues, agents, comments, documents, interactions, workspaceDigest] = await Promise.all([
|
||||
api.get<IssueRecord>(`/api/issues/${issue.id}`),
|
||||
api.get<RunRecord[]>(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`),
|
||||
api.get<Array<{ id: string }>>(`/api/companies/${fixtures.company.id}/issues?limit=100`),
|
||||
api.get<Array<{ id: string }>>(`/api/companies/${fixtures.company.id}/agents`),
|
||||
api.get<CommentRecord[]>(`/api/issues/${issue.id}/comments`),
|
||||
api.get<IssueDocumentRecord[]>(`/api/issues/${issue.id}/documents`),
|
||||
api.get<InteractionRecord[]>(`/api/issues/${issue.id}/interactions`), nativeWorkspaceDigest(),
|
||||
]);
|
||||
const detailedRuns = await Promise.all(companyRuns.map(candidate => api.get<RunRecord>(`/api/heartbeat-runs/${candidate.id}`)));
|
||||
selectedRuns = detailedRuns; issue = currentIssue;
|
||||
const events = detailedRuns.length === 1 ? await collectRunEvents<RunEventRecord>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${detailedRuns[0]!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)) : [];
|
||||
const observation = { caseId: execution.task.id as "assigned-skill-explicit-invocation" | "native-blocked-report",
|
||||
companyId: fixtures.company.id, agentId: fixtures.agent.id, issue: currentIssue as unknown as Record<string, unknown>,
|
||||
runs: detailedRuns as unknown as Record<string, unknown>[], comments: comments as unknown as Record<string, unknown>[], events: events as unknown as Record<string, unknown>[],
|
||||
initial: nativeInitial, state: { issueIds: issues.map(value => value.id), agentIds: agents.map(value => value.id), documentCount: documents.length, interactionCount: interactions.length },
|
||||
workspaceChanged: workspaceDigest !== nativeInitial.workspaceDigest, marker };
|
||||
const grade = gradeNativeCompletion(observation);
|
||||
const integrity = detailedRuns.flatMap(candidate => nativeRunEventIntegrityFailures(candidate, events));
|
||||
await writeSanitizedJson(snapshotsDir, "native-completion.json", { observation, grade, integrity }, secrets);
|
||||
matcherResults.push(...grade.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `nativeCompletion.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })));
|
||||
if (!grade.passed || integrity.length) throw new Error(`Native completion qualification failed: ${[...grade.checks.filter(check => !check.passed).map(check => check.id), ...integrity].join("; ")}`);
|
||||
}
|
||||
if (runsCompletionUpdateProbe(execution) && credentials.OPENAI_API_KEY) {
|
||||
const qualification = completionQualityStatus(completionQuality);
|
||||
if (qualification === "unqualified") {
|
||||
|
||||
@@ -138,6 +138,8 @@ export interface RunnerTaskFixture {
|
||||
expectedRunCount: number;
|
||||
/** Optional lower bound; expectedRunCount remains the maximum/cost estimate. */
|
||||
minimumExpectedRunCount?: number;
|
||||
/** Admit only the first attempt, including provider or infrastructure failures. */
|
||||
automaticRetryPolicy?: "single_attempt";
|
||||
attemptTimeoutMs: Readonly<Record<RunnerEnvironmentId, number>>;
|
||||
expectedTerminalState: {
|
||||
issue: "done" | "in_review" | "blocked" | "in_progress";
|
||||
|
||||
Reference in new issue
Block a user