Preserve document links in completion feedback and navigation

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-04 21:01:12 -05:00
1 parent be604fcbf8
commit 103063f81d
14 files changed
+263 -32

No files matched your search

+1 -1
View File
@@ -426,7 +426,7 @@ The independent, explicit-only `native-completion` suite qualifies native finish
The separate, explicit-only `native-instruction-consolidation` suite reuses those original tasks and strict graders to compare completion constraints on the production defaults at `2a8a99e4a5f69aa803b3f10b982f583e75a87042`. It declares six local cells: document completion and whole-task blocking on native Codex, ACPX Claude, and OpenCode. Each cell allows one attempt and applies a 1,000-cent company and agent budget hard stop. Its source gate rejects dirty, mixed, unknown, or unrelated source changes before credentials load. A provider-free fixture captures the complete Paperclip instruction/tool/message projection at the scripted runnerd RPC boundary on start, full-task resume and compact user-follow-up continuation for native input v4 and v5. That capture measures bytes; it does not measure vendor-owned prompts, tokens, billing, or model behavior. See the [comparison plan](../../doc/plans/2026-10-03-native-completion-consolidation.md) for exact scope and live qualification limits. Existing `native-completion` results do not qualify this new reduction.
The corrected source variant adds explicit blocker explanations and canonical document citations. Observation v3 independently requires the persisted provider final to explain missing release/deployment access and, for completion, link this task's one revisioned document on the same origin. A correct structured blocker, an unblock action alone, or an unbound/foreign document URL cannot pass. These stricter checks supplement the unchanged task prompts and durable-output oracle. Replay of retained v2 evidence is a separate diagnostic, never a replacement for its original verdict. See the [answer correction](../../doc/plans/2026-10-04-native-completion-answer-fix.md).
The corrected source variants add explicit blocker explanations and canonical document citations. Accepted feedback repeats links only for this run's current saved revisions, and Markdown navigation preserves document anchors after issue details load. Observation v3 independently requires the persisted provider final to explain missing release/deployment access and, for completion, link this task's one revisioned document on the same origin. A correct structured blocker, an unblock action alone, or an unbound/foreign document URL cannot pass. These stricter checks and browser navigation apply only to the manual instruction comparison; the existing `native-completion` suite keeps v2 checks. Task prompts and the durable-output oracle remain unchanged. Admission now requires eighteen shared runnerd RPC captures and six direct OpenCode HTTP captures using a local fake server, all provider-free. Replay of retained v2 evidence is a separate diagnostic, never a replacement for its original verdict. See the [answer correction](../../doc/plans/2026-10-04-native-completion-answer-fix.md).
`context-integrity` is an explicit-only local suite with two bounded cases across
ten listed legacy/native profiles (20 cells). Six cells are pending-prerequisite
@@ -1,7 +1,7 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import { rehydrateRunnerdItemNotification } from "../../packages/paperclip-runner/src/live/runnerd-codex-transport.js";
import { gradeNativeCompletion, type NativeCompletionObservation } from "./native-completion-scoring.js";
import { gradeNativeCompletion as gradeLegacyNativeCompletion, gradeNativeCompletionFinalAnswer as gradeNativeCompletion, type NativeCompletionObservation } from "./native-completion-scoring.js";
type Row = Record<string, unknown>;
function sample(blocked = true, compatibility = false): NativeCompletionObservation {
const body = blocked ? "Deployment remains blocked until access is granted. Release Owner must Grant deployment access. BLOCKED_probe" : "Saved [the requested document](/RUN/issues/RUN-1#document-output).";
@@ -26,6 +26,15 @@ function sample(blocked = true, compatibility = false): NativeCompletionObservat
}
function payload(input: NativeCompletionObservation, index: number): Row { return ((input.events[index]!.payload as Row).prpEvent as Row).payload as Row; }
describe("native completion independent oracle", () => {
it("keeps legacy completion verdicts separate from final-answer diagnostics", () => {
const value = sample(false);
const text = "Saved the requested document.";
(payload(value, 3).item as Row).text = text; value.comments[0]!.body = text;
expect(gradeLegacyNativeCompletion(value)).toMatchObject({ schema: "paperclip.native-completion-observation.v2", passed: true });
expect(gradeNativeCompletion(value)).toMatchObject({ schema: "paperclip.native-completion-observation.v3", passed: false });
expect(gradeLegacyNativeCompletion(value).checks.some(check => check.id === "saved-document-final-link")).toBe(false);
});
it("rejects a correct structured blocker whose visible reply only labels it blocked and repeats the action", () => {
const value = sample();
const text = "The whole task is blocked. Owner: Release Owner.\n\nUnblock action: Grant deployment access\n\nBLOCKED_probe";
+12 -3
View File
@@ -20,6 +20,15 @@ export interface NativeCompletionObservation {
/** Observable tool/result/final sequence, not a claim that the provider consumed feedback. */
export function gradeNativeCompletion(input: NativeCompletionObservation) {
return gradeObservation(input, false);
}
/** Stricter final-answer checks belong to the instruction comparison only. */
export function gradeNativeCompletionFinalAnswer(input: NativeCompletionObservation) {
return gradeObservation(input, true);
}
function gradeObservation(input: NativeCompletionObservation, finalAnswer: boolean) {
const checks: Array<{ id: string; passed: boolean; detail: string }> = [];
const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail });
const blocked = input.caseId === "native-blocked-report";
@@ -117,9 +126,9 @@ export function gradeNativeCompletion(input: NativeCompletionObservation) {
&& /\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\b/i.test(finalText)
&& !/\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\b/i.test(finalText),
"The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker.");
if (blocked) check("visible-blocker-reason", explainsMissingReleaseAccess(finalText),
if (finalAnswer && blocked) check("visible-blocker-reason", explainsMissingReleaseAccess(finalText),
"The persisted provider final independently explains the missing release/deployment access; a blocked label or unblock action alone is insufficient.");
else check("saved-document-final-link", linksSavedNativeDocument(finalText, input.documentLinkContext),
else if (finalAnswer) check("saved-document-final-link", linksSavedNativeDocument(finalText, input.documentLinkContext),
"The persisted provider final links this task's one saved, revisioned document at the canonical same-origin anchor.");
const same = (a: string[], b: string[]) => a.length === b.length && a.every(id => b.includes(id)) && new Set(a).size === a.length;
check("bounded-durable-work", same(input.state.issueIds, [...input.initial.issueIds, String(input.issue.id)])
@@ -136,6 +145,6 @@ export function gradeNativeCompletion(input: NativeCompletionObservation) {
check("no-deployment-or-file-work", !input.workspaceChanged && !forbidden,
"The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed.");
}
return { schema: "paperclip.native-completion-observation.v3", passed: checks.every(value => value.passed), checks,
return { schema: finalAnswer ? "paperclip.native-completion-observation.v3" : "paperclip.native-completion-observation.v2", passed: checks.every(value => value.passed), checks,
limitations: ["Exact provider feedback identity/consumption is not measured by the public sequence."] };
}
@@ -37,7 +37,7 @@ describe("native instruction comparison admission", () => {
expect(nativeInstructionVariant(file => baseline.get(file)!)).toBe("baseline");
const candidate = new Map(files.map(file => [file, readFileSync(new URL(`../../${file}`, import.meta.url))]));
const currentVariant = nativeInstructionVariant(file => candidate.get(file)!);
expect(["baseline", "candidate", "corrected"]).toContain(currentVariant);
expect(["baseline", "candidate", "corrected", "feedback"]).toContain(currentVariant);
candidate.set(files[0]!, currentVariant === "candidate" ? baseline.get(files[0]!)! : Buffer.from("unknown source"));
expect(() => nativeInstructionVariant(file => candidate.get(file)!)).toThrow("Mixed or unknown");
baseline.set(files[0]!, Buffer.from("unknown source"));
@@ -74,15 +74,19 @@ describe("native instruction comparison admission", () => {
it("rejects duplicate or incomplete capture, dirty or paid source, and stale fixture evidence", () => {
const measurement = {
schema: "paperclip.native-instruction-measurement.v1", sourceSha: "frozen", sourceDirty: false, providerCalls: 0,
schema: "paperclip.native-instruction-measurement.v2", sourceSha: "frozen", sourceDirty: false, providerCalls: 0,
fixtureSha256: createHash("sha256").update(readFileSync(new URL("../../packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts", import.meta.url))).digest("hex"),
sourceHashes: Object.fromEntries(Object.entries(NATIVE_INSTRUCTION_VARIANTS.baseline).map(([file, digest]) => [file.split('/').at(-1)!, digest])),
directOpenCodeReceipts: ["v4", "v5"].flatMap(schema => ["start", "resume", "continuation"].map(phase => ({ provider: "opencode", schema, phase }))),
receipts: ['codex', 'acpx', 'opencode'].flatMap(provider => ['v4', 'v5'].flatMap(schema => ['start', 'resume', 'continuation'].map(phase => ({ provider, schema, phase })))),
};
const source = { sourceSha: "frozen", variant: "baseline" };
expect(() => validateNativeInstructionMeasurement(measurement, source)).not.toThrow();
for (const invalid of [
{ ...measurement, receipts: measurement.receipts.slice(1) },
{ ...measurement, directOpenCodeReceipts: undefined },
{ ...measurement, directOpenCodeReceipts: measurement.directOpenCodeReceipts.slice(1) },
{ ...measurement, directOpenCodeReceipts: measurement.directOpenCodeReceipts.map(() => measurement.directOpenCodeReceipts[0]!) },
{ ...measurement, receipts: measurement.receipts.map(() => measurement.receipts[0]!) },
{ ...measurement, sourceDirty: true }, { ...measurement, providerCalls: 1 },
{ ...measurement, sourceSha: "other" }, { ...measurement, fixtureSha256: "stale" },
@@ -19,22 +19,40 @@ export const NATIVE_INSTRUCTION_VARIANTS = {
"packages/paperclip-runner/src/backends/codex-native-backend.ts": "f276d436391c9e2f0a7e58ce1302b5c9f1a39ac01c148be257f9d455515a31b9",
"packages/paperclip-runner/src/backends/opencode-native-backend.ts": "d54ddde3a3fdffcda3490d813d27b3b4891448e11cd05f61f9bb4b292d9fd6c2",
"server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "2da6963c690a3988bf2617d39e85025600b12a056a61c7b528c3d94ac636f777",
"server/src/services/native-runtime/native-completion-feedback.ts": "72a8da837e93d1ab6f732bc88cc05de784ed771ab2f538ca1cf4768864cc678b",
"ui/src/lib/issue-reference.ts": "ed965e30b6221a455e1dec4c6e51b1659b122828b70c289eeb0d95d9c897d242",
"ui/src/components/MarkdownBody.tsx": "07543156b81e51e72cfc77262f135bdb19eaa6bcccf6a3aef60c20faddcca759",
},
candidate: {
"packages/paperclip-runner/src/backends/runtime-context.ts": "cae9075fac25f168972c2be4ddb58052ef2940554458e2757d88a5d0e39805c2",
"packages/paperclip-runner/src/backends/codex-native-backend.ts": "affecc515a623e0dfeb338f553ea53ebb7d18baa3d13e4395d17b21000dbee93",
"packages/paperclip-runner/src/backends/opencode-native-backend.ts": "d5dc4cc2c06b37d22be47c9f15f828c06b439d8241a8d34498ca4339273eed96",
"server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "2da6963c690a3988bf2617d39e85025600b12a056a61c7b528c3d94ac636f777",
"server/src/services/native-runtime/native-completion-feedback.ts": "72a8da837e93d1ab6f732bc88cc05de784ed771ab2f538ca1cf4768864cc678b",
"ui/src/lib/issue-reference.ts": "ed965e30b6221a455e1dec4c6e51b1659b122828b70c289eeb0d95d9c897d242",
"ui/src/components/MarkdownBody.tsx": "07543156b81e51e72cfc77262f135bdb19eaa6bcccf6a3aef60c20faddcca759",
},
corrected: {
"packages/paperclip-runner/src/backends/runtime-context.ts": "bbdab79c5b1bd57c4ddbc44edfe745b40eb694aa28a465452964109aedd6104b",
"packages/paperclip-runner/src/backends/codex-native-backend.ts": "affecc515a623e0dfeb338f553ea53ebb7d18baa3d13e4395d17b21000dbee93",
"packages/paperclip-runner/src/backends/opencode-native-backend.ts": "d5dc4cc2c06b37d22be47c9f15f828c06b439d8241a8d34498ca4339273eed96",
"server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "d2360cdaa63902cbf0c6a6109da452ba7e0b2a5aaa76ee5cab9c63b4bf12e584",
"server/src/services/native-runtime/native-completion-feedback.ts": "72a8da837e93d1ab6f732bc88cc05de784ed771ab2f538ca1cf4768864cc678b",
"ui/src/lib/issue-reference.ts": "ed965e30b6221a455e1dec4c6e51b1659b122828b70c289eeb0d95d9c897d242",
"ui/src/components/MarkdownBody.tsx": "07543156b81e51e72cfc77262f135bdb19eaa6bcccf6a3aef60c20faddcca759",
},
feedback: {
"packages/paperclip-runner/src/backends/runtime-context.ts": "bbdab79c5b1bd57c4ddbc44edfe745b40eb694aa28a465452964109aedd6104b",
"packages/paperclip-runner/src/backends/codex-native-backend.ts": "affecc515a623e0dfeb338f553ea53ebb7d18baa3d13e4395d17b21000dbee93",
"packages/paperclip-runner/src/backends/opencode-native-backend.ts": "d5dc4cc2c06b37d22be47c9f15f828c06b439d8241a8d34498ca4339273eed96",
"server/src/services/native-runtime/paperclip-runner-tool-authority.ts": "d2360cdaa63902cbf0c6a6109da452ba7e0b2a5aaa76ee5cab9c63b4bf12e584",
"server/src/services/native-runtime/native-completion-feedback.ts": "1f9da911ec5107c6b7543bc1b4134fc74597a29af6ed40101a895045a9e26ce0",
"ui/src/lib/issue-reference.ts": "ab578752acc7e185333cb2b701f6db71cedbb44675db042f78fa88417e7b2ddf",
"ui/src/components/MarkdownBody.tsx": "f3608614b143667f7dba087be81fd1449e4eac268a203e41256d1c99691c2541",
},
} as const;
// These are common comparison setup, not additional production changes.
// Admission explicitly binds every changed production path as well as comparison setup.
const comparisonFiles = new Set([
...Object.keys(NATIVE_INSTRUCTION_VARIANTS.baseline),
"packages/paperclip-runner/src/backends/runtime-context.test.ts",
@@ -48,6 +66,8 @@ const comparisonFiles = new Set([
"tests/runner-e2e/native-completion-scoring.test.ts", "tests/runner-e2e/native-completion-content.ts",
"tests/runner-e2e/native-completion-content.test.ts",
"server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts",
"server/src/services/native-runtime/native-completion-feedback.test.ts",
"ui/src/lib/issue-reference.test.ts", "ui/src/components/MarkdownBody.test.tsx",
"tests/runner-e2e/catalog.ts", "tests/runner-e2e/catalog.test.ts", "tests/runner-e2e/launch.ts",
"tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/live-fixtures.ts",
"tests/runner-e2e/README.md", "doc/evals.md",
@@ -146,12 +166,16 @@ function runnerdProof(source: ReturnType<typeof sourceReceipt>) {
export function validateNativeInstructionMeasurement(measurement: {
schema: string; sourceSha: string; sourceDirty: boolean; providerCalls: number; fixtureSha256: string;
sourceHashes: Record<string, string>; receipts: Array<{ provider: string; schema: string; phase: string }>;
directOpenCodeReceipts?: Array<{ provider: string; schema: string; phase: string }>;
}, source: { sourceSha: string; variant: string }) {
const expected = ['codex', 'acpx', 'opencode'].flatMap(provider => ['v4', 'v5'].flatMap(schema =>
['start', 'resume', 'continuation'].map(phase => `${provider}/${schema}/${phase}`))).sort();
const actual = measurement.receipts.map(value => `${value.provider}/${value.schema}/${value.phase}`).sort();
const files = NATIVE_INSTRUCTION_VARIANTS[source.variant as keyof typeof NATIVE_INSTRUCTION_VARIANTS];
if (measurement.schema !== "paperclip.native-instruction-measurement.v1"
const directExpected = ["v4", "v5"].flatMap(schema => ["start", "resume", "continuation"].map(phase => `opencode/${schema}/${phase}`)).sort();
const directActual = measurement.directOpenCodeReceipts?.map(value => `${value.provider}/${value.schema}/${value.phase}`).sort();
if (measurement.schema !== "paperclip.native-instruction-measurement.v2"
|| JSON.stringify(directActual) !== JSON.stringify(directExpected)
|| measurement.sourceSha !== source.sourceSha || measurement.sourceDirty !== false || measurement.providerCalls !== 0
|| measurement.fixtureSha256 !== hash(readFileSync(join(root, "packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts")))
|| JSON.stringify(actual) !== JSON.stringify(expected) || !files
+4 -3
View File
@@ -1,7 +1,7 @@
import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js";
import { assertNativeInstructionSelection, verifyNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_DEFAULT_SHA256 } from "./native-instruction-consolidation.js";
import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js";
import { gradeNativeCompletion } from "./native-completion-scoring.js";
import { gradeNativeCompletion, gradeNativeCompletionFinalAnswer } from "./native-completion-scoring.js";
import { assertNativeBlockerReply } from "./native-blocker-visible.js";
import { warmManagedFileEvidence } from "./warm-managed-files.js";
import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js";
@@ -2702,12 +2702,13 @@ for (const execution of executions) {
workspaceChanged: workspaceDigest !== nativeInitial.workspaceDigest, marker,
documentLinkContext: { appOrigin: new URL(page.url()).origin, issuePrefix: fixtures.company.issuePrefix ?? "",
issueIdentifier: currentIssue.identifier ?? "", documents } };
const grade = gradeNativeCompletion(observation);
const grade = execution.suite.id === NATIVE_INSTRUCTION_SUITE
? gradeNativeCompletionFinalAnswer(observation) : gradeNativeCompletion(observation);
const integrity = detailedRuns.flatMap(candidate => nativeRunEventIntegrityFailures(candidate, events));
await writeSanitizedJson(snapshotsDir, "native-completion.json", { observation, grade, integrity }, secrets);
matcherResults.push(...grade.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `nativeCompletion.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })));
if (!grade.passed || integrity.length) throw new Error(`Native completion matcher failure: ${[...grade.checks.filter(check => !check.passed).map(check => check.id), ...integrity].join("; ")}`);
if (execution.task.id === "assigned-skill-explicit-invocation") {
if (execution.suite.id === NATIVE_INSTRUCTION_SUITE && execution.task.id === "assigned-skill-explicit-invocation") {
const document = documents[0]!;
const href = `/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(currentIssue.identifier!)}#document-${encodeURIComponent(document.key)}`;
let opened = false;