mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 07:55:47 +01:00
161 lines
14 KiB
TypeScript
161 lines
14 KiB
TypeScript
import { constants } from "node:fs";
|
|
import { lstat, open, readFile, realpath } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { PAPERCLIP_PROTOCOL_ACTIONS } from "../../packages/paperclip-runner/src/protocol-actions/index.js";
|
|
|
|
// Reject malformed counters wholesale. Unknown input never becomes a diagnostic
|
|
// string, path, or authority; even a complete scan remains unqualified evidence.
|
|
function closedPackagingBudget(input: unknown, message: string) {
|
|
const b = input && typeof input === "object" ? input as Record<string, any> : {};
|
|
const kinds = ["json", "log", "image", "video", "zip", "html", "text", "asset"];
|
|
const number = (v: unknown) => Number.isSafeInteger(v) && (v as number) >= 0;
|
|
if (![b.listedFiles, b.filesExamined, b.bytesExamined].every(number) || b.listedFiles > 4097 || b.filesExamined > 4096
|
|
|| b.filesExamined > b.listedFiles || typeof b.inventoryComplete !== "boolean" || typeof b.scanFinished !== "boolean"
|
|
|| ![true, false, null].includes(b.requiredEvidenceComplete)) return null;
|
|
const byKind: Record<string, { files: number; bytes: number }> = {};
|
|
for (const kind of kinds) {
|
|
const row = b.byKind?.[kind];
|
|
if (!row || !number(row.files) || !number(row.bytes)) return null;
|
|
byKind[kind] = { files: row.files, bytes: row.bytes };
|
|
}
|
|
if (Object.values(byKind).reduce((n, r) => n + r.files, 0) !== b.filesExamined
|
|
|| Object.values(byKind).reduce((n, r) => n + r.bytes, 0) !== b.bytesExamined
|
|
|| b.inventoryComplete && b.filesExamined !== b.listedFiles || b.scanFinished && !b.inventoryComplete) return null;
|
|
if (b.requiredEvidenceComplete === true && (!b.scanFinished || b.bound !== "none")) return null;
|
|
const observedBound = ["none", "physical-bytes", "entries", "zip-expanded-bytes", "zip-time", "html-encoded-bytes"].includes(b.bound) ? b.bound : "unknown";
|
|
const bound = message === "Complete original evidence exceeds byte allowance" ? "original-seal-bytes"
|
|
: message === "Complete original evidence exceeds entry allowance" ? "original-seal-entries"
|
|
: message === "Complete packaged evidence exceeds allowance" ? "final-seal-allowance" : observedBound;
|
|
return { scope: "package-standard-files", bound, listedFiles: b.listedFiles, filesExamined: b.filesExamined, bytesExamined: b.bytesExamined,
|
|
inventoryComplete: b.inventoryComplete, scanFinished: b.scanFinished, requiredEvidenceComplete: b.requiredEvidenceComplete,
|
|
byKind, qualification: "unqualified" };
|
|
}
|
|
export interface PrivateAttemptDiagnosticInput { stage: unknown; result?: unknown; error?: unknown; observation?: unknown; evidence?: unknown }
|
|
const record = (value: unknown): Record<string, any> => value && typeof value === "object" ? value as Record<string, any> : {};
|
|
const member = (value: unknown, choices: string[]) => typeof value === "string" && choices.includes(value) ? value : "unknown";
|
|
const failures = new Map([
|
|
["Credential-contaminated original evidence cannot be retained", "credential-contamination"],
|
|
["Unsupported original evidence carrier cannot be retained", "unsupported-carrier"],
|
|
["Complete original evidence exceeds byte allowance", "byte-bound"],
|
|
["Complete original evidence exceeds entry allowance", "entry-bound"],
|
|
["Private cell evidence exceeds byte allowance", "byte-bound"],
|
|
["Private cell evidence exceeds entry allowance", "entry-bound"],
|
|
["Unsupported private cell evidence carrier", "unsupported-carrier"],
|
|
["Unsupported private cell trace owner", "trace-owner"],
|
|
["Invalid private cell trace asset", "trace-asset"],
|
|
["Private cell trace content differs from canonical producer", "trace-producer-mismatch"],
|
|
["Private cell report producer differs from trusted preflight", "report-producer-mismatch"],
|
|
["Canonical report source changed", "report-source-changed"],
|
|
["Invalid canonical report source", "report-source"],
|
|
]);
|
|
const phases = [["Real prior active Launch checkout", "held-work-timeout"], ["Canonical project-defaults turn settles", "canonical-turn-timeout"], ["Actual new native task reads inherited project context", "future-work-timeout"]];
|
|
const toolNames = ["bash", "paperclip_call_api", "paperclip_finish", "paperclip_get_workspace", "paperclip_get_run", "paperclip_list_runs"];
|
|
const recognizedTools = new Set([...toolNames, "read", "glob", "grep", "task", "todowrite", "question",
|
|
...PAPERCLIP_PROTOCOL_ACTIONS.map(action => `paperclip_${action.id}`)]);
|
|
const toolName = (value: unknown) => typeof value === "string" && recognizedTools.has(value) ? value : "other";
|
|
const apiOperations = new Map([
|
|
["GET /api/projects/{id}", "read-project"], ["PATCH /api/projects/{id}", "update-project"],
|
|
["GET /api/companies/{companyId}/projects", "list-projects"], ["GET /api/companies/{companyId}/goals", "list-goals"],
|
|
["GET /api/issues/{id}", "read-task"], ["PATCH /api/issues/{id}", "update-task"],
|
|
["GET /api/companies/{companyId}/issues", "list-tasks"], ["GET /api/companies/{companyId}/agents", "list-agents"],
|
|
]);
|
|
function sourceProgress(value: unknown) {
|
|
const rows = Array.isArray(value) ? value : [];
|
|
const events = rows.map(row => record(record(row).payload).prpEvent ?? row).map(record);
|
|
const tools = events.filter(event => ["tool.execution.started", "tool.execution.progressed", "tool.execution.completed"].includes(event.eventType));
|
|
const names = [...new Set([...toolNames, ...tools.map(event => toolName(record(event.payload).name)), "other"])];
|
|
return {
|
|
events: Math.min(events.length, 100000),
|
|
turns: Math.min(events.filter(event => event.eventType === "turn.started").length, 1000),
|
|
finishProposals: Math.min(events.filter(event => event.eventType === "run.result.proposed").length, 1000),
|
|
finishFeedback: Object.fromEntries(([
|
|
["timeout", /MCP error -32001|timed out/i], ["contract", /Stale completionClaim\.contractRevision/i],
|
|
["criteria", /completionClaim\.criteria must contain exactly/i], ["incomplete", /done report includes unfinished work/i],
|
|
["approval", /requires a persisted human decision|Required human quality review is incomplete/i],
|
|
] as const).map(([label, pattern]) => [label, Math.min(tools.filter(event => event.eventType === "tool.execution.completed"
|
|
&& record(event.payload).name === "paperclip_finish" && pattern.test(String(record(event.payload).output ?? ""))).length, 1000)])),
|
|
tools: Object.fromEntries(names.map(name => [name, Object.fromEntries(["started", "progressed", "completed"].map(status =>
|
|
[status, Math.min(tools.filter(event => event.eventType === `tool.execution.${status}` &&
|
|
toolName(record(event.payload).name) === name).length, 10000)]))])),
|
|
lastTools: tools.filter(event => event.eventType !== "tool.execution.progressed").slice(-12).map(event => ({
|
|
name: toolName(record(event.payload).name), phase: event.eventType === "tool.execution.started" ? "started" : "completed",
|
|
status: member(record(event.payload).status, ["running", "completed", "failed", "cancelled"]),
|
|
})),
|
|
apiResults: events.filter(event => event.eventType === "item.completed" && record(event.payload).kind === "dynamicToolCall")
|
|
.map(event => record(record(record(event.payload).item).result)).filter(result => typeof result.apiOperationId === "string").slice(-12)
|
|
.map(result => ({ operation: apiOperations.get(result.apiOperationId) ?? "other",
|
|
status: Number.isSafeInteger(result.status) && result.status >= 100 && result.status <= 599 ? result.status : null, ok: result.ok === true })),
|
|
};
|
|
}
|
|
/** Closed labels/counts only; never copy an error, identity, path, body or provider value. */
|
|
export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) {
|
|
const result = record(input.result), envelope = record(input.observation), observation = record(envelope.observation), state = record(observation.state);
|
|
const runs = Array.isArray(observation.runs) ? observation.runs.map(record) : [];
|
|
const status = (run: Record<string, any> | undefined) => member(run?.status, ["queued", "running", "succeeded", "failed", "cancelled", "timed_out"]);
|
|
const message = input.error instanceof Error ? input.error.message : "", productError = typeof result.error === "string" ? result.error : "";
|
|
const privacy = record(envelope.privacy), original = record(privacy.originalResult), persistedState = record(envelope.persistedState);
|
|
const errors = [productError, original.error, persistedState.scanError, record(persistedState.originalResult).error].filter(value => typeof value === "string") as string[];
|
|
const evidence = record(input.evidence), leaks = Array.isArray(evidence.leaks) ? evidence.leaks : [];
|
|
return {
|
|
schema: "paperclip.private-attempt-diagnostic/v1",
|
|
packagingBudget: closedPackagingBudget(record(input.error).evidenceBudget ?? evidence.budget, message),
|
|
stage: member(input.stage, ["execution", "package-original", "seal-original"]),
|
|
status: member(result.status, ["passed", "failed", "skipped"]),
|
|
failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]),
|
|
finalization: failures.get(message) ?? (record(input.error).code === "ENOENT" ? "missing-file"
|
|
: /^Secret leak in [\s\S]*: (?:exact secret value|secret-shaped value)$/.test(message) ? "text-secret-finding" : "unclassified"),
|
|
phase: phases.find(([label]) => [...errors, message].some(error => error.includes(label!)))?.[1] ?? "unclassified",
|
|
privacy: { persistedState: errors.some(error => error.startsWith("Secret leak in persisted ")),
|
|
reporter: Array.isArray(privacy.issues) ? Math.min(privacy.issues.length, 1000) : 0,
|
|
reporterReasons: Object.fromEntries(["zip inspection exceeded its expanded-byte bound", "zip inspection exceeded its time bound", "exact secret value", "secret-shaped value"].map(reason =>
|
|
[reason, Array.isArray(privacy.issues) ? Math.min(privacy.issues.filter(issue => record(issue).reason === reason).length, 1000) : 0])),
|
|
packaging: /^Secret leak in /.test(message) ? (/^Secret leak in snapshots\//.test(message) ? "snapshot" : /^Secret leak in .*\.json:/.test(message) ? "json" : "text") : "unknown" },
|
|
runStatusCounts: Object.fromEntries(["queued", "running", "succeeded", "failed", "cancelled", "timed_out"].map(label =>
|
|
[label, Math.min(runs.filter(run => run.status === label).length, 1000)])),
|
|
runs: Math.min(runs.length || (Array.isArray(result.runIds) ? result.runIds.length : 0), 1000),
|
|
checks: Array.isArray(result.matcherResults) ? Math.min(result.matcherResults.length, 1000) : 0,
|
|
source: status(runs.find(run => typeof observation.leadId === "string" && run.agentId === observation.leadId)),
|
|
future: status(runs.find(run => typeof record(state.futureWorker).id === "string" && run.agentId === record(state.futureWorker).id)),
|
|
held: status(runs.find(run => typeof state.activeRunId === "string" && run.id === state.activeRunId)),
|
|
productGradePassed: record(envelope.grade).passed === true,
|
|
sourceProgress: sourceProgress(envelope.sourceEvents),
|
|
carrier: leaks.some(leak => record(leak).reason === "zip inspection exceeded its expanded-byte bound") ? "zip-byte-bound"
|
|
: leaks.some(leak => record(leak).reason === "zip inspection exceeded its time bound") ? "zip-time-bound" : leaks.length ? "other-finding" : "none",
|
|
};
|
|
}
|
|
export async function readPrivateAttemptObservation(casePrivateDir: string) {
|
|
const read = async (name: string, limit = 8 * 1024 * 1024) => {
|
|
try {
|
|
const file = path.join(casePrivateDir, "snapshots", name), metadata = await lstat(file);
|
|
if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > limit) return;
|
|
return JSON.parse(await readFile(file, "utf8"));
|
|
} catch { return; }
|
|
};
|
|
const envelope = record(await read("chat-catalog-observation.json"));
|
|
const observation = record(envelope.observation);
|
|
const runs = Array.isArray(observation.runs) ? observation.runs.map(record) : [];
|
|
const source = runs.find(run => typeof observation.leadId === "string" && run.agentId === observation.leadId);
|
|
const evidence = await read("chat-catalog-runs.json", 64 * 1024 * 1024);
|
|
const sourceEvents = Array.isArray(evidence) ? record(evidence.find(row => source && record(row).runId === source.id)).events : undefined;
|
|
return { ...envelope, sourceEvents, privacy: await read("diagnostic-privacy.json"), persistedState: await read("persisted-state-scan.json") };
|
|
}
|
|
/**
|
|
* GITHUB_ENV survives the paid step's private stdout redirect. SetEnvFileCommand
|
|
* updates the env context; ActionRunner passes it to ScriptHandler, whose
|
|
* PrintActionDetails prints it before the next trusted shell step runs.
|
|
* https://github.com/actions/runner/blob/main/src/Runner.Worker/Handlers/ScriptHandler.cs#L120-L126
|
|
* The variable is not consumed as authority by the protected transfer.
|
|
*/
|
|
export async function appendPrivateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput, env = process.env) {
|
|
if (env.GITHUB_ACTIONS !== "true") return;
|
|
if (!env.GITHUB_ENV || !env.RUNNER_TEMP) throw new Error("Closed private diagnostic channel is unavailable");
|
|
const root = await realpath(env.RUNNER_TEMP), parent = await realpath(path.dirname(env.GITHUB_ENV));
|
|
if (!parent.startsWith(`${root}${path.sep}`) || !path.basename(env.GITHUB_ENV).startsWith("set_env_")) throw new Error("Diagnostic environment file escaped runner commands");
|
|
const file = await open(env.GITHUB_ENV, constants.O_WRONLY | constants.O_APPEND | constants.O_NOFOLLOW);
|
|
try {
|
|
const metadata = await file.stat();
|
|
if (!metadata.isFile() || typeof process.getuid === "function" && metadata.uid !== process.getuid()) throw new Error("Invalid diagnostic environment file owner");
|
|
await file.write(`PAPERCLIP_E2E_CLOSED_DIAGNOSTIC_V1=${JSON.stringify(privateAttemptDiagnostic(input))}\n`);
|
|
} finally { await file.close(); }
|
|
}
|