Merge latest Codex qualification and master fixes

Co-Authored-By: Paperclip <noreply@paperclip.ing>

* codex/claude-runner-default: (25 commits)
  fix: pin ARM qualification helper to workflow source
  perf(ci): cache the Runner's release crate artifacts keyed by their inputs (#15932)
  perf(build): reuse a current Runner build in the server's prepare:runner-vendor step (#15926)
  fix(test): make the vitest home teardown remove read-only runtime-context bundles (#15931)
  fix(test): stop the Sentry timeout assertion matching a random event id (#15929)
  test: qualify Linux ARM with stock sandbox prerequisites
  docs(release): align 2026.1006.0-beta.0 stable notes header with v2026.1012.0 (#16000)
  fix: use Hermes adapter API URL env in prompt (#9425)
  fix(docker): skip the full-tree ownership walk on routine boots (#15992)
  refactor(chat): isolate Telegram, Slack, GitHub and Discord workflows (#15976)
  Preserve repository dependencies when importing GitHub skills (#15731)
  refactor(heartbeat): extract instruction working-copy lifecycle (#15984)
  test: retire redundant suites and consolidate owner coverage (#15978)
  refactor(heartbeat): extract runtime service coordination (#15980)
  refactor(heartbeat): extract execution configuration (#15973)
  refactor(heartbeat): extract run output recording (#15969)
  refactor(chat): extract failed-run retry coordination (#15845)
  refactor(chat): extract publication replacement decisions (#15844)
  refactor(chat): extract identity linking (#15836)
  refactor(heartbeat): extract session preparation (#15966)
  ...

# Conflicts:
#	packages/paperclip-runner/package.json
This commit is contained in:
Dotta committed 2026-10-12 13:45:03 -05:00
commit adf258ede2
210 files changed
+25068 -25979

No files matched your search

+35
View File
@@ -2214,3 +2214,38 @@ Keep exact source, runtime executable digests and reports for each candidate.
Effort or runtime changes require affected acceptance reruns. No automatic paid
retry campaign is permitted under the separate $50 Claude qualification ceiling.
Docker and expensive Rust builds run in cloud CI, never on the local Mac.
## Repository skill packages
The explicit `repository-skills` suite imports the private
`paperclipai/poteto-stack` repository through the production GitHub Sources UI,
selects `architect` and `bro` in repository mode, and assigns those imported
skills to a production-default Codex agent. It disables the source connection
before creating the task through the browser. No fixture copies the repository
into an agent workspace.
Set `GITHUB_TOKEN` with read access to that repository, `OPENAI_API_KEY`, and,
for Daytona, `DAYTONA_API_KEY` and the immutable `PAPERCLIP_E2E_DAYTONA_IMAGE`.
On macOS, also set `PAPERCLIP_RUNNER_REMOTE_BINARY_PATH` to the verified Linux
runner from that pinned image, as described above.
Credentials follow the existing encrypted company-secret and redaction paths.
These tests send the selected private skill instructions to OpenAI and deliver
the package to the selected runtime.
```sh
pnpm test:e2e:runner -- --id repository-skills.runner-codex.local.poteto-package
pnpm test:e2e:runner -- --id repository-skills.runner-codex.daytona.poteto-package
```
Each case has one provider run and no automatic retries. While the agent pauses
after reading its two installed skills and publishing their runtime locations
in a run-attributed progress comment,
the test independently reads both delivered directories. On Daytona it uses the
SDK only to inspect the exact active sandbox lease owned by this test's company,
agent, issue and run. Paths must belong to that native session's provider home
or its immutable context bundles. Local prepared assets must be explicitly
listed for the selected skills in the active session's digest-validated runtime
context; an arbitrary cache directory does not qualify. Git blob hashes, sizes and executable bits must match every
tracked file in the imported commit, including sibling skills, shared agents,
binary assets and upstream credits. Both distinct discovery wrappers must point
to their canonical entrypoints. The task must finish successfully in the UI.
+2 -2
View File
@@ -46,8 +46,8 @@ export class RunnerApi {
return response.json() as Promise<T>;
}
async post<T>(path: string, data?: unknown): Promise<T> {
const response = await this.request.post(path, { data, ...this.sessionOptions() });
async post<T>(path: string, data?: unknown, options?: { timeout: number }): Promise<T> {
const response = await this.request.post(path, { data, ...this.sessionOptions(), ...options });
if (!response.ok()) throw new Error(await failureMessage(response, "POST"));
return response.json() as Promise<T>;
}
+16 -3
View File
@@ -32,6 +32,19 @@ import {
} from "./selectors.js";
describe("runner E2E catalog", () => {
it("keeps private repository skill delivery explicit and bounded in both environments", () => {
const cases = runnerMatrix.filter(entry => entry.suite.id === "repository-skills");
expect(cases).toHaveLength(2);
expect(cases.map(entry => entry.environment.id)).toEqual(["local", "daytona"]);
for (const entry of cases) {
expect(entry.suite.manualOnly).toBe(true);
expect(entry.task.automaticRetryPolicy).toBe("single_attempt");
expect(entry.task.expectedRunCount).toBe(1);
expect(entry.requiredCredentials).toContain("GITHUB_TOKEN");
expect(entry.requiredCredentials).toContain("OPENAI_API_KEY");
expect(entry.requiredCredentials.includes("DAYTONA_API_KEY")).toBe(entry.environment.id === "daytona");
}
});
it.each(["daytona-journal-continuity"])(
"%s retains native processes with fixed external instructions",
(suiteId) => {
@@ -146,10 +159,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
4, 8, 10, 4, 4, 6, 63, 12, 6, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116,
4, 8, 10, 4, 4, 6, 63, 12, 6, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, 2,
]);
expect(validateRunnerCatalog()).toHaveLength(724);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(724);
expect(validateRunnerCatalog()).toHaveLength(726);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(726);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(48);
+13
View File
@@ -39,6 +39,7 @@ import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/dri
import { ACPX_QUALIFICATION_MODELS } from "../acpx-qualification-models.js";
import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js";
import { CREDENTIAL_NAMES } from "./types.js";
import { repositorySkillsTask, repositorySkillsProfile } from "./repository-skills.js";
import { createGitStreamingTask } from "./daytona-git-streaming.js";
import { PENDING_PROFILE_PREREQUISITES } from "./prerequisites.js";
import {
@@ -1607,6 +1608,15 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
definitionMetadata: { version: 8, instructions: "fixed-external", nativeIdleTimeoutMs: 1_200_000, autoStopIntervalMinutes: 25, generatedFileCount: 60_000, filenameBytes: 39_828_890, scheduling: "explicit-only", finalization: "committed-without-active-sync-or-retry", copyback: "all-generated-file-contents-change-each-turn" },
},
buildConnectionSuite(runnerProfiles, runnerEnvironments),
{
id: "repository-skills", label: "Repository skill packages", manualOnly: true,
description: "Live private Poteto import and byte-for-byte delivery to local and Daytona agent runtimes.",
groups: ["native"],
profiles: runnerProfiles.filter(profile => profile.id === "runner-codex").map(repositorySkillsProfile),
environments: [localEnvironment, daytonaWarmEnvironment], tasks: [repositorySkillsTask], expectedMatrixSize: 2,
definitionMetadata: { version: 1, scheduling: "explicit-only", maximumAttemptsPerCell: 1,
source: "paperclipai/poteto-stack", skills: ["architect", "bro"], grading: "independent-git-blob-hashes-and-file-modes-from-owned-runtime", budgetMonthlyCents: 1000 },
},
] as const;
export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
@@ -1628,6 +1638,7 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
id: task.id,
flow: task.flow,
expectedRunCount: task.expectedRunCount,
...(task.requiredCredentials ? { requiredCredentials: task.requiredCredentials } : {}),
...(task.minimumExpectedRunCount === undefined ? {} : { minimumExpectedRunCount: task.minimumExpectedRunCount }),
restartServerBeforeQuestionAnswer:
task.restartServerBeforeQuestionAnswer ?? false,
@@ -1670,6 +1681,7 @@ export function buildRunnerMatrix(
requiredCredentials: [
...(task.flow === "provider_connection" ? [] : [profile.credential]),
...(task.flow !== "provider_connection" && environment.credential ? [environment.credential] : []),
...(task.requiredCredentials ?? []),
],
}))
.filter((execution) => !excludedExecutionIds.has(execution.id)),
@@ -1718,6 +1730,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
const connectionSuite = runnerSuites.find(suite => suite.id === "provider-connections")!;
const allProfiles = [...connectionSuite.profiles, ...extendedHarnessProfiles, ...runnerProfiles, ...legacyAcpxProfiles, ...pendingContextIntegrityProfiles, ...openRouterBreadthProfiles, ...everydayProfiles.filter(p => !runnerProfiles.some(existing => existing.id === p.id))];
const allTasks = [
repositorySkillsTask,
...piNativeTasks,
...copilotProtectionTasks,
...connectionSuite.tasks,
+1 -1
View File
@@ -842,7 +842,7 @@ async function runAttempt(input: {
await chmod(temporaryRoot, 0o700);
console.warn(`Retained private failed-case diagnostics: ${temporaryRoot}`);
} else if (
temporaryRoot.startsWith(`${os.tmpdir()}${path.sep}paperclip-runner-e2e-`)
temporaryRoot.startsWith(`${temporaryParent}${path.sep}paperclip-runner-e2e-`)
) {
for (let cleanupAttempt = 1; cleanupAttempt <= 3; cleanupAttempt += 1) {
try {
+1 -1
View File
@@ -123,7 +123,7 @@ export async function setupLiveFixtures(input: {
"../../packages/plugins/sandbox-providers/daytona",
),
isLocalPath: true,
});
}, { timeout: 120_000 });
},
async teardown() {
// The plugin is installed only in the isolated instance/database. The
@@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => {
expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true);
expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]);
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false);
expect(validateRunnerCatalog()).toHaveLength(724);
expect(validateRunnerCatalog()).toHaveLength(726);
for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow();
});
+2 -1
View File
@@ -75,7 +75,8 @@ export default defineConfig({
gracefulShutdown: runnerE2EWebServerGracefulShutdown,
url: `${baseURL}/api/health`,
reuseExistingServer: false,
timeout: 180_000,
// Fresh databases apply the complete migration history before health is ready.
timeout: 300_000,
stdout: "pipe",
stderr: "pipe",
},
@@ -0,0 +1,69 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, mkdirSync, writeFileSync, chmodSync, rmSync, symlinkSync, realpathSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { gradeRepositoryBundles, inspectRepositoryBundle, isOwnedRepositoryRuntimeRoot, remoteRepositoryInspector, repositoryRuntimeSessionRoot } from "./repository-skill-evidence.js";
describe("repository runtime delivery evidence", () => {
it("binds local and remote copies to the owned native session and rejects library or foreign paths", () => {
const session = "54db6e46-cd6d-468e-8864-fd71658b355b";
const digest = "09a8b6d19d867b102dcb01a222d2dde7d3cb0054031af0d73ed93f3c718516a7";
const local = { kind: "local" as const, instanceRoot: "/owned/instance", owner: { companyId: "owned-company", agentId: "owned-agent",
driverKind: "codex_app_server", workspace: { cwd: "/owned/workspace", repoUrl: null, repoRef: null, branchName: null } } };
const localSessionRoot = "/owned/instance/runtime/paperclip-runner/durable-sessions/2881ee3ad02d8c09ec2af1ef786d588f92b7f25bb002b3da9ca19fa8d9331b5d";
expect(repositoryRuntimeSessionRoot(local.instanceRoot, session, local.owner)).toBe(localSessionRoot);
const localRoot = `${localSessionRoot}/codex-home/skills/architect`;
expect(isOwnedRepositoryRuntimeRoot(localRoot, session, local)).toBe(true);
expect(isOwnedRepositoryRuntimeRoot(localRoot, "another-session", local)).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(localRoot, session, { ...local, instanceRoot: "/foreign/instance" })).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(localRoot, session, { ...local, owner: { ...local.owner, companyId: "foreign-company" } })).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(localRoot, session, { ...local, owner: { ...local.owner, agentId: "foreign-agent" } })).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(localRoot, session, { ...local, owner: { ...local.owner, workspace: { ...local.owner.workspace, cwd: "/foreign/workspace" } } })).toBe(false);
expect(isOwnedRepositoryRuntimeRoot("/owned/instance/runtime-context-assets/bundles/digest", session, local)).toBe(false);
const preparedRoot = "/owned/instance/runtime-context-assets/bundles/assigned-digest";
const prepared = { ...local, contextSkills: { sessionId: session, roots: [preparedRoot] } };
expect(isOwnedRepositoryRuntimeRoot(preparedRoot, session, prepared)).toBe(true);
expect(isOwnedRepositoryRuntimeRoot(preparedRoot, "another-session", prepared)).toBe(false);
expect(isOwnedRepositoryRuntimeRoot("/owned/instance/runtime-context-assets/bundles/unassigned", session, prepared)).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(preparedRoot, session, { ...prepared, instanceRoot: "/foreign/instance" })).toBe(false);
expect(isOwnedRepositoryRuntimeRoot("/owned/instance/skills/library", session, { ...local, contextSkills: { sessionId: session, roots: ["/owned/instance/skills/library"] } })).toBe(false);
const remote = { kind: "daytona" as const, remoteCwd: "/home/daytona/workspace" };
const filesystem = `/home/daytona/workspace/.paperclip-runtime/paperclip-runner/sessions/${digest}/filesystem`;
for (const root of [`${filesystem}/context/skills/0-digest`, `${filesystem}/codex-home/skills/architect`]) {
expect(isOwnedRepositoryRuntimeRoot(root, session, remote)).toBe(true);
expect(isOwnedRepositoryRuntimeRoot(root, "another-session", remote)).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(root, session, { ...remote, remoteCwd: "/foreign/workspace" })).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(`${root}/nested`, session, remote)).toBe(false);
expect(isOwnedRepositoryRuntimeRoot(`${root}/../architect`, session, remote)).toBe(false);
}
});
it("checks actual bytes, binary files and executable bits locally and with the sandbox inspector", () => {
const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "repository-delivery-")));
try {
mkdirSync(path.join(root, ".paperclip-repository", "shared"), { recursive: true });
writeFileSync(path.join(root, "SKILL.md"), "Read .paperclip-repository/skills/main/SKILL.md");
const binary = path.join(root, ".paperclip-repository/shared/binary.dat");
writeFileSync(binary, Buffer.from([0, 255, 128]));
const script = path.join(root, ".paperclip-repository/shared/run.sh");
writeFileSync(script, "#!/bin/sh\necho ready\n"); chmodSync(script, 0o755);
const local = inspectRepositoryBundle(root);
// Run the exact shell program sent to Daytona, without a fake SDK or fake filesystem.
const remote = JSON.parse(execFileSync("sh", ["-c", remoteRepositoryInspector([root])], { encoding: "utf8" }));
expect(remote).toEqual([local]);
expect(local.files.find(file => file.path.endsWith("binary.dat"))?.sha).toBe("d90dae2bbeb97246b9570981c3eac7b6c403df7b");
expect(gradeRepositoryBundles([local], local.files, ["main"]).every(check => check.passed)).toBe(true);
writeFileSync(binary, Buffer.from([0, 255, 129]));
expect(gradeRepositoryBundles([inspectRepositoryBundle(root)], local.files, ["main"]).at(-1)?.passed).toBe(false);
writeFileSync(binary, Buffer.from([0, 255, 128])); chmodSync(script, 0o644);
expect(gradeRepositoryBundles([inspectRepositoryBundle(root)], local.files, ["main"]).at(-1)?.passed).toBe(false);
symlinkSync("/etc", path.join(root, ".paperclip-repository/escape"));
expect(() => inspectRepositoryBundle(root)).toThrow("Non-regular repository file");
} finally { rmSync(root, { recursive: true, force: true }); }
});
it("rejects a missing selection or duplicate runtime directory", () => {
const bundle = { root: "/fixture/skills/main", wrapper: ".paperclip-repository/skills/main/SKILL.md", files: [] };
expect(gradeRepositoryBundles([bundle, bundle], [], ["main", "other"]).filter(check => !check.passed).map(check => check.id))
.toEqual(["two-distinct-runtime-packages", "canonical-main", "canonical-other"]);
});
});
@@ -0,0 +1,88 @@
import { createHash } from "node:crypto";
import { lstatSync, readdirSync, readFileSync, realpathSync } from "node:fs";
import path from "node:path";
export interface RepositoryFileReceipt { path: string; sha: string; size: number; executable: boolean }
export interface RepositoryBundleReceipt { root: string; wrapper: string; files: RepositoryFileReceipt[] }
export interface LocalRepositorySessionOwner {
companyId: string; agentId: string; driverKind: string;
workspace: { cwd: string; repoUrl: string | null; repoRef: string | null; branchName: string | null };
}
export function repositoryRuntimeSessionRoot(instanceRoot: string, sessionId: string, owner: LocalRepositorySessionOwner): string {
// Local projectless native runs use the canonical v2 actor/workspace scope;
// Daytona's guest directory instead uses the normalized session ID alone.
const canonical = (value: any): string => value && typeof value === "object"
? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(",")}}`
: JSON.stringify(value);
const scope = { schema: "paperclip.native-session-scope.v2", companyId: owner.companyId, agentId: owner.agentId,
workspace: { kind: "transient", ...owner.workspace }, provider: { driverKind: owner.driverKind, identity: { kind: "codex" } }, normalizedSessionId: sessionId };
return path.join(instanceRoot, "runtime", "paperclip-runner", "durable-sessions", createHash("sha256").update(canonical(scope)).digest("hex"));
}
/** Bind disk locations to the active native session, rather than library caches. */
export function isOwnedRepositoryRuntimeRoot(root: string, sessionId: string, location:
{ kind: "local"; instanceRoot: string; owner: LocalRepositorySessionOwner; contextSkills?: { sessionId: string; roots: string[] } } | { kind: "daytona"; remoteCwd: string }): boolean {
if (!sessionId || path.posix.normalize(root) !== root) return false;
const digest = createHash("sha256").update(sessionId).digest("hex");
const parent = path.posix.dirname(root);
if (location.kind === "local") return parent === path.join(repositoryRuntimeSessionRoot(location.instanceRoot, sessionId, location.owner), "codex-home", "skills")
|| (location.contextSkills?.sessionId === sessionId && location.contextSkills.roots.includes(root)
&& parent === path.join(location.instanceRoot, "runtime-context-assets", "bundles"));
const filesystem = path.posix.join(location.remoteCwd, ".paperclip-runtime", "paperclip-runner", "sessions", digest, "filesystem");
return parent === path.posix.join(filesystem, "codex-home", "skills")
|| parent === path.posix.join(filesystem, "context", "skills");
}
/** Independent disk evidence; never follows imported links or reads provider auth. */
export function inspectRepositoryBundle(root: string): RepositoryBundleReceipt {
const canonical = path.join(root, ".paperclip-repository");
if (lstatSync(canonical).isSymbolicLink() || realpathSync(canonical) !== canonical) throw new Error("Repository root is not a regular directory");
const files: RepositoryFileReceipt[] = [];
function walk(directory: string) {
for (const name of readdirSync(directory).sort()) {
const file = path.join(directory, name), stat = lstatSync(file);
if (stat.isDirectory()) walk(file);
else {
if (!stat.isFile()) throw new Error("Non-regular repository file");
const bytes = readFileSync(file);
const sha = createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");
files.push({ path: path.relative(canonical, file).split(path.sep).join("/"), sha, size: bytes.length, executable: Boolean(stat.mode & 0o111) });
}
}
}
walk(canonical);
return { root, wrapper: readFileSync(path.join(root, "SKILL.md"), "utf8"), files };
}
/** Equivalent read-only inspector for the owned Linux sandbox. */
export function remoteRepositoryInspector(roots: string[]): string {
const source = `const fs=require('node:fs'),path=require('node:path'),crypto=require('node:crypto');
function inspect(root) {
root=fs.realpathSync(root);
const canonical=path.join(root,'.paperclip-repository'),files=[];
if(fs.lstatSync(canonical).isSymbolicLink()||fs.realpathSync(canonical)!==canonical) throw Error('Invalid repository root');
function walk(directory) { for(const name of fs.readdirSync(directory).sort()) {
const file=path.join(directory,name),stat=fs.lstatSync(file);
if(stat.isDirectory()) walk(file);
else { if(!stat.isFile()) throw Error('Non-regular repository file');
const bytes=fs.readFileSync(file);
files.push({path:path.relative(canonical,file).split(path.sep).join('/'),sha:crypto.createHash('sha1').update('blob '+bytes.length+'\\0').update(bytes).digest('hex'),size:bytes.length,executable:Boolean(stat.mode&0o111)});
}
} }
walk(canonical);
return {root,wrapper:fs.readFileSync(path.join(root,'SKILL.md'),'utf8'),files};
}
process.stdout.write(JSON.stringify(${JSON.stringify(roots)}.map(inspect)));`;
return `node -e 'eval(Buffer.from("${Buffer.from(source).toString("base64")}","base64").toString())'`;
}
export function gradeRepositoryBundles(bundles: RepositoryBundleReceipt[], expected: RepositoryFileReceipt[], skills: string[]) {
const sorted = (files: RepositoryFileReceipt[]) => [...files].sort((a, b) => a.path.localeCompare(b.path));
return [
{ id: "two-distinct-runtime-packages", passed: bundles.length === skills.length && new Set(bundles.map(bundle => bundle.root)).size === skills.length },
...skills.map(skill => ({ id: `canonical-${skill}`, passed: bundles.filter(bundle => bundle.wrapper.includes(`.paperclip-repository/skills/${skill}/SKILL.md`)).length === 1 })),
{ id: "complete-original-repository-bytes-and-modes", passed: bundles.length === skills.length && bundles.every(bundle => JSON.stringify(sorted(bundle.files)) === JSON.stringify(sorted(expected))) },
];
}
+169
View File
@@ -0,0 +1,169 @@
import { expect, type Page } from "@playwright/test";
import { realpath } from "node:fs/promises";
import path from "node:path";
import { parseNativeExecutionInput } from "../../packages/paperclip-runner/src/contracts/native-execution.js";
import { pollUntil, type RunnerApi } from "./api.js";
import { nativeCompletionProfile } from "./native-completion-defaults.js";
import { createRemoteFixtureClient } from "./remote-native-bootstrap.js";
import { gradeRepositoryBundles, inspectRepositoryBundle, isOwnedRepositoryRuntimeRoot, remoteRepositoryInspector, type RepositoryBundleReceipt, type RepositoryFileReceipt } from "./repository-skill-evidence.js";
import { createTaskThroughUi } from "./user-actions.js";
import type { LiveFixtureValues } from "./live-fixtures.js";
import type { CredentialName, MatrixExecution, RunnerTaskFixture } from "./types.js";
type Row = Record<string, any>;
export { nativeCompletionProfile as repositorySkillsProfile };
export const POTETO_REPOSITORY = "https://github.com/paperclipai/poteto-stack";
const SKILLS = ["architect", "bro"];
export const repositorySkillsTask: RunnerTaskFixture = {
id: "poteto-package", label: "Private Poteto repository package", flow: "repository_skills",
groups: [], requiredCredentials: ["GITHUB_TOKEN"], workMode: "standard",
expectedRunCount: 1, automaticRetryPolicy: "single_attempt",
attemptTimeoutMs: { local: 600_000, daytona: 900_000 },
expectedTerminalState: { issue: "done", run: "succeeded" },
buildTitle: nonce => `Verify Poteto skill delivery ${nonce}`,
buildPrompt: () => "Read the assigned Poteto repository skills and verify their delivered files.",
buildVisibleMarker: nonce => `POTETO-VERIFIED-${nonce}`,
buildMatchers: () => [],
};
export async function runRepositorySkillsFlow(input: {
page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string;
credentials: Partial<Record<CredentialName, string>>; secrets: string[]; deadlineAt: number;
observe(issue: Row, runs: Row[]): void;
capture(id: string, label: string, file: string): Promise<void>;
evidence(name: string, data: unknown): Promise<void>;
}) {
const { api, page, fixtures, nonce, execution } = input;
const company = `/api/companies/${fixtures.company.id}`;
await api.patch("/api/instance/settings/experimental", { enableBetaSkills: true, enableClassicTaskInterface: false });
await api.patch(`${company}/budgets`, { budgetMonthlyCents: 1_000 });
const github = await api.post<Row>(`${company}/tools/connections`, {
name: `Repository E2E GitHub ${nonce}`, applicationName: "GitHub", transport: "mcp_remote", authKind: "api_key",
credentialPolicy: "shared", status: "active", enabled: true,
config: { sourceTemplateKey: "github", url: "https://api.githubcopilot.com/mcp/" }, transportConfig: { sourceTemplateKey: "github", url: "https://api.githubcopilot.com/mcp/" },
credentialRefs: [{ name: "GitHub", secretId: fixtures.secretRefs.GITHUB_TOKEN!.secretId, version: "latest", placement: "header", key: "Authorization", prefix: "Bearer " }],
credentialSecretRefs: [{ secretId: fixtures.secretRefs.GITHUB_TOKEN!.secretId, versionSelector: "latest", configPath: "headers.Authorization", required: true }],
});
await page.goto(`/${fixtures.company.issuePrefix}/skills/sources/new`);
const dialog = page.getByRole("dialog", { name: "Import from GitHub", exact: true });
await expect(dialog).toBeVisible();
await dialog.getByRole("button", { name: "... or add public repo by URL", exact: true }).click();
await dialog.getByRole("textbox", { name: "Repository URL", exact: true }).fill(POTETO_REPOSITORY);
await dialog.getByRole("button", { name: "Find skills", exact: true }).click();
await expect(dialog.getByRole("button", { name: /View \d+ files?/ })).toBeVisible({ timeout: 120_000 });
await expect(dialog.getByRole("button", { name: "Import repository", exact: true })).toBeEnabled({ timeout: 120_000 });
await expect(dialog.getByRole("checkbox", { name: "Import folder Repository", exact: true })).toBeChecked();
await expect(dialog.getByRole("button", { name: "More import options", exact: true })).toHaveCount(0);
// Deselect every default; exactly two become discoverable, all siblings stay supporting files.
const all = dialog.getByRole("checkbox", { name: "Import folder Repository", exact: true });
if (await all.isChecked()) await all.uncheck();
else { await all.check(); await all.uncheck(); }
for (const skill of SKILLS) await dialog.getByRole("checkbox", { name: `Import skills/${skill}/SKILL.md`, exact: true }).check();
const importedResponse = page.waitForResponse(response => response.request().method() === "POST" && new URL(response.url()).pathname === `${company}/skill-sources`, { timeout: 120_000 });
await dialog.getByRole("button", { name: "Import repository", exact: true }).click();
const response = await importedResponse;
expect(response.ok()).toBe(true);
const imported = await response.json() as Row;
expect(imported.imported).toHaveLength(2);
expect(imported.source.packageMode).toBe("repository");
const commit = imported.source.lastScanCommit as string;
const treeResponse = await fetch(`https://api.github.com/repos/paperclipai/poteto-stack/git/trees/${commit}?recursive=1`, {
headers: { Authorization: `Bearer ${input.credentials.GITHUB_TOKEN}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000),
});
if (!treeResponse.ok) throw new Error(`Independent GitHub tree read failed (${treeResponse.status})`);
const tree = await treeResponse.json() as { truncated: boolean; tree: Array<{ path: string; type: string; mode: string; sha: string; size: number }> };
expect(tree.truncated).toBe(false);
const expected: RepositoryFileReceipt[] = tree.tree.filter(file => file.type === "blob").map(file => ({ path: file.path, sha: file.sha, size: file.size, executable: file.mode === "100755" }));
expect(expected.some(file => file.path === "agents/poteto-agent.md")).toBe(true);
expect(expected.some(file => file.path === "NOTICE.md")).toBe(true);
await api.post(`/api/agents/${fixtures.agent.id}/skills/sync?companyId=${fixtures.company.id}`, { desiredSkills: imported.imported.map((skill: Row) => skill.key), mode: "replace" });
const assigned = await api.get<Row>(`/api/agents/${fixtures.agent.id}/skills?companyId=${fixtures.company.id}`);
expect(assigned.desiredSkills).toHaveLength(2);
// Source credentials are disabled before the provider starts: delivery must use persisted bytes.
await api.patch(`/api/tool-connections/${github.id}`, { enabled: false });
await input.evidence("repository-import.json", { commit, source: imported.source, skills: imported.imported, assigned, expected });
const names = imported.imported.map((skill: Row) => skill.slug);
const prompt = [
`Read both assigned skills ${names.map((name: string) => `/${name}`).join(" and ")} using the skill mechanism. Follow each wrapper to its canonical original SKILL.md. This task only verifies their installation; do not perform architecture changes.`,
"Use the filesystem paths provided for your assigned skills to read the two selected wrappers on this provider's filesystem. Read their canonical original instructions, then use Node.js fs to traverse each .paperclip-repository and verify that sibling skills, agents/poteto-agent.md and NOTICE.md can be read. Use the delivered provider home or prepared runtime-context bundle, not an ordinary company skill-library directory; CODEX_HOME may be absent from the shell environment.",
`Publish a progress comment on this task containing exactly POTETO-ROOTS-${nonce} followed by JSON {"roots":["<absolute provider wrapper parent for first skill>","<absolute provider wrapper parent for second skill>"]}.`,
"After publishing that comment, execute the ordinary shell command sleep 120 and wait for it to finish. The operator will independently inspect these files during that pause. Do not alter any imported file, install dependencies, fetch repository contents, or finish before the pause completes.",
`Then reply ${execution.task.buildVisibleMarker(nonce)} and complete this task.`,
].join("\n");
const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name,
title: execution.task.buildTitle(nonce), prompt, workMode: "standard", requireExplicitTitle: true });
let issue = await api.get<Row>(`/api/issues/${created.issueId}`), runs: Row[] = [];
const refresh = async () => {
issue = await api.get<Row>(`/api/issues/${created.issueId}`);
const listed = await api.get<Row[]>(`/api/issues/${created.issueId}/runs`);
runs = await Promise.all(listed.map(row => api.get<Row>(`/api/heartbeat-runs/${row.runId}`)));
input.observe(issue, runs);
};
const receiptMarker = `POTETO-ROOTS-${nonce}`;
const comments = await pollUntil({ label: "provider read repository skills", deadlineAt: input.deadlineAt - 150_000,
load: async () => { await refresh(); return api.get<Row[]>(`/api/issues/${created.issueId}/comments`); },
accept: rows => rows.some(row => row.authorAgentId === fixtures.agent.id && row.createdByRunId === runs[0]?.id && row.body?.startsWith(receiptMarker)),
reject: () => runs.some(run => ["failed", "cancelled", "timed_out", "succeeded"].includes(run.status)) ? "Repository delivery provider ended without a runtime receipt" : undefined,
});
const proof = comments.find(row => row.authorAgentId === fixtures.agent.id && row.createdByRunId === runs[0]?.id && row.body?.startsWith(receiptMarker))!;
const roots = JSON.parse(proof.body.slice(receiptMarker.length).trim()).roots as string[];
expect(roots).toHaveLength(2);
expect(roots.every(root => typeof root === "string" && root.startsWith("/") && !root.split("/").some(part => part === ".."))).toBe(true);
expect(runs).toHaveLength(1);
expect(runs[0]!.status).toBe("running");
expect(runs[0]!.runtimeMode).toBe(execution.profile.expectedRuntimeMode);
expect(runs[0]!.companyId).toBe(fixtures.company.id);
expect(runs[0]!.agentId).toBe(fixtures.agent.id);
expect(issue.companyId).toBe(fixtures.company.id);
expect(issue.assigneeAgentId).toBe(fixtures.agent.id);
expect(runs[0]!.contextSnapshot?.paperclipEnvironment?.id).toBe(fixtures.environment.id);
expect(typeof runs[0]!.nativeSessionId).toBe("string");
let bundles: RepositoryBundleReceipt[];
let runtimeLocation: Parameters<typeof isOwnedRepositoryRuntimeRoot>[2];
if (execution.environment.id === "local") {
const temporaryRoot = await realpath(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT!);
for (const root of roots) expect((await realpath(root)).startsWith(`${temporaryRoot}${path.sep}`)).toBe(true);
bundles = (await Promise.all(roots.map(root => realpath(root)))).map(inspectRepositoryBundle);
const instanceRoot = await realpath(path.dirname(process.env.PAPERCLIP_CONFIG!));
const sessionId = runs[0]!.nativeSessionId as string;
// Read the active run's persisted, digest-validated execution context. These
// immutable assets are normal provider inputs, not a fixture or library copy.
const executionInput = parseNativeExecutionInput(runs[0]!.runnerProfileJson.nativeExecutionInput);
expect(executionInput.binding).toMatchObject({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runId: runs[0]!.id, issueId: issue.id, executionWorkspaceId: runs[0]!.id });
expect(executionInput.session.normalizedSessionId).toBe(sessionId);
expect(executionInput.provider.kind).toBe("codex");
if (!("runtimeContext" in executionInput) || "access" in executionInput.workspace) throw new Error("Missing local repository runtime context");
const context = executionInput.runtimeContext;
expect(context.skills.map(skill => skill.key).sort()).toEqual(imported.imported.map((skill: Row) => skill.key).sort());
runtimeLocation = { kind: "local", instanceRoot, owner: { companyId: executionInput.binding.companyId, agentId: executionInput.binding.agentId,
driverKind: executionInput.session.driverKind, workspace: executionInput.workspace }, contextSkills: { sessionId, roots: context.skills.map(skill => skill.bundle.rootPath) } };
await input.evidence("repository-owned-context.json", { nativeSessionId: sessionId, aggregateDigest: context.aggregateDigest,
skills: context.skills.map(skill => ({ key: skill.key, versionId: skill.versionId, root: skill.bundle.rootPath, digest: skill.bundle.digest })) });
} else {
const leases = await api.get<Row[]>(`/api/environments/${fixtures.environment.id}/leases`);
const owned = leases.filter(lease => lease.companyId === fixtures.company.id && lease.environmentId === fixtures.environment.id && lease.heartbeatRunId === runs[0]!.id && lease.issueId === issue.id && lease.status === "active" && lease.provider === "daytona");
expect(owned).toHaveLength(1);
expect(owned[0]!.metadata.agentId).toBe(fixtures.agent.id);
runtimeLocation = { kind: "daytona", remoteCwd: owned[0]!.metadata.remoteCwd };
const client = await createRemoteFixtureClient(input.credentials.DAYTONA_API_KEY!);
const sandbox = await client.get(owned[0]!.providerLeaseId);
expect(sandbox.id).toBe(owned[0]!.providerLeaseId);
const result = await sandbox.process.executeCommand(remoteRepositoryInspector(roots), undefined, {}, 20);
expect(result.exitCode).toBe(0);
bundles = JSON.parse(result.result);
await input.evidence("repository-owned-sandbox.json", { lease: owned[0], sandboxId: sandbox.id });
}
const checks = gradeRepositoryBundles(bundles, expected, SKILLS);
checks.push({ id: "owned-native-session-skill-directories", passed: typeof runs[0]!.nativeSessionId === "string" && bundles.every(bundle =>
isOwnedRepositoryRuntimeRoot(bundle.root, runs[0]!.nativeSessionId, runtimeLocation)) });
await input.evidence("repository-runtime-files.json", { commit, bundles, checks });
expect(checks.filter(check => !check.passed)).toEqual([]);
await pollUntil({ label: "repository task completes", deadlineAt: input.deadlineAt, load: async () => { await refresh(); return { issue, runs }; },
accept: state => state.issue.status === "done" && state.runs.length === 1 && state.runs[0]!.status === "succeeded",
reject: state => state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Repository task failed" : undefined });
await input.evidence("api-state.json", { issue, run: runs[0], runs, comments: await api.get<Row[]>(`/api/issues/${created.issueId}/comments`), checks });
await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`);
await expect(page.getByTestId("task-chat-agent-bubble").filter({ hasText: execution.task.buildVisibleMarker(nonce) }).last()).toBeVisible();
await input.capture("final-state", "Poteto repository skills verified in the actual agent runtime", "final-state.png");
return { issue, runs, checks };
}
+12 -3
View File
@@ -30,6 +30,7 @@ import { runContinuationFlow } from "./continuation-flow.js";
import { runEverydayFlow } from "./everyday-flow.js";
import { gradeTaskTitle } from "./task-titles.js";
import { runContextIntegrityFlow } from "./context-integrity-flow.js";
import { runRepositorySkillsFlow } from "./repository-skills.js";
import { captureStockHarness, gradeStockHarness, gradeStockHire } from "./stock-harness.js";
import { verifyStockHarnessPreflight, STOCK_PREFLIGHT_ENV } from "./stock-harness-admission.js";
import { createTaskThroughUi, submitTaskReply } from "./user-actions.js";
@@ -593,7 +594,7 @@ for (const execution of executions) {
const credentials = credentialValues();
const secrets = normalizedSecrets(Object.values(credentials));
const api = new RunnerApi(request);
const companyRunFlow = execution.suite.id === "task-titles" || ["plan_task_guidance", "blocker_guidance", "continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "pi_controls", "copilot_protection", "cursor_native", "native_active_stop", "native_provider_loss", "public_mcp"].includes(execution.task.flow);
const companyRunFlow = execution.suite.id === "task-titles" || ["plan_task_guidance", "blocker_guidance", "continuation_accounting", "continuation", "context_integrity", "repository_skills", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "pi_controls", "copilot_protection", "cursor_native", "native_active_stop", "native_provider_loss", "public_mcp"].includes(execution.task.flow);
const publicMcpUsage = execution.task.flow === "public_mcp" ? assistantUsage(execution.profile.provider === "claude" ? "anthropic" : "openai", execution.profile.model) : undefined;
let publicMcpUserId = "";
const consoleDiagnostics: Array<Record<string, unknown>> = [];
@@ -1120,6 +1121,14 @@ for (const execution of executions) {
});
issue = story.issue; selectedRuns = story.runs;
matcherResults = story.evidence.checks.map(check => ({ matcher: { kind: "json_path" as const, path: check.id, expected: true }, passed: check.passed, detail: check.detail }));
} else if (execution.task.flow === "repository_skills") {
const story = await runRepositorySkillsFlow({ page, api, fixtures, execution, nonce, credentials, secrets,
deadlineAt: startedAtMs + deadlineMs, capture: captureScreenshot,
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
observe: (currentIssue, currentRuns) => { issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; },
});
issue = story.issue as IssueRecord; selectedRuns = story.runs as RunRecord[];
matcherResults = story.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `repositorySkills.${check.id}`, expected: true }, passed: check.passed, detail: check.id }));
} else if (execution.task.flow === "context_integrity") {
const contextIntegrity = await runContextIntegrityFlow({
page, api, fixtures, execution, nonce, deadlineAt: startedAtMs + deadlineMs,
@@ -2327,7 +2336,7 @@ for (const execution of executions) {
if (registeredArtifacts.length > 0) {
await writeSanitizedJson(snapshotsDir, "downloaded-registered-artifacts.json", registeredArtifacts, secrets);
}
matcherResults = await Promise.all(
matcherResults.push(...await Promise.all(
taskMatchers.map((matcher) =>
evaluateMatcher(matcher, {
...matcherObservation,
@@ -2341,7 +2350,7 @@ for (const execution of executions) {
matcher.kind === "message_exact" ? finalRunMessage : message,
}),
),
);
));
const failedMatchers = matcherResults.filter((result) => !result.passed);
const exactMessageMatcher = taskMatchers.find(
(matcher) => matcher.kind === "message_exact",
@@ -10,7 +10,6 @@ import { readStockInstructionVariant } from "./stock-harness-instruction-variant
const root = resolve(import.meta.dirname, "../..");
export const stockHarnessGates = [
{ id: "SH-1", name: "Native Codex additive instructions", cwd: "packages/paperclip-runner", files: [
"src/drivers/codex/codex-app-server-driver.test.ts",
"src/drivers/codex/codex-app-server-driver.lifecycle.test.ts",
"src/live/runnerd-codex-transport.test.ts",
"src/live/live-session.test.ts",
+4
View File
@@ -922,6 +922,8 @@ describe("runner E2E server isolation", () => {
XAI_API_KEY: "xai",
GROK_AUTH_JSON: "grok-auth-json",
DAYTONA_API_KEY: "daytona",
GITHUB_TOKEN: "github-private-repository",
GH_TOKEN: "github-cli",
XAI_ORG_ID: "xai-sensitive",
GROK_HOME: "/outside/grok",
OPENAI_ORG_ID: "also-provider-sensitive",
@@ -946,6 +948,8 @@ describe("runner E2E server isolation", () => {
expect(env.PATH).toBe("/bin");
expect(env.DATABASE_URL).toBeUndefined();
expect(env.OPENAI_API_KEY).toBeUndefined();
expect(env.GITHUB_TOKEN).toBeUndefined();
expect(env.GH_TOKEN).toBeUndefined();
expect(env.KIMI_MODEL_API_KEY).toBeUndefined();
expect(env.XAI_API_KEY).toBeUndefined();
expect(env.GROK_AUTH_JSON).toBeUndefined();
+3
View File
@@ -7,6 +7,7 @@ export const CREDENTIAL_NAMES = [
"GROK_AUTH_JSON",
"GEMINI_API_KEY",
"DAYTONA_API_KEY",
"GITHUB_TOKEN",
"CURSOR_AUTH_TOKEN",
"COPILOT_GITHUB_TOKEN",
] as const;
@@ -16,6 +17,7 @@ export type RunnerGeneration = "legacy" | "native";
export type RunnerEnvironmentId = "local" | "daytona";
export type RunnerTaskWorkMode = "standard" | "planning" | "ask";
export type RunnerTaskFlow =
| "repository_skills"
| "provider_connection"
| "plan_task_guidance"
| "blocker_guidance"
@@ -141,6 +143,7 @@ export type Matcher =
| { kind: "json_schema"; schema: Record<string, unknown> };
export interface RunnerTaskFixture {
requiredCredentials?: readonly CredentialName[];
id: string;
label: string;
groups: readonly string[];