diff --git a/doc/grok-native-runner.md b/doc/grok-native-runner.md index 5cc10d8daee..584f149ac26 100644 --- a/doc/grok-native-runner.md +++ b/doc/grok-native-runner.md @@ -8,7 +8,9 @@ Grok Build speaks [ACP over stdio](https://github.com/xai-org/grok-build/blob/ma The runner owns `grok agent --no-leader stdio` through ACPX, including session identity, cancellation, recovery and the authenticated Paperclip MCP bridge. It does not add `--always-approve`. Restricted operations use the selected ACPX -permission policy and return the existing approval-required outcome. +permission policy and return the existing approval-required outcome. Isolated ask +rules override project allow rules, and compatible always-approve settings are +locked off. Compatible hook/MCP discovery and shell login capture are disabled. ## Installation and identity @@ -46,7 +48,8 @@ lane remains disabled pending complete live qualification. Product E2E exposes `runner-acpx-grok` in core local/Daytona compatibility and local session integrity. The explicit `grok-qualification` suite covers replies, -planning approvals, structured questions and continuation after controller restart +planning approvals, structured questions, downloadable project revisions, stop/resume +and continuation after controller restart in both environments. Run with `--suite grok-qualification`; it is excluded from scheduled `--all`. Use the canonical Product E2E dashboard and Evalbook reports. diff --git a/packages/paperclip-runner/scripts/grok-native-smoke.mjs b/packages/paperclip-runner/scripts/grok-native-smoke.mjs index 9a0a5952cbd..5d4dcfe4f99 100644 --- a/packages/paperclip-runner/scripts/grok-native-smoke.mjs +++ b/packages/paperclip-runner/scripts/grok-native-smoke.mjs @@ -3,8 +3,9 @@ import { randomUUID, createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { readFile, writeFile, mkdir, mkdtemp, rm, realpath } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { basename, dirname, join, resolve } from 'node:path'; import { parseArgs } from 'node:util'; +import { createRequire } from 'node:module'; import { AcpxRuntimeHost } from '../dist/drivers/acpx/runtime-host.js'; import { openQualifiedAcpxRuntime } from '../dist/drivers/acpx/codex-runtime-adapter.js'; import { acpxProviderSessionIdentity } from '../dist/drivers/acpx/recovery-identity.js'; @@ -16,11 +17,16 @@ assert(['subscription', 'api'].includes(values.auth)); const repetitions = Number(values.repetitions); assert(Number.isSafeInteger(repetitions) && repetitions > 0 && repetitions <= 3); const environment = { PATH: process.env.PATH }; +let copyBackGrokAuth; if (values.auth === 'api') { assert(process.env.XAI_API_KEY, 'Explicit XAI_API_KEY is required'); environment.XAI_API_KEY = process.env.XAI_API_KEY; } else { assert(values['auth-file'], 'An explicit subscription auth-file is required'); + assert.equal(basename(values['auth-file']), 'auth.json', 'Subscription copyback requires an auth.json source'); + const serverRequire=createRequire(new URL('../../../server/package.json',import.meta.url)); + const { tsImport } = await import(serverRequire.resolve('tsx/esm/api')); + ({ copyBackGrokAuth } = await tsImport('../../adapters/grok-local/src/server/grok-auth-copyback.ts', import.meta.url)); environment.PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET = await readFile(values['auth-file'], 'utf8'); } const report = { schema: 'paperclip.grok-native-smoke.v1', auth: values.auth, sourceRevision: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), sourceDirty: execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).length > 0, profile: QUALIFIED_ACPX_PROFILES.grok, platform: `${process.platform}-${process.arch}`, startedAt: new Date().toISOString(), attempts: [] }; @@ -38,6 +44,15 @@ for (let repetition=1; repetition<=repetitions; repetition++) { async function phase(value) { attempt.phase=value; await persist(); } const options = { runtimeDirectory:join(root,'runtime'), normalizedSessionId:'grok-smoke', workingDirectory:join(root,'workspace'), agent:'grok', model:'grok-4.7', permissionMode:'approve-all', environment, systemInstructions:'Use the requested MCP tool and retain its nonce in this session. Do not use other tools. Answer briefly.', semanticTools: { tools:[{ name:'check_context', description:'Read this task’s qualification nonce.', inputSchema:{type:'object', properties:{}, additionalProperties:false} }], handler:async call=>{assert.equal(call.tool,'check_context'); calls++; return {nonce};} } }; const dependencies={openRuntime:openQualifiedAcpxRuntime, reportRetainedCleanupFailure:()=>{ attempt.checks.cleanup=false; }}; + async function closeHost(reason) { + const runtimeRoot=host.binding().runtimeRoot; + await host.close({reason}); + if(copyBackGrokAuth) { + await copyBackGrokAuth({hostHomeDir:dirname(resolve(values['auth-file'])),readSandboxAuth:()=>readFile(join(runtimeRoot,'grok-home','auth-refresh.json')),log:()=>{}}); + environment.PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET=await readFile(values['auth-file'],'utf8'); + } + host=null; + } async function turn(text,id) { const handle=host.startTurn({text, requestId:id, signal:AbortSignal.timeout(120000)}); let output=''; const counts={}; const drain=(async()=>{for await(const event of handle.events){counts[event.type]=(counts[event.type]??0)+1;if(event.type==='text_delta' && event.stream!=='thought')output+=event.text;}})(); @@ -51,13 +66,17 @@ for (let repetition=1; repetition<=repetitions; repetition++) { const first=await turn('Call check_context exactly once, then reply with only the returned nonce.',`tool-${repetition}`); attempt.toolCalls=calls;attempt.nonceObserved=first.output.includes(nonce);assert.equal(calls,1);assert(first.output.includes(nonce));attempt.checks.authenticatedTool=true;attempt.events=first.counts; await phase('resume'); - await host.close({reason:'verify persisted resume'});host=null; + await closeHost('verify persisted resume'); host=await AcpxRuntimeHost.open({...options,expectedIdentity:identity,signal:AbortSignal.timeout(45000)},dependencies); assert.equal(host.identity().agentSessionId,identity.agentSessionId); const second=await turn('Without using tools, reply with only the nonce returned in the previous turn.',`resume-${repetition}`); assert(second.output.includes(nonce));assert.equal(calls,1);attempt.checks.resumeIdentity=true; await phase('permissions'); - await host.close({reason:'verify restrictive permissions'});host=null; + await closeHost('verify restrictive permissions'); + await mkdir(join(root,'workspace','.grok')); + await writeFile(join(root,'workspace','.grok','config.toml'),'[permission]\nallow = ["Bash", "Edit"]\n'); + await mkdir(join(root,'workspace','.claude')); + await writeFile(join(root,'workspace','.claude','settings.json'),JSON.stringify({permissions:{defaultMode:'bypassPermissions',allow:['Bash','Edit']}})); const restricted={...options,normalizedSessionId:'grok-restricted',permissionMode:'approve-reads',systemInstructions:'Follow the requested action using native tools and respect permission decisions.'}; host=await AcpxRuntimeHost.open({...restricted,signal:AbortSignal.timeout(45000)},dependencies); let restrictedResult; @@ -69,7 +88,7 @@ for (let repetition=1; repetition<=repetitions; repetition++) { attempt.checks.permissionBoundary=true; attempt.status='passed'; } catch(error) { attempt.status='failed'; attempt.errorCode=error.code??error.name; /* Raw errors and provider text stay out of this sanitized proof. */ } - finally { if(host) {try{await host.close({reason:'qualification cleanup'});}catch{attempt.status='failed';attempt.checks.cleanup=false;}} await rm(root,{recursive:true,force:true}); if(attempt.checks.cleanup!==false) attempt.checks.cleanup=true; attempt.elapsedMs=Date.now()-start;await persist(); } + finally { if(host) {try{await closeHost('qualification cleanup');}catch{attempt.status='failed';attempt.checks.cleanup=false;}} await rm(root,{recursive:true,force:true}); if(attempt.checks.cleanup!==false) attempt.checks.cleanup=true; attempt.elapsedMs=Date.now()-start;await persist(); } console.log(`Grok ${values.auth} repetition ${repetition}: ${attempt.status}`); } report.finishedAt=new Date().toISOString(); await persist(); diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts index cba482523a5..e3c7ffd1f3a 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts @@ -34,6 +34,24 @@ afterEach(async () => { }); describe("ACPX runtime sandbox", () => { + it("restores Grok permission controls on recovery without importing compatible hooks or credentials", async () => { + const fixture = await sandboxFixture("grok"); + const sandbox = await prepareAcpxRuntimeSandbox({ binding: fixture.binding, agent: "grok" }); + const configPath = join(sandbox.agentHomeDirectory, "config.toml"); + const requirementsPath = join(sandbox.agentHomeDirectory, "requirements.toml"); + await writeFile(configPath, '[permission]\nallow = ["Bash"]\n'); + await writeFile(requirementsPath, '[ui]\ndisable_bypass_permissions_mode = false\n'); + await prepareAcpxRuntimeSandbox({ binding: fixture.binding, agent: "grok" }); + const config = await readFile(configPath, "utf8"); + expect(config).toContain('ask = ["Bash", "Read", "Edit", "Grep", "MCPTool", "WebFetch", "WebSearch"]'); + expect(config).not.toContain('allow ='); + expect(config).toContain('login_shell_capture = false'); + expect(config).toContain('exclude = ["XAI_API_KEY"]'); + expect(config).toContain('[compat.claude]\nhooks = false\nmcps = false'); + expect(await readFile(requirementsPath, "utf8")).toContain('disable_bypass_permissions_mode = true'); + expect((await stat(requirementsPath)).mode & 0o777).toBe(0o600); + }); + it.each(["claude-sonnet-5", "sonnet", "custom-deployment-id"])( "preserves the requested Claude model %s in its isolated settings", async (model) => { @@ -318,13 +336,14 @@ describe("ACPX runtime sandbox", () => { ); }); -async function sandboxFixture(agent: "pi" | "claude" | "codex") { +async function sandboxFixture(agent: "pi" | "claude" | "codex" | "grok") { const root = await mkdtemp(join(tmpdir(), "paperclip-acpx-sandbox-")); temporaryDirectories.push(root); const workspace = join(root, "workspace"); const runtimeDirectory = join(root, "runtime"); await Promise.all([mkdir(workspace), mkdir(runtimeDirectory)]); const models = { + grok: "grok-4.7", pi: "openrouter/deepseek/deepseek-v4-flash-0731", claude: "claude-sonnet-5", codex: "gpt-5.6-sol", diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts index f8f622794f8..44782dae8e0 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts @@ -384,6 +384,27 @@ export async function prepareAcpxRuntimeSandbox(input: { })}\n`, ); } + if (input.agent === "grok") { + // Native project rules merge with global rules. Ask rules win over project + // allows, keeping operation approval at the runner's authenticated boundary. + await writePrivateFile(join(agentHomeDirectory, "config.toml"), [ + "[permission]", + 'ask = ["Bash", "Read", "Edit", "Grep", "MCPTool", "WebFetch", "WebSearch"]', + "[ui]", 'permission_mode = "ask"', + "[compat.claude]", "hooks = false", "mcps = false", + "[compat.cursor]", "hooks = false", "mcps = false", + "[managed_mcps]", "enabled = false", "gateway_tools_enabled = false", + "[toolset.bash]", "login_shell_capture = false", + "[shell_environment_policy]", 'exclude = ["XAI_API_KEY"]', + "[subagents]", "enabled = false", + "[workflows]", "enabled = false", + "[goal]", "enabled = false", "", + ].join("\n")); + // Always-approve may otherwise be enabled by compatible project settings. + // Even approve-all runs use one host decision per requested operation. + await writePrivateFile(join(agentHomeDirectory, "requirements.toml"), + "[ui]\ndisable_bypass_permissions_mode = true\n"); + } if (input.agent === "pi") { await writePrivateFile( join(agentHomeDirectory, "settings.json"), diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 1f1c53dd2a5..8fbe7c01426 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -71,10 +71,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 10, 23, 38, 52, 28, 18, 6, 6, 48, 16, 10, 2, + 16, 23, 38, 52, 28, 18, 6, 6, 48, 16, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(257); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(257); + expect(validateRunnerCatalog()).toHaveLength(263); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(263); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); @@ -550,6 +550,16 @@ describe("runner E2E selectors", () => { ]); }); + it("keeps Grok artifact and stop/resume qualification explicit in both environments", () => { + const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", "grok-qualification"])); + expect(selected).toHaveLength(16); + for (const environment of ["local", "daytona"]) { + for (const task of ["build-revise", "stop-new-resume", "continuity-restart"]) { + expect(selected.some(execution => execution.id === `grok-qualification.runner-acpx-grok.${environment}.${task}`)).toBe(true); + } + } + }); + it("selects a suite without exploding its environment matrix", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "openrouter-model-breadth"]), diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 8f54114ba67..70269d88043 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -919,12 +919,16 @@ const everydayProfiles = [ export const runnerSuites: readonly RunnerSuiteFixture[] = [ { id: "grok-qualification", label: "Grok Build Qualification", manualOnly: true, - description: "Grok task replies, planning approval, questions and controller restart in local and Daytona environments.", + description: "Grok replies, planning approval, questions, downloadable artifacts, stop/resume and controller restart in local and Daytona environments.", groups: ["native"], profiles: runnerProfiles.filter(profile => profile.id === "runner-acpx-grok"), environments: [localEnvironment, daytonaWarmEnvironment], - tasks: [...runnerTasks, ...localIntegrityTasks], expectedMatrixSize: 10, - definitionMetadata: { version: 1, binary: "1.0.13", model: "grok-4.7", scheduling: "explicit-only", repetitionsRequired: 3 }, + tasks: [ + ...runnerTasks, ...localIntegrityTasks, + ...everydayTasks.filter(task => task.id === "build-revise"), + ...chatHardeningTasks.filter(task => ["stop-new-resume", "continuity-restart"].includes(task.id)), + ], expectedMatrixSize: 16, + definitionMetadata: { version: 2, binary: "1.0.13", model: "grok-4.7", scheduling: "explicit-only", repetitionsRequired: 3, artifactOracle: "independent-python-contract", stopBoundary: "provider-turn-started" }, }, { id: "continuation", label: "Task continuation",