PaperClipAI/packages/paperclip-runner/scripts/grok-native-smoke.mjs

import assert from 'node:assert/strict';
import { randomUUID, createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { readFile, writeFile, mkdir, mkdtemp, rm, realpath } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { basename, dirname, join, resolve } from 'node:path';
import { parseArgs } from 'node:util';
import { createRequire } from 'node:module';
import { AcpxRuntimeHost } from '../dist/drivers/acpx/runtime-host.js';
import { openQualifiedAcpxRuntime } from '../dist/drivers/acpx/acpx-runtime-adapter.js';
import { acpxProviderSessionIdentity } from '../dist/drivers/acpx/recovery-identity.js';
import { QUALIFIED_ACPX_PROFILES } from '../dist/drivers/acpx/qualified-profiles.js';

const { values } = parseArgs({ options: { auth: { type: 'string', default: 'subscription' }, 'auth-file': { type: 'string' }, output: { type: 'string' }, repetitions: { type: 'string', default: '3' } } });
assert(values.output, 'An output path is required to retain every attempt');
assert(['subscription', 'api'].includes(values.auth));
const repetitions = Number(values.repetitions);
assert(Number.isSafeInteger(repetitions) && repetitions > 0 && repetitions <= 3);
const environment = { PATH: process.env.PATH };
let copyBackGrokAuth;
if (values.auth === 'api') {
  assert(process.env.XAI_API_KEY, 'Explicit XAI_API_KEY is required');
  environment.XAI_API_KEY = process.env.XAI_API_KEY;
} else {
  assert(values['auth-file'], 'An explicit subscription auth-file is required');
  assert.equal(basename(values['auth-file']), 'auth.json', 'Subscription copyback requires an auth.json source');
  const serverRequire=createRequire(new URL('../../../server/package.json',import.meta.url));
  const { tsImport } = await import(serverRequire.resolve('tsx/esm/api'));
  ({ copyBackGrokAuth } = await tsImport('../../adapters/grok-local/src/server/grok-auth-copyback.ts', import.meta.url));
  environment.PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET = await readFile(values['auth-file'], 'utf8');
}
const report = { schema: 'paperclip.grok-native-smoke.v1', auth: values.auth, sourceRevision: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), sourceDirty: execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).length > 0, profile: QUALIFIED_ACPX_PROFILES.grok, platform: `${process.platform}-${process.arch}`, startedAt: new Date().toISOString(), attempts: [] };
const binary = await readFile('/opt/paperclip/providers/grok/1.0.13/grok');
report.binaryDigest = `sha256:${createHash('sha256').update(binary).digest('hex')}`;
async function persist() { await writeFile(resolve(values.output), JSON.stringify(report, null, 2)+'\n', { mode: 0o600 }); }
await persist();
for (let repetition=1; repetition<=repetitions; repetition++) {
  const root = await realpath(await mkdtemp(join(tmpdir(), 'paperclip-grok-smoke-')));
  await mkdir(join(root,'runtime'), { mode:0o700 }); await mkdir(join(root,'workspace'), { mode:0o700 });
  const nonce=randomUUID(); let calls=0; let host;
  const attempt = { repetition, startedAt: new Date().toISOString(), status: 'running', checks: {}, usage: null, cost: null };
  report.attempts.push(attempt); await persist();
  const start = Date.now();
  async function phase(value) { attempt.phase=value; await persist(); }
  const options = { runtimeDirectory:join(root,'runtime'), normalizedSessionId:'grok-smoke', workingDirectory:join(root,'workspace'), agent:'grok', model:'grok-4.7', permissionMode:'approve-all', environment, systemInstructions:'Use the requested MCP tool and retain its nonce in this session. Do not use other tools. Answer briefly.', semanticTools: { tools:[{ name:'check_context', description:'Read this task’s qualification nonce.', inputSchema:{type:'object', properties:{}, additionalProperties:false} }], handler:async call=>{assert.equal(call.tool,'check_context'); calls++; return {nonce};} } };
  const dependencies={openRuntime:openQualifiedAcpxRuntime, reportRetainedCleanupFailure:()=>{ attempt.checks.cleanup=false; }};
  async function closeHost(reason) {
    const runtimeRoot=host.binding().runtimeRoot;
    await host.close({reason});
    if(copyBackGrokAuth) {
      await copyBackGrokAuth({hostHomeDir:dirname(resolve(values['auth-file'])),readSandboxAuth:()=>readFile(join(runtimeRoot,'grok-home','auth-refresh.json')),log:()=>{}});
      environment.PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET=await readFile(values['auth-file'],'utf8');
    }
    host=null;
  }
  async function turn(text,id) {
    const handle=host.startTurn({text, requestId:id, signal:AbortSignal.timeout(120000)}); let output=''; const counts={};
    const drain=(async()=>{for await(const event of handle.events){counts[event.type]=(counts[event.type]??0)+1;if(event.type==='text_delta' && event.stream!=='thought')output+=event.text;}})();
    const [result]=await Promise.all([handle.result,drain]);attempt.toolCalls=calls;attempt.providerFailureReason=/rate.?limit|quota/i.test(result.error?.message??'')?'rate_limited':/auth|token|login|credential/i.test(result.error?.message??'')?'authentication':result.error?'other_runtime':null;attempt.lastTurnStatus=result.status;attempt.stopReason=result.stopReason;attempt.failureCode=result.error?.code??null;attempt.failureCategory=result.error?.category??null; attempt.lastEventCounts=counts;attempt.outputLength=output.length;if(result.status!=='completed') throw Object.assign(new Error('Provider turn did not complete'),{code:result.error?.code ?? result.status});return {output,counts};
  }
  try {
    await phase('initialize');
    host=await AcpxRuntimeHost.open({...options,signal:AbortSignal.timeout(45000)},dependencies);
    const identity=acpxProviderSessionIdentity(host.identity(),host.binding()); assert.equal(identity.effectiveModel,'grok-4.7');attempt.checks.model=true;
    await phase('authenticated-tool');
    const first=await turn('Call check_context exactly once, then reply with only the returned nonce.',`tool-${repetition}`);
    attempt.toolCalls=calls;attempt.nonceObserved=first.output.includes(nonce);assert.equal(calls,1);assert(first.output.includes(nonce));attempt.checks.authenticatedTool=true;attempt.events=first.counts;
    await phase('resume');
    await closeHost('verify persisted resume');
    host=await AcpxRuntimeHost.open({...options,expectedIdentity:identity,signal:AbortSignal.timeout(45000)},dependencies);
    assert.equal(host.identity().agentSessionId,identity.agentSessionId);
    const second=await turn('Without using tools, reply with only the nonce returned in the previous turn.',`resume-${repetition}`);
    assert(second.output.includes(nonce));assert.equal(calls,1);attempt.checks.resumeIdentity=true;
    await phase('permissions');
    await closeHost('verify restrictive permissions');
    await mkdir(join(root,'workspace','.grok'));
    await writeFile(join(root,'workspace','.grok','config.toml'),'[permission]\nallow = ["Bash", "Edit"]\n');
    await mkdir(join(root,'workspace','.claude'));
    await writeFile(join(root,'workspace','.claude','settings.json'),JSON.stringify({permissions:{defaultMode:'bypassPermissions',allow:['Bash','Edit']}}));
    const restricted={...options,normalizedSessionId:'grok-restricted',permissionMode:'approve-reads',systemInstructions:'Follow the requested action using native tools and respect permission decisions.'};
    host=await AcpxRuntimeHost.open({...restricted,signal:AbortSignal.timeout(45000)},dependencies);
    let restrictedResult;
    try { await turn('Use the shell to create a file named restricted-proof.txt in the working directory, then stop.',`permission-${repetition}`); }
    catch(error) { restrictedResult=error.code ?? error.name; attempt.permissionOutcome=restrictedResult; }
    // Restricted Grok writes must route through the existing approval-required outcome.
    assert.equal(restrictedResult,'approval_required');
    await assert.rejects(readFile(join(root,'workspace','restricted-proof.txt')),{code:'ENOENT'});
    attempt.checks.permissionBoundary=true;
    attempt.status='passed';
  } catch(error) { attempt.status='failed'; attempt.errorCode=error.code??error.name; /* Raw errors and provider text stay out of this sanitized proof. */ }
  finally { if(host) {try{await closeHost('qualification cleanup');}catch{attempt.status='failed';attempt.checks.cleanup=false;}} await rm(root,{recursive:true,force:true}); if(attempt.checks.cleanup!==false) attempt.checks.cleanup=true; attempt.elapsedMs=Date.now()-start;await persist(); }
  console.log(`Grok ${values.auth} repetition ${repetition}: ${attempt.status}`);
}
report.finishedAt=new Date().toISOString(); await persist();
process.exitCode=report.attempts.every(attempt=>attempt.status==='passed')?0:1;