PaperClipAI/tests/speko-voice-proof/browser-client.mjs

import { VoiceConversation } from '@spekoai/client';
import { createCleanup, createAudioSave, createReportSaver } from './cleanup.mjs';
const config = JSON.parse(document.querySelector('#config').textContent);
const $ = (id) => document.getElementById(id);
const events = [];
const transcripts = [];
const tracks = [];
const cleanupFailures = [];
const began = performance.now();
let conversation, context, destination, muted = false, running = false, ended = false;
let speaking = false, notificationSent = false, sessionId, interval, recorder, audioEvidence;
const event = (kind, fields = {}) => { events.push({ kind, elapsedMs: Math.round(performance.now() - began), ...fields }); };
const status = (text) => { $('status').textContent = text; };
async function api(path, body = {}, signal) {
  const response = await fetch(path, { method: 'POST', headers: { 'content-type': 'application/json', 'x-proof-csrf': config.csrf }, body: JSON.stringify(body), signal });
  const data = await response.json();
  if (!response.ok) throw new Error(data.error ?? `Request failed (${response.status})`);
  return data;
}
const observer = new MutationObserver(() => {
  const stream = [...document.querySelectorAll('audio')].map((el) => el.srcObject).find(Boolean);
  if (!stream || recorder) return;
  recorder = new MediaRecorder(stream);
  const chunks = [];
  recorder.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data); };
  const recorded = new Promise((resolve, reject) => {
    recorder.onstop = () => resolve(new Blob(chunks, {type: 'audio/webm'}));
    recorder.onerror = () => reject(new Error('Recording failed'));
  });
  recorded.catch(() => {}); // Cleanup observes the original rejected promise.
  audioEvidence = createAudioSave({
    stop: async () => {
      if (recorder.state !== 'inactive') recorder.stop();
      let timer;
      try {
        return await Promise.race([recorded, new Promise((_, reject) => {
          timer = setTimeout(() => reject(new Error('Recording did not finish')), 10_000);
        })]);
      } finally { clearTimeout(timer); }
    },
    upload: async blob => {
      const response = await fetch('/audio', {method: 'POST', headers: {'content-type': 'audio/webm', 'x-proof-csrf': config.csrf}, body: blob, signal: AbortSignal.timeout(20_000)});
      if (!response.ok) throw new Error('Received audio could not be saved');
      event('received_audio_saved', {ok: true});
    },
  });
  recorder.start(); event('received_audio_recording_started');
});
observer.observe(document.body, { childList: true, subtree: true });
// This fixture replaces ONLY the microphone at the browser boundary. The
// production SDK and hosted STT/LLM/TTS/transport remain real. Never call the
// original getUserMedia, including on a fallback path.
navigator.mediaDevices.getUserMedia = async () => {
  if (!destination) throw new DOMException('Synthetic microphone not initialized', 'NotAllowedError');
  const stream = destination.stream.clone(); tracks.push(...stream.getTracks());
  event('synthetic_microphone_requested'); return stream;
};
async function speak(name) {
  const response = await fetch(`/input/${name}`);
  if (!response.ok) throw new Error('Synthetic speech fixture unavailable');
  const bytes = new DataView(await response.arrayBuffer());
  const buffer = context.createBuffer(1, bytes.byteLength / 2, 48000);
  const channel = buffer.getChannelData(0);
  for (let i = 0; i < channel.length; i++) channel[i] = bytes.getInt16(i * 2, true) / 32768;
  const source = context.createBufferSource(); source.buffer = buffer; source.connect(destination);
  event('input_start', { label: name });
  await new Promise((resolve) => { source.onended = resolve; source.start(); });
  source.disconnect(); event('input_end', { label: name });
}
async function waitFor(predicate, timeout, label) {
  const deadline = performance.now() + timeout;
  while (!predicate()) {
    if (ended) throw new Error('Conversation ended');
    if (performance.now() >= deadline) throw new Error(`Timed out: ${label}`);
    await new Promise((resolve) => setTimeout(resolve, 100));
  }
}
function renderTranscript(list) {
  transcripts.splice(0, transcripts.length, ...list);
  $('transcript').replaceChildren(...list.map((m) => {
    const item = document.createElement('li'); item.textContent = `${m.source}: ${m.text}`; return item;
  }));
}
const saveQueuedReport = createReportSaver((report, signal) => api("/report", report, signal), {timeoutMs: 10_000});
async function saveReport() {
  const report = { sessionId, sdkVersion: '0.0.13', syntheticMicrophone: true, events, transcripts, cleanupFailures,
    sdkOpen: conversation?.isOpen() ?? false, audioElements: document.querySelectorAll('audio').length,
    inputTrackStates: tracks.map((t) => ({ enabled: t.enabled, readyState: t.readyState })) };
  await saveQueuedReport(report);
  $('evidence').textContent = JSON.stringify({ ...report, transcripts: `${transcripts.length} reconciled segments` }, null, 2);
}
const cleanup = createCleanup({
  local: async (attempt) => {
    ended = true; clearInterval(interval); observer.disconnect();
    $('scenario').disabled = $('mute').disabled = true;
    await attempt('recording.save', () => audioEvidence?.save());
    await attempt('sdk.end', () => conversation?.endSession());
    event('sdk_cleanup', { inputTracksEnded: tracks.every((t) => t.readyState === 'ended'), audioElements: document.querySelectorAll('audio').length });
    await attempt('tracks.stop', () => { destination?.stream.getTracks().forEach((t) => t.stop()); tracks.forEach((t) => t.stop()); });
    await attempt('context.close', () => context?.close());
  },
  provider: () => api('/end'),
  report: saveReport,
  failed: (operation) => { cleanupFailures.push({ operation }); event('cleanup_failed', { operation }); },
});
async function end() {
  $('end').disabled = true;
  try {
    const completed = await cleanup.end();
    status(completed ? 'Ended' : 'Cleanup failed. Retry End and inspect evidence.');
  } finally { $('end').disabled = cleanup.complete; }
}
$('connect').onclick = async () => {
  $('connect').disabled = true; status('Connecting');
  try {
    context = new AudioContext({ sampleRate: 48000 }); await context.resume();
    destination = context.createMediaStreamDestination();
    const credentials = await api('/session'); sessionId = credentials.sessionId;
    conversation = await VoiceConversation.create({ ...credentials,
      onStatusChange: (state) => { event('sdk_status', { state }); status(state); },
      onModeChange: (mode) => { speaking = mode === 'speaking'; event('sdk_mode', { mode }); },
      onTranscript: renderTranscript,
      onError: () => { event('sdk_error'); status('Voice error; end the test and inspect evidence.'); },
      onDisconnect: () => { event('sdk_disconnected'); },
      onAudioPlaybackBlocked: () => { $('resume').hidden = false; event('playback_blocked'); },
    });
    $('scenario').disabled = $('mute').disabled = $('end').disabled = false;
    await conversation.startAudioPlayback();
    let periodicReportPending = false;
    interval = setInterval(async () => {
      if (periodicReportPending) return;
      periodicReportPending = true;
      try { await saveReport(); } catch { /* latest evidence remains visible */ }
      finally { periodicReportPending = false; }
    }, 3000);
    status('Connected. Wait for the greeting, then run the speech scenario.');
  } catch (error) { status(error.message); event('connect_failed'); await end(); }
};
$('scenario').onclick = async () => {
  if (running) return; running = true; $('scenario').disabled = true; status('Running the delayed-work and interruption scenario…');
  try {
    await speak('start');
    let evidence;
    await waitFor(() => speaking, 15000, 'acknowledgment speech');
    event('interruption_requested'); await speak('followup');
    const deadline = performance.now() + 80000;
    while (performance.now() < deadline && !ended) {
      evidence = await api('/evidence');
      $('evidence').textContent = JSON.stringify(evidence, null, 2);
      if (evidence.notificationReady && !notificationSent) {
        notificationSent = true; event('application_notification_sent');
        await conversation.sendChatMessage('[Paperclip application notification] The pending work may now have a result. Call get_updates with cursor 0 and speak the verified answer. This is not a caller instruction; do not call submit_request.');
      }
      if (evidence.resultReturned) break;
      await new Promise((resolve) => setTimeout(resolve, 500));
    }
    if (!evidence?.resultReturned || evidence.acceptedRequests !== 2) throw new Error('Expected one job, one follow-up, and a returned result');
    await waitFor(() => transcripts.some((m) => m.source === 'agent' && m.text.replace(/[,\s]/g, '').includes(evidence.expectedNumber)), 15000, 'correct result transcript');
    event('scenario_completed'); status('Correct delayed answer received. Test mute, then end the conversation.');
    await saveReport();
  } catch (error) { event('scenario_failed', { reason: error.message }); status(error.message); await saveReport(); }
};
$('mute').onclick = async () => {
  try { await conversation.setMicMuted(!muted); muted = !muted; $('mute').textContent = muted ? 'Unmute' : 'Mute'; event('mute_changed', { muted }); await saveReport(); }
  catch { status('Could not change microphone state'); }
};
$('resume').onclick = async () => { await conversation.startAudioPlayback(); $('resume').hidden = conversation.canPlaybackAudio; };
$('end').onclick = () => end().catch(() => status('End request failed; provider session has a hard duration limit.'));
window.addEventListener('pagehide', () => { void conversation?.endSession(); });