Azure AI Voicelive TS

function inferPcmSampleRateFromOutputFormat(outputAudioFormat: string | undefined): number {
  if (!outputAudioFormat) return 24000;
  const fmt = outputAudioFormat.toLowerCase();
  if (fmt === "pcm16") return 24000;
  if (fmt === "pcm16-16000hz") return 16000;
  if (fmt === "pcm16-8000hz") return 8000;
  return 24000;
}

export class SimpleAudioCapture {
  private audioContext?: AudioContext;
  private mediaStream?: MediaStream;
  private scriptProcessor?: ScriptProcessorNode;
  private dataCallback: (data: ArrayBuffer) => void;
  
  private readonly targetSampleRate = 24000;

  constructor(onData: (data: ArrayBuffer) => void) {
    this.dataCallback = onData;
  }

  async initialize(): Promise<void> {
    // Request microphone access
    this.mediaStream = await navigator.mediaDevices.getUserMedia({
      audio: {
        channelCount: 1,
        sampleRate: this.targetSampleRate,
        echoCancellation: true,
        noiseSuppression: true,
      },
    });

    // Create audio context at target sample rate
    this.audioContext = new AudioContext({ sampleRate: this.targetSampleRate });
    
    // Create processing chain
    const source = this.audioContext.createMediaStreamSource(this.mediaStream);
    this.scriptProcessor = this.audioContext.createScriptProcessor(4096, 1, 1);
    
    // Connect nodes
    source.connect(this.scriptProcessor);
    this.scriptProcessor.connect(this.audioContext.destination);
    
    // Process audio data
    this.scriptProcessor.onaudioprocess = (event) => {
      const inputData = event.inputBuffer.getChannelData(0);
      const pcm16Data = this.convertToPCM16(inputData);
      this.dataCallback(pcm16Data.buffer);
    };
  }

  private convertToPCM16(floatData: Float32Array): Int16Array {
    const pcm16 = new Int16Array(floatData.length);
    for (let i = 0; i < floatData.length; i++) {
      const sample = Math.max(-1, Math.min(1, floatData[i]));
      pcm16[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
    }
    return pcm16;
  }

  stop(): void {
    this.scriptProcessor?.disconnect();
    this.mediaStream?.getTracks().forEach((track) => track.stop());
    this.audioContext?.close();
  }
}

// Main thread code
export class AudioWorkletCapture {
  private audioContext?: AudioContext;
  private mediaStream?: MediaStream;
  private workletNode?: AudioWorkletNode;
  private onAudioData: (data: Uint8Array) => void;

  constructor(onData: (data: Uint8Array) => void) {
    this.onAudioData = onData;
  }

  async start(): Promise<void> {
    const ac = new AudioContext();
    this.audioContext = ac;

    // Load AudioWorklet module
    const workletUrl = new URL("worklets/pcm16-downsampler.js", document.baseURI).toString();
    await ac.audioWorklet.addModule(workletUrl);

    // Get microphone stream
    // Disable browser processing - server handles VAD
    const stream = await navigator.mediaDevices.getUserMedia({
      audio: {
        echoCancellation: false,
        noiseSuppression: false,
        autoGainControl: false,
      },
    });
    this.mediaStream = stream;

    // Create processing chain
    const source = ac.createMediaStreamSource(stream);
    const worklet = new AudioWorkletNode(ac, "pcm16-downsampler");
    this.workletNode = worklet;

    // Receive processed PCM16 data from worklet
    worklet.port.onmessage = (e) => {
      const buffer = e.data as ArrayBuffer;
      const pcm16Bytes = new Uint8Array(buffer);
      this.onAudioData(pcm16Bytes);
    };

    // Connect with muted output to prevent feedback
    const gain = ac.createGain();
    gain.gain.value = 0;
    source.connect(worklet);
    worklet.connect(gain);
    gain.connect(ac.destination);
  }

  stop(): void {
    this.workletNode?.disconnect();
    this.mediaStream?.getTracks().forEach((track) => track.stop());
    this.audioContext?.close();
  }
}

class Pcm16DownsamplerProcessor extends AudioWorkletProcessor {
  constructor() {
    super();
    this._targetSampleRate = 24000;
    this._inputSampleRate = sampleRate;  // Browser's native rate (e.g., 48000)
    this._ratio = this._inputSampleRate / this._targetSampleRate;
    this._carry = 0;
  }

  process(inputs) {
    const input = inputs[0];
    if (!input || input.length === 0) return true;

    const channel0 = input[0];
    if (!channel0 || channel0.length === 0) return true;

    // Downsample by averaging
    const outLength = Math.max(0, Math.floor((channel0.length - this._carry) / this._ratio));
    if (outLength === 0) return true;

    const pcm16 = new Int16Array(outLength);
    let outIndex = 0;
    let i = this._carry;

    while (outIndex < outLength) {
      const start = Math.floor(i);
      const end = Math.floor(i + this._ratio);

      let sum = 0, count = 0;
      for (let j = start; j < end && j < channel0.length; j++) {
        sum += channel0[j];
        count++;
      }

      const sample = count > 0 ? sum / count : 0;
      const clamped = Math.max(-1, Math.min(1, sample));
      pcm16[outIndex] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;

      outIndex++;
      i += this._ratio;
    }

    this._carry = i - channel0.length;
    if (this._carry < 0) this._carry = 0;

    // Transfer buffer to main thread
    this.port.postMessage(pcm16.buffer, [pcm16.buffer]);
    return true;
  }
}

registerProcessor("pcm16-downsampler", Pcm16DownsamplerProcessor);

export class Pcm16Player {
  private readonly audioContext: AudioContext;
  private nextStartTime = 0;
  private sources: AudioBufferSourceNode[] = [];
  private sampleRate: number;

  constructor(sampleRate = 24000) {
    this.audioContext = new AudioContext();
    this.sampleRate = sampleRate;
  }

  async resume(): Promise<void> {
    if (this.audioContext.state !== "running") {
      await this.audioContext.resume();
    }
  }

  enqueuePcm16(bytes: Uint8Array): void {
    // Convert PCM16 bytes to Float32 for Web Audio API
    const floatData = this.pcm16BytesToFloat32(bytes);
    
    // Create audio buffer
    const buffer = this.audioContext.createBuffer(1, floatData.length, this.sampleRate);
    buffer.getChannelData(0).set(floatData);

    // Create and schedule source
    const src = this.audioContext.createBufferSource();
    src.buffer = buffer;
    src.connect(this.audioContext.destination);

    // Schedule seamless playback
    const now = this.audioContext.currentTime;
    if (this.nextStartTime < now) {
      this.nextStartTime = now;
    }
    const startAt = this.nextStartTime;
    this.nextStartTime += buffer.duration;

    // Track for cleanup
    src.onended = () => {
      this.sources = this.sources.filter((s) => s !== src);
    };
    this.sources.push(src);
    src.start(startAt);
  }

  private pcm16BytesToFloat32(bytes: Uint8Array): Float32Array {
    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
    const sampleCount = Math.floor(bytes.byteLength / 2);
    const out = new Float32Array(sampleCount);
    for (let i = 0; i < sampleCount; i++) {
      const s = view.getInt16(i * 2, true);  // Little-endian
      out[i] = s / 0x8000;
    }
    return out;
  }

  stop(): void {
    for (const s of this.sources) {
      try {
        s.stop();
      } catch {
        /* ignore */
      }
    }
    this.sources = [];
    this.nextStartTime = 0;
  }
}

// audio-playback-worklet.js
class AudioPlaybackWorklet extends AudioWorkletProcessor {
  constructor() {
    super();
    this.port.onmessage = this.handleMessage.bind(this);
    this.buffer = [];
  }

  handleMessage(event) {
    if (event.data === null) {
      // Clear buffer on barge-in
      this.buffer = [];
      return;
    }
    // Append Int16 samples
    this.buffer.push(...event.data);
  }

  process(inputs, outputs) {
    const output = outputs[0];
    const channel = output[0];

    if (this.buffer.length > channel.length) {
      const toProcess = this.buffer.slice(0, channel.length);
      this.buffer = this.buffer.slice(channel.length);
      channel.set(toProcess.map((v) => v / 32768));  // Int16 to Float32
    } else {
      channel.set(this.buffer.map((v) => v / 32768));
      this.buffer = [];
    }

    return true;
  }
}

registerProcessor("audio-playback-worklet", AudioPlaybackWorklet);

// Float32 to PCM16 (for sending to server)
export function floatTo16BitPCM(input: Float32Array): Int16Array {
  const output = new Int16Array(input.length);
  for (let i = 0; i < input.length; i++) {
    const s = Math.max(-1, Math.min(1, input[i] ?? 0));
    output[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
  }
  return output;
}

// PCM16 bytes to Float32 (for playback)
export function pcm16BytesToFloat32LE(bytes: Uint8Array): Float32Array {
  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
  const sampleCount = Math.floor(bytes.byteLength / 2);
  const out = new Float32Array(sampleCount);
  for (let i = 0; i < sampleCount; i++) {
    const s = view.getInt16(i * 2, true);  // Little-endian
    out[i] = s / 0x8000;
  }
  return out;
}

// Int16Array to Uint8Array (for sendAudio)
export function int16ToUint8(int16Array: Int16Array): Uint8Array {
  return new Uint8Array(int16Array.buffer);
}

import { VoiceLiveClient } from "@azure/ai-voicelive";
import { DefaultAzureCredential } from "@azure/identity";

const client = new VoiceLiveClient(endpoint, new DefaultAzureCredential());
const session = await client.startSession("gpt-4o-mini-realtime-preview");

// Configure audio formats
await session.updateSession({
  modalities: ["audio", "text"],
  inputAudioFormat: "pcm16",
  outputAudioFormat: "pcm16",
  turnDetection: {
    type: "server_vad",
    threshold: 0.5,
    silenceDurationMs: 500,
  },
});

// Initialize audio capture and playback
const player = new Pcm16Player(24000);
const capture = new AudioWorkletCapture((data) => {
  session.sendAudio(data);
});

// Handle audio output from assistant
const subscription = session.subscribe({
  onResponseAudioDelta: async (event) => {
    player.enqueuePcm16(event.delta);
  },
  onInputAudioBufferSpeechStarted: async () => {
    // User started speaking - stop playback (barge-in)
    player.stop();
  },
});

// Start capture
await capture.start();
await player.resume();

// Cleanup
async function cleanup() {
  capture.stop();
  player.stop();
  await subscription.close();
  await session.close();
}