Building Low-Latency Voice AI Agents: WebSockets, WebRTC & Gemini Live API

Legacy Cascaded Voice Pipeline (1,850ms Total Round-Trip Delay):
[User Speaks] ──► [ASR: 450ms] ──► [LLM Time-To-First-Token: 600ms] ──► [TTS Synthesis: 800ms] ──► [Audio Output]

/**
 * gemini-live-voice-client.ts
 * Sub-300ms Full-Duplex Real-Time Voice Agent Client
 */

import WebSocket from "isomorphic-ws";

export interface VoiceAgentConfig {
  ephemeralToken: string;
  model?: string;
  systemInstruction?: string;
  voiceName?: "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede";
  onAudioData: (pcmBuffer: Int16Array) => void;
  onInterrupted: () => void;
  onTurnComplete: () => void;
}

export class GeminiLiveVoiceAgent {
  private ws: WebSocket | null = null;
  private isConnected: boolean = false;

  constructor(private config: VoiceAgentConfig) {}

  public async connect(): Promise<void> {
    const model = this.config.model || "gemini-2.0-flash-exp";
    const uri = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key=${this.config.ephemeralToken}`;

    this.ws = new WebSocket(uri);

    return new Promise((resolve, reject) => {
      if (!this.ws) return reject(new Error("Failed to initialize WebSocket."));

      this.ws.onopen = () => {
        this.isConnected = true;
        this.sendSessionSetup();
        resolve();
      };

      this.ws.onmessage = (event: WebSocket.MessageEvent) => {
        this.handleIncomingMessage(event.data);
      };

      this.ws.onerror = (error) => {
        console.error("Gemini Live WebSocket Error:", error);
      };

      this.ws.onclose = () => {
        this.isConnected = false;
      };
    });
  }

  // ═══════════════════════════════════════════════════════════════
  // 1. Send Initial Session Configuration
  // ═══════════════════════════════════════════════════════════════
  private sendSessionSetup(): void {
    const setupPayload = {
      setup: {
        model: `models/${this.config.model || "gemini-2.0-flash-exp"}`,
        generation_config: {
          response_modalities: ["AUDIO"],
          speech_config: {
            voice_config: {
              prebuilt_voice_config: {
                voice_name: this.config.voiceName || "Aoede",
              },
            },
          },
        },
        system_instruction: {
          parts: [{ text: this.config.systemInstruction || "You are a helpful, conversational customer support agent. Speak concisely." }],
        },
      },
    };

    this.ws?.send(JSON.stringify(setupPayload));
  }

  // ═══════════════════════════════════════════════════════════════
  // 2. Stream Inbound Microphone PCM Audio (16kHz Mono)
  // ═══════════════════════════════════════════════════════════════
  public sendMicrophoneChunk(pcm16Array: Int16Array): void {
    if (!this.isConnected || !this.ws) return;

    // Convert Int16Array to Base64 PCM
    const buffer = Buffer.from(pcm16Array.buffer);
    const base64Data = buffer.toString("base64");

    const realtimeInput = {
      realtime_input: {
        media_chunks: [
          {
            mime_type: "audio/pcm;rate=16000",
            data: base64Data,
          },
        ],
      },
    };

    this.ws.send(JSON.stringify(realtimeInput));
  }

  // ═══════════════════════════════════════════════════════════════
  // 3. Client-Side Interruption Signal
  // ═══════════════════════════════════════════════════════════════
  public triggerClientInterruption(): void {
    this.config.onInterrupted();
    if (!this.isConnected || !this.ws) return;

    // Send interruption flag to server
    this.ws.send(JSON.stringify({ client_content: { turn_complete: false, interrupted: true } }));
  }

  // ═══════════════════════════════════════════════════════════════
  // 4. Handle Incoming Model Audio Stream (24kHz Mono)
  // ═══════════════════════════════════════════════════════════════
  private handleIncomingMessage(data: any): void {
    try {
      const response = JSON.parse(data.toString());

      // Check for server-detected interruption
      if (response.serverContent?.interrupted) {
        this.config.onInterrupted();
        return;
      }

      // Process outbound audio chunks
      const parts = response.serverContent?.modelTurn?.parts || [];
      for (const part of parts) {
        if (part.inlineData && part.inlineData.mimeType?.startsWith("audio/pcm")) {
          const rawBuffer = Buffer.from(part.inlineData.data, "base64");
          const int16Samples = new Int16Array(
            rawBuffer.buffer,
            rawBuffer.byteOffset,
            rawBuffer.byteLength / Int16Array.BYTES_PER_ELEMENT
          );
          this.config.onAudioData(int16Samples);
        }
      }

      if (response.serverContent?.turnComplete) {
        this.config.onTurnComplete();
      }
    } catch (err) {
      console.error("Error parsing Gemini Live message:", err);
    }
  }

  public disconnect(): void {
    if (this.ws) {
      this.ws.close();
      this.isConnected = false;
    }
  }
}