### Install and Start Project

Source: https://github.com/google-gemini/live-api-web-console/blob/main/README.md

Install project dependencies and start the development server. Ensure you have a Gemini API key added to your .env file.

```bash
npm install && npm start
```

--------------------------------

### Run Development Server

Source: https://github.com/google-gemini/live-api-web-console/blob/main/README.md

Starts the React development server. The application will be accessible at http://localhost:3000. Lint errors will be displayed in the console.

```bash
npm start
```

--------------------------------

### Render Altair Graphs with Live API

Source: https://github.com/google-gemini/live-api-web-console/blob/main/README.md

This React component configures the Live API client to use Google Search and a custom function declaration for rendering Altair graphs. It listens for 'toolcall' events to extract JSON graph data and uses vega-embed to render the graph. Ensure the 'vega-embed' library is installed.

```typescript
import { type FunctionDeclaration, SchemaType } from "@google/generative-ai";
import { useEffect, useRef, useState, memo } from "react";
import vegaEmbed from "vega-embed";
import { useLiveAPIContext } from "../../contexts/LiveAPIContext";

export const declaration: FunctionDeclaration = {
  name: "render_altair",
  description: "Displays an altair graph in json format.",
  parameters: {
    type: SchemaType.OBJECT,
    properties: {
      json_graph: {
        type: SchemaType.STRING,
        description:
          "JSON STRING representation of the graph to render. Must be a string, not a json object",
      },
    },
    required: ["json_graph"],
  },
};

export function Altair() {
  const [jsonString, setJSONString] = useState<string>("");
  const { client, setConfig } = useLiveAPIContext();

  useEffect(() => {
    setConfig({
      model: "models/gemini-2.0-flash-exp",
      systemInstruction: {
        parts: [
          {
            text: 'You are my helpful assistant. Any time I ask you for a graph call the "render_altair" function I have provided you. Dont ask for additional information just make your best judgement.',
          },
        ],
      },
      tools: [{ googleSearch: {} }, { functionDeclarations: [declaration] }],
    });
  }, [setConfig]);

  useEffect(() => {
    const onToolCall = (toolCall: ToolCall) => {
      console.log(`got toolcall`, toolCall);
      const fc = toolCall.functionCalls.find(
        (fc) => fc.name === declaration.name
      );
      if (fc) {
        const str = (fc.args as any).json_graph;
        setJSONString(str);
      }
    };
    client.on("toolcall", onToolCall);
    return () => {
      client.off("toolcall", onToolCall);
    };
  }, [client]);

  const embedRef = useRef<HTMLDivElement>(null);

  useEffect(() => {
    if (embedRef.current && jsonString) {
      vegaEmbed(embedRef.current, JSON.parse(jsonString));
    }
  }, [embedRef, jsonString]);
  return <div className="vega-embed" ref={embedRef} />;
}

```

--------------------------------

### GenAILiveClient Usage

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Demonstrates how to instantiate, connect, send messages, listen to events, and disconnect the GenAILiveClient for real-time AI interactions.

```APIDOC
## GenAILiveClient

### Description
Manages the lifecycle of a Live API session, wraps the `@google/genai` SDK, exposes typed events for server messages, and provides methods to send text content, real-time media input, and tool responses. It can be used independently of React.

### Instantiate
```typescript
import { GenAILiveClient } from "./src/lib/genai-live-client";

const client = new GenAILiveClient({ apiKey: process.env.REACT_APP_GEMINI_API_KEY! });
```

### Event Listeners
```typescript
client.on("open", () => console.log("Session open"));
client.on("setupcomplete", () => console.log("Model ready to receive input"));
client.on("audio", (data: ArrayBuffer) => {
  console.log(`Received audio buffer: ${data.byteLength} bytes`);
});
client.on("content", (data) => {
  console.log("Model content:", JSON.stringify(data));
});
client.on("toolcall", (toolCall) => {
  console.log("Tool call received:", toolCall.functionCalls);
});
client.on("turncomplete", () => console.log("Model turn finished"));
client.on("interrupted", () => console.log("Generation interrupted by user input"));
client.on("close", (e: CloseEvent) => console.log("Session closed:", e.reason));
client.on("error", (e: ErrorEvent) => console.error("Error:", e.message));
```

### Connect to Model
```typescript
import { Modality } from "@google/genai";

const connected = await client.connect("models/gemini-2.0-flash-exp", {
  responseModalities: [Modality.AUDIO],
  speechConfig: {
    voiceConfig: { prebuiltVoiceConfig: { voiceName: "Aoede" } },
  },
  systemInstruction: {
    parts: [{ text: "You are a helpful assistant." }],
  },
  tools: [{ googleSearch: {} }],
});

if (!connected) {
  console.error("Already connected or connection failed");
}
```

### Send Messages
```typescript
// Send a text message (triggers the model to respond)
client.send({ text: "Hello! Can you tell me about the weather in Paris?" });

// Send a text message without ending the turn (for multi-part input)
client.send({ text: "Also," }, false);
client.send({ text: "what about London?" }, true);
```

### Inspect State
```typescript
console.log(client.status);  // "connected" | "disconnected" | "connecting"
console.log(client.model);   // "models/gemini-2.0-flash-exp"
console.log(client.getConfig()); // Returns a copy of the active LiveConnectConfig
```

### Disconnect
```typescript
client.disconnect();
```
```

--------------------------------

### GenAILiveClient: WebSocket Client for Live API

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Instantiate and manage the GenAILiveClient for real-time communication with the Gemini Live API. Listen to various events, connect to a model, send messages, and disconnect.

```typescript
import { GenAILiveClient } from "./src/lib/genai-live-client";
import { Modality, Type } from "@google/genai";

// Instantiate with your API key
const client = new GenAILiveClient({ apiKey: process.env.REACT_APP_GEMINI_API_KEY! });

// Listen to lifecycle and data events
client.on("open", () => console.log("Session open"));
client.on("setupcomplete", () => console.log("Model ready to receive input"));
client.on("audio", (data: ArrayBuffer) => {
  // data is raw PCM16 audio — pipe into AudioStreamer
  console.log(`Received audio buffer: ${data.byteLength} bytes`);
});
client.on("content", (data) => {
  // Non-audio model turn parts (text, code execution results, etc.)
  console.log("Model content:", JSON.stringify(data));
});
client.on("toolcall", (toolCall) => {
  // Handle function calls declared in config
  console.log("Tool call received:", toolCall.functionCalls);
});
client.on("turncomplete", () => console.log("Model turn finished"));
client.on("interrupted", () => console.log("Generation interrupted by user input"));
client.on("close", (e: CloseEvent) => console.log("Session closed:", e.reason));
client.on("error", (e: ErrorEvent) => console.error("Error:", e.message));

// Connect to a model with session configuration
const connected = await client.connect("models/gemini-2.0-flash-exp", {
  responseModalities: [Modality.AUDIO],
  speechConfig: {
    voiceConfig: { prebuiltVoiceConfig: { voiceName: "Aoede" } },
  },
  systemInstruction: {
    parts: [{ text: "You are a helpful assistant." }],
  },
  tools: [{ googleSearch: {} }],
});

if (!connected) {
  console.error("Already connected or connection failed");
}

// Send a text message (triggers the model to respond)
client.send({ text: "Hello! Can you tell me about the weather in Paris?" });

// Send a text message without ending the turn (for multi-part input)
client.send({ text: "Also," }, false);
client.send({ text: "what about London?" }, true);

// Inspect current state
console.log(client.status);  // "connected" | "disconnected" | "connecting"
console.log(client.model);   // "models/gemini-2.0-flash-exp"
console.log(client.getConfig()); // Returns a copy of the active LiveConnectConfig

// Disconnect
client.disconnect();

```

--------------------------------

### LiveAPIProvider and useLiveAPIContext

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

The `LiveAPIProvider` component allows you to wrap your application with a shared `useLiveAPI` instance, distributing it via React context. Descendant components can then use `useLiveAPIContext` to access the client and session state without prop drilling.

```APIDOC
## LiveAPIProvider / useLiveAPIContext — React context distribution

`LiveAPIProvider` wraps your application with a single shared `useLiveAPI` instance distributed through React context. Any descendant can call `useLiveAPIContext()` to access the client and session state without prop drilling.

```typescript
// index.tsx — Wrap your app at the root
import React from "react";
import ReactDOM from "react-dom/client";
import { LiveAPIProvider } from "./contexts/LiveAPIContext";
import App from "./App";

ReactDOM.createRoot(document.getElementById("root")!).render(
  <LiveAPIProvider options={{ apiKey: process.env.REACT_APP_GEMINI_API_KEY! }}>
    <App />
  </LiveAPIProvider>
);

// ---- In any child component ----
import { useLiveAPIContext } from "./contexts/LiveAPIContext";
import { FunctionDeclaration, Type, Modality } from "@google/genai";

const searchDeclaration: FunctionDeclaration = {
  name: "search_database",
  description: "Searches the product database.",
  parameters: {
    type: Type.OBJECT,
    properties: { query: { type: Type.STRING } },
    required: ["query"],
  },
};

function ProductSearchWidget() {
  const { client, setConfig, connected, connect, disconnect } = useLiveAPIContext();

  useEffect(() => {
    setConfig({
      responseModalities: [Modality.AUDIO],
      tools: [{ functionDeclarations: [searchDeclaration] }],
    });
  }, [setConfig]);

  useEffect(() => {
    const handleToolCall = (toolCall: any) => {
      const fc = toolCall.functionCalls?.find((f: any) => f.name === "search_database");
      if (fc) {
        const results = performLocalSearch((fc.args as any).query);
        client.sendToolResponse({
          functionResponses: [{
            id: fc.id,
            name: fc.name,
            response: { output: { results } },
          }],
        });
      }
    };
    client.on("toolcall", handleToolCall);
    return () => { client.off("toolcall", handleToolCall); };
  }, [client]);

  return (
    <button onClick={connected ? disconnect : connect}>
      {connected ? "End Session" : "Start Session"}
    </button>
  );
}
```
```

--------------------------------

### Implement Logger for Streaming Logs

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Utilize the Zustand store `useLoggerStore` and the `Logger` component to buffer and display streaming logs. Forward client logs to the store and render logs with filtering options for conversations, tools, or all entries.

```typescript
import { useLoggerStore } from "./src/lib/store-logger";
import Logger from "./src/components/logger/Logger";
import { GenAILiveClient } from "./src/lib/genai-live-client";

// --- Bridge client logs into the store ---
function App() {
  const { log, clearLogs, logs } = useLoggerStore();
  const client = new GenAILiveClient({ apiKey: "YOUR_API_KEY" });

  useEffect(() => {
    // GenAILiveClient emits "log" events for every send/receive — forward them
    const onLog = (entry: StreamingLog) => log(entry);
    client.on("log", onLog);
    return () => { client.off("log", onLog); };
  }, [client, log]);

  return (
    <div style={{ display: "flex", gap: 16 }}>
      <div>
        {/* Show only conversation turns */}
        <h3>Conversation</h3>
        <Logger filter="conversations" />
      </div>
      <div>
        {/* Show only tool call/response entries */}
        <h3>Tool Calls</h3>
        <Logger filter="tools" />
      </div>
      <div>
        {/* Show everything */}
        <h3>All Logs</h3>
        <Logger filter="none" />
      </div>
      <button onClick={clearLogs}>Clear</button>
      <p>Buffered entries: {logs.length} / 100</p>
    </div>
  );
}

```

--------------------------------

### LiveAPIProvider for Context Distribution

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Wrap your application with LiveAPIProvider to share a single useLiveAPI instance via React context. Descendant components can use useLiveAPIContext to access the client and session state without prop drilling.

```typescript
// index.tsx — Wrap your app at the root
import React from "react";
import ReactDOM from "react-dom/client";
import { LiveAPIProvider } from "./contexts/LiveAPIContext";
import App from "./App";

ReactDOM.createRoot(document.getElementById("root")!).render(
  <LiveAPIProvider options={{ apiKey: process.env.REACT_APP_GEMINI_API_KEY! }}>
    <App />
  </LiveAPIProvider>
);

// ---- In any child component ----
import { useLiveAPIContext } from "./contexts/LiveAPIContext";
import { FunctionDeclaration, Type, Modality } from "@google/genai";

const searchDeclaration: FunctionDeclaration = {
  name: "search_database",
  description: "Searches the product database.",
  parameters: {
    type: Type.OBJECT,
    properties: { query: { type: Type.STRING } },
    required: ["query"],
  },
};

function ProductSearchWidget() {
  const { client, setConfig, connected, connect, disconnect } = useLiveAPIContext();

  useEffect(() => {
    setConfig({
      responseModalities: [Modality.AUDIO],
      tools: [{ functionDeclarations: [searchDeclaration] }],
    });
  }, [setConfig]);

  useEffect(() => {
    const handleToolCall = (toolCall: any) => {
      const fc = toolCall.functionCalls?.find((f: any) => f.name === "search_database");
      if (fc) {
        const results = performLocalSearch((fc.args as any).query);
        client.sendToolResponse({
          functionResponses: [{
            id: fc.id,
            name: fc.name,
            response: { output: { results } },
          }],
        });
      }
    };
    client.on("toolcall", handleToolCall);
    return () => { client.off("toolcall", handleToolCall); };
  }, [client]);

  return (
    <button onClick={connected ? disconnect : connect}>
      {connected ? "End Session" : "Start Session"}
    </button>
  );
}

```

--------------------------------

### useLiveAPI Hook for Session State Management

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Use this hook to manage session state, audio output, and interact with the GenAI Live Client in React applications. Configure the session before connecting by setting the model and other options.

```typescript
import { useLiveAPI } from "./src/hooks/use-live-api";
import { Modality } from "@google/genai";

function MyApp() {
  const {
    client,        // GenAILiveClient instance
    connected,     // boolean — is the session active?
    connect,       // () => Promise<void> — opens the WebSocket session
    disconnect,    // () => Promise<void> — closes the session
    config,        // LiveConnectConfig — current session config
    setConfig,     // (config: LiveConnectConfig) => void
    model,         // string — current model identifier
    setModel,      // (model: string) => void
    volume,        // number — real-time output audio volume (0–1)
  } = useLiveAPI({ apiKey: process.env.REACT_APP_GEMINI_API_KEY! });

  // Configure the session before connecting
  useEffect(() => {
    setModel("models/gemini-2.0-flash-exp");
    setConfig({
      responseModalities: [Modality.AUDIO],
      speechConfig: {
        voiceConfig: { prebuiltVoiceConfig: { voiceName: "Puck" } },
      },
      systemInstruction: {
        parts: [{ text: "You are a concise assistant who speaks in bullet points." }],
      },
      tools: [{ googleSearch: {} }],
    });
  }, [setConfig, setModel]);

  const handleToggle = async () => {
    if (connected) {
      await disconnect();
    } else {
      await connect(); // Uses current model + config
    }
  };

  // Send text once connected
  const handleSend = () => {
    if (connected) {
      client.send({ text: "Summarise the latest AI news." });
    }
  };

  return (
    <div>
      <p>Status: {connected ? "Connected" : "Disconnected"}</p>
      <p>Output volume: {(volume * 100).toFixed(0)}%</p>
      <button onClick={handleToggle}>{connected ? "Stop" : "Start"}</button>
      <button onClick={handleSend} disabled={!connected}>Send</button>
    </div>
  );
}

```

--------------------------------

### Capture Microphone Audio with AudioRecorder

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Captures microphone input, converts it to PCM16 at 16 kHz, and emits base64-encoded chunks. It also emits a 'volume' event for VU-meter display. Use this to send live audio to a model.

```typescript
import { AudioRecorder } from "./src/lib/audio-recorder";
import { GenAILiveClient } from "./src/lib/genai-live-client";

const recorder = new AudioRecorder(16000); // 16 kHz sample rate
const client = new GenAILiveClient({ apiKey: "YOUR_API_KEY" });

await client.connect("models/gemini-2.0-flash-exp", { responseModalities: ["AUDIO"] });

// Forward microphone audio to the model
recorder.on("data", (base64: string) => {
  client.sendRealtimeInput([{ mimeType: "audio/pcm;rate=16000", data: base64 }]);
});

// Monitor microphone input level (0–1)
recorder.on("volume", (level: number) => {
  document.getElementById("vu-meter")!.style.width = `${level * 100}%`;
});

// Start recording (requests microphone permission)
try {
  await recorder.start();
  console.log("Recording started");
} catch (e) {
  console.error("Microphone access denied:", e);
}

// Stop when the session ends
client.on("close", () => {
  recorder.stop();
});
```

--------------------------------

### Integrate ControlTray for Media Control

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Use ControlTray to manage audio recording, video streaming, and session connect/disconnect. It reads connection state from context and sends video frames. Configure stream support and editing settings.

```typescript
import { useRef, useState } from "react";
import { LiveAPIProvider } from "./contexts/LiveAPIContext";
import ControlTray from "./components/control-tray/ControlTray";

function App() {
  const videoRef = useRef<HTMLVideoElement>(null);
  const [videoStream, setVideoStream] = useState<MediaStream | null>(null);

  return (
    <LiveAPIProvider options={{ apiKey: process.env.REACT_APP_GEMINI_API_KEY! }}>
      {/* Preview the active video stream */}
      <video
        ref={videoRef}
        autoPlay
        playsInline
        style={{ display: videoStream ? "block" : "none", width: 480 }}
      />

      <ControlTray
        videoRef={videoRef}
        supportsVideo={true}               // Show webcam + screen-share buttons
        onVideoStreamChange={setVideoStream} // Track which stream is active
        enableEditingSettings={true}       // Show settings dialog button
      >
        {/* Inject custom action buttons into the tray */}
        <button className="action-button" onClick={() => alert("Custom action!")}>
          <span className="material-symbols-outlined">star</span>
        </button>
      </ControlTray>
    </LiveAPIProvider>
  );
}

```

--------------------------------

### useLiveAPI Hook

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

The `useLiveAPI` hook provides a convenient way to manage session state, audio output, and interact with the Gemini API within React components. It handles connection, disconnection, configuration, and monitoring of audio volume.

```APIDOC
## useLiveAPI — React hook for session state management

`useLiveAPI` wraps `GenAILiveClient` with React state, automatically manages audio output via `AudioStreamer`, and exposes a clean interface for connecting, disconnecting, setting config, and monitoring output volume. It is the recommended integration point for React applications.

```typescript
import { useLiveAPI } from "./src/hooks/use-live-api";
import { Modality } from "@google/genai";

function MyApp() {
  const {
    client,        // GenAILiveClient instance
    connected,     // boolean — is the session active?
    connect,       // () => Promise<void> — opens the WebSocket session
    disconnect,    // () => Promise<void> — closes the session
    config,        // LiveConnectConfig — current session config
    setConfig,     // (config: LiveConnectConfig) => void
    model,         // string — current model identifier
    setModel,      // (model: string) => void
    volume,        // number — real-time output audio volume (0–1)
  } = useLiveAPI({ apiKey: process.env.REACT_APP_GEMINI_API_KEY! });

  // Configure the session before connecting
  useEffect(() => {
    setModel("models/gemini-2.0-flash-exp");
    setConfig({
      responseModalities: [Modality.AUDIO],
      speechConfig: { 
        voiceConfig: { prebuiltVoiceConfig: { voiceName: "Puck" } }, 
      },
      systemInstruction: {
        parts: [{ text: "You are a concise assistant who speaks in bullet points." }],
      },
      tools: [{ googleSearch: {} }],
    });
  }, [setConfig, setModel]);

  const handleToggle = async () => {
    if (connected) {
      await disconnect();
    } else {
      await connect(); // Uses current model + config
    }
  };

  // Send text once connected
  const handleSend = () => {
    if (connected) {
      client.send({ text: "Summarise the latest AI news." });
    }
  };

  return (
    <div>
      <p>Status: {connected ? "Connected" : "Disconnected"}</p>
      <p>Output volume: {(volume * 100).toFixed(0)}%</p>
      <button onClick={handleToggle}>{connected ? "Stop" : "Start"}</button>
      <button onClick={handleSend} disabled={!connected}>Send</button>
    </div>
  );
}
```
```

--------------------------------

### Manage Screen Capture with useScreenCapture Hook

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

A React hook that manages a getDisplayMedia screen-capture stream with the same interface as useWebcam. It allows users to share their screen or a specific window to the model. Use this for screen or window sharing.

```typescript
import { useScreenCapture } from "./src/hooks/use-screen-capture";

function ScreenShareButton() {
  const { start, stop, isStreaming, stream } = useScreenCapture();

  return (
    <button
      onClick={async () => {
        if (isStreaming) {
          stop();
        } else {
          try {
            await start(); // Triggers browser's screen picker dialog
          } catch (e) {
            console.error("Screen share cancelled or denied:", e);
          }
        }
      }} >
      {isStreaming ? "Stop Sharing" : "Share Screen"}
    </button>
  );
}
```

--------------------------------

### Stream Real-time Media with sendRealtimeInput

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Use this method to send base64-encoded audio (PCM) and video (JPEG) frames to the model. It's suitable for continuous streaming during an active session. Ensure media is correctly formatted and encoded.

```typescript
import { GenAILiveClient } from "./src/lib/genai-live-client";

const client = new GenAILiveClient({ apiKey: "YOUR_API_KEY" });
await client.connect("models/gemini-2.0-flash-exp", { responseModalities: ["AUDIO"] });

// --- Send microphone audio (PCM16 at 16 kHz, base64-encoded) ---
// This is typically produced by AudioRecorder and sent on the "data" event
function onAudioData(base64: string) {
  client.sendRealtimeInput([
    { mimeType: "audio/pcm;rate=16000", data: base64 },
  ]);
}

// --- Capture a video frame from a <canvas> and send as JPEG ---
function sendVideoFrame(video: HTMLVideoElement) {
  const canvas = document.createElement("canvas");
  canvas.width = video.videoWidth * 0.25;   // downscale to 25%
  canvas.height = video.videoHeight * 0.25;
  const ctx = canvas.getContext("2d")!;
  ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
  const base64 = canvas.toDataURL("image/jpeg", 1.0).split(",")[1];
  client.sendRealtimeInput([{ mimeType: "image/jpeg", data: base64 }]);
}

// --- Send both audio and video together in one call ---
client.sendRealtimeInput([
  { mimeType: "audio/pcm;rate=16000", data: audioBase64 },
  { mimeType: "image/jpeg", data: frameBase64 },
]);
// Logs: "client.realtimeInput" → "audio + video"
```

--------------------------------

### Manage Webcam Stream with useWebcam Hook

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

A React hook that manages a getUserMedia video stream lifecycle as React state. It provides start/stop controls and streaming status. Use this to feed live video frames to the model.

```typescript
import { useWebcam } from "./src/hooks/use-webcam";

function WebcamButton() {
  const { start, stop, isStreaming, stream } = useWebcam();

  // Attach stream to a video element for preview
  const videoRef = useRef<HTMLVideoElement>(null);
  useEffect(() => {
    if (videoRef.current) videoRef.current.srcObject = stream;
  }, [stream]);

  return (
    <div>
      <video ref={videoRef} autoPlay playsInline muted style={{ width: 320 }} />
      <button onClick={isStreaming ? stop : start}>
        {isStreaming ? "Stop Camera" : "Start Camera"}
      </button>
    </div>
  );
}
```

--------------------------------

### Respond to Model Tool Calls with sendToolResponse

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

After the model emits a `toolcall` event, use this method to provide results for each function call. The response maps function call IDs to their return values. Ensure the response format matches the expected output for each function.

```typescript
import { GenAILiveClient } from "./src/lib/genai-live-client";
import { FunctionDeclaration, Type } from "@google/genai";

const getWeatherDeclaration: FunctionDeclaration = {
  name: "get_weather",
  description: "Returns the current weather for a given city.",
  parameters: {
    type: Type.OBJECT,
    properties: {
      city: { type: Type.STRING, description: "City name" },
    },
    required: ["city"],
  },
};

const client = new GenAILiveClient({ apiKey: "YOUR_API_KEY" });
await client.connect("models/gemini-2.0-flash-exp", {
  responseModalities: ["AUDIO"],
  tools: [{ functionDeclarations: [getWeatherDeclaration] }],
});

// Handle incoming tool calls and respond
client.on("toolcall", (toolCall) => {
  if (!toolCall.functionCalls) return;

  const responses = toolCall.functionCalls.map((fc) => {
    let result: unknown;

    if (fc.name === "get_weather") {
      const city = (fc.args as { city: string }).city;
      // Call your actual weather API here
      result = { temperature: "18°C", condition: "Partly cloudy", city };
    } else {
      result = { error: "Unknown function" };
    }

    return {
      id: fc.id,
      name: fc.name,
      response: { output: result },
    };
  });

  // Small delay to allow model to process before receiving the response
  setTimeout(() => {
    client.sendToolResponse({ functionResponses: responses });
  }, 200);
});

client.send({ text: "What is the weather like in Tokyo?" });
```

--------------------------------

### client.sendRealtimeInput

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Streams base64-encoded realtime media chunks (audio PCM or JPEG images) to the model during an active session. This method is used for continuous microphone and video streaming.

```APIDOC
## client.sendRealtimeInput — Stream audio and video frames

Sends base64-encoded realtime media chunks to the model during an active session. Accepts an array of `{ mimeType, data }` objects and can mix audio PCM and JPEG image frames in a single call. Used internally by `ControlTray` for continuous microphone and video streaming.

```typescript
import { GenAILiveClient } from "./src/lib/genai-live-client";

const client = new GenAILiveClient({ apiKey: "YOUR_API_KEY" });
await client.connect("models/gemini-2.0-flash-exp", { responseModalities: ["AUDIO"] });

// --- Send microphone audio (PCM16 at 16 kHz, base64-encoded) ---
// This is typically produced by AudioRecorder and sent on the "data" event
function onAudioData(base64: string) {
  client.sendRealtimeInput([
    { mimeType: "audio/pcm;rate=16000", data: base64 },
  ]);
}

// --- Capture a video frame from a <canvas> and send as JPEG ---
function sendVideoFrame(video: HTMLVideoElement) {
  const canvas = document.createElement("canvas");
  canvas.width = video.videoWidth * 0.25;   // downscale to 25%
  canvas.height = video.videoHeight * 0.25;
  const ctx = canvas.getContext("2d")!;
  ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
  const base64 = canvas.toDataURL("image/jpeg", 1.0).split(",")[1];
  client.sendRealtimeInput([{ mimeType: "image/jpeg", data: base64 }]);
}

// --- Send both audio and video together in one call ---
client.sendRealtimeInput([
  { mimeType: "audio/pcm;rate=16000", data: audioBase64 },
  { mimeType: "image/jpeg", data: frameBase64 },
]);
// Logs: "client.realtimeInput" → "audio + video"
```
```

--------------------------------

### client.sendToolResponse

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Responds to model function calls after a `toolcall` event is emitted. This method maps function call IDs to their return values, allowing your application to provide results for the model's requested actions.

```APIDOC
## client.sendToolResponse — Respond to model function calls

After the model emits a `toolcall` event, your application must respond with results for each function call using `sendToolResponse`. The response maps function call IDs to their return values.

```typescript
import { GenAILiveClient } from "./src/lib/genai-live-client";
import { FunctionDeclaration, Type } from "@google/genai";

const getWeatherDeclaration: FunctionDeclaration = {
  name: "get_weather",
  description: "Returns the current weather for a given city.",
  parameters: {
    type: Type.OBJECT,
    properties: {
      city: { type: Type.STRING, description: "City name" },
    },
    required: ["city"],
  },
};

const client = new GenAILiveClient({ apiKey: "YOUR_API_KEY" });
await client.connect("models/gemini-2.0-flash-exp", {
  responseModalities: ["AUDIO"],
  tools: [{ functionDeclarations: [getWeatherDeclaration] }],
});

// Handle incoming tool calls and respond
client.on("toolcall", (toolCall) => {
  if (!toolCall.functionCalls) return;

  const responses = toolCall.functionCalls.map((fc) => {
    let result: unknown;

    if (fc.name === "get_weather") {
      const city = (fc.args as { city: string }).city;
      // Call your actual weather API here
      result = { temperature: "18°C", condition: "Partly cloudy", city };
    } else {
      result = { error: "Unknown function" };
    }

    return {
      id: fc.id,
      name: fc.name,
      response: { output: result },
    };
  });

  // Small delay to allow model to process before receiving the response
  setTimeout(() => {
    client.sendToolResponse({ functionResponses: responses });
  }, 200);
});

client.send({ text: "What is the weather like in Tokyo?" });
```
```

--------------------------------

### Play PCM16 Audio with AudioStreamer

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

Receives raw PCM16 Uint8Array chunks, converts them to Float32Array, and plays them using the Web Audio API. Supports AudioWorklet nodes for real-time effects and volume metering. Use this to play model audio responses.

```typescript
import { AudioStreamer } from "./src/lib/audio-streamer";
import { audioContext } from "./src/lib/utils";
import VolMeterWorklet from "./src/lib/worklets/vol-meter";

// Get (or create) a shared AudioContext — waits for user interaction if needed
const ctx = await audioContext({ id: "audio-out" });
const streamer = new AudioStreamer(ctx);

// Attach a VU meter worklet for output volume monitoring
await streamer.addWorklet("vumeter-out", VolMeterWorklet, (ev: MessageEvent) => {
  const volume: number = ev.data.volume; // 0–1
  console.log("Output volume:", volume.toFixed(2));
});

// Called when the audio queue drains completely
streamer.onComplete = () => console.log("Playback finished");

// Feed incoming PCM16 data from the model
function onAudio(data: ArrayBuffer) {
  streamer.addPCM16(new Uint8Array(data)); // Buffers and schedules automatically
}

// Adjust output gain
streamer.gainNode.gain.setValueAtTime(0.8, ctx.currentTime);

// Stop immediately (e.g., on "interrupted" event) with a 100 ms fade-out
streamer.stop();

// Resume after a browser-policy suspension
await streamer.resume();
```

--------------------------------

### Declare Altair Function for Vega-Lite Graph Rendering

Source: https://context7.com/google-gemini/live-api-web-console/llms.txt

This TypeScript code declares a function named 'render_altair' that the model can call to display Altair graphs. It specifies that the function expects a single argument, 'json_graph', which is a string representation of the Vega-Lite specification. This is used within a React component to enable dynamic chart generation based on model output.

```typescript
import {
  memo,
  useEffect,
  useRef,
  useState,
} from "react";
import vegaEmbed from "vega-embed";
import { useLiveAPIContext } from "./contexts/LiveAPIContext";
import { FunctionDeclaration, LiveServerToolCall, Modality, Type } from "@google/genai";

// Declare the function the model will call to render charts
const declaration: FunctionDeclaration = {
  name: "render_altair",
  description: "Displays an altair graph in json format.",
  parameters: {
    type: Type.OBJECT,
    properties: {
      json_graph: {
        type: Type.STRING,
        description: "JSON STRING representation of the Vega-Lite spec.",
      },
    },
    required: ["json_graph"],
  },
};

function AltairComponent() {
  const [jsonString, setJSONString] = useState("");
  const embedRef = useRef<HTMLDivElement>(null);
  const { client, setConfig, setModel } = useLiveAPIContext();

  useEffect(() => {
    setModel("models/gemini-2.0-flash-exp");
    setConfig({
      responseModalities: [Modality.AUDIO],
      speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: "Aoede" } } },
      systemInstruction: {
        parts: [{ text: 'Any time a graph is requested, call "render_altair".' }],
      },
      tools: [{ googleSearch: {} }, { functionDeclarations: [declaration] }],
    });
  }, [setConfig, setModel]);

  useEffect(() => {
    const onToolCall = (toolCall: LiveServerToolCall) => {
      const fc = toolCall.functionCalls?.find((f) => f.name === "render_altair");
      if (fc) setJSONString((fc.args as any).json_graph);
      // Acknowledge all function calls
      if (toolCall.functionCalls?.length) {
        setTimeout(() => client.sendToolResponse({
          functionResponses: toolCall.functionCalls!.map((f) => ({
            id: f.id, name: f.name, response: { output: { success: true } },
          })),
        }), 200);
      }
    };
    client.on("toolcall", onToolCall);
    return () => { client.off("toolcall", onToolCall); };
  }, [client]);

  useEffect(() => {
    if (embedRef.current && jsonString) {
      vegaEmbed(embedRef.current, JSON.parse(jsonString));
    }
  }, [jsonString]);

  return <div ref={embedRef} />;
}

export const Altair = memo(AltairComponent);

// Usage: Ask the model "Plot a bar chart of top 5 programming languages by popularity"
// The model calls render_altair({ json_graph: "{ ... vega-lite spec ... }" })
// The component parses and renders the chart automatically.
```

=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.