### Run Frontend
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/nextjs_voice_chat/README.md
Installs frontend dependencies and starts the development server for the Next.js application.
```bash
npm install
npm run dev
```
--------------------------------
### WebRTC and Audio Setup
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/whisper_realtime/index.html
Initializes WebRTC peer connection, gets user media (audio), and sets up audio visualization. Handles connection states and potential connection delays.
```javascript
let peerConnection; let webrtc_id; let audioContext, analyser, audioSource; let audioLevel = 0; let animationFrame; let isMuted = false;
const startButton = document.getElementById('start-button');
const transcriptDiv = document.getElementById('transcript');
// SVG Icons
const micIconSVG =
`
`;
const micMutedIconSVG =
`
`;
function showError(message) {
const toast = document.getElementById('error-toast');
toast.textContent = message;
toast.style.display = 'block';
// Hide toast after 5 seconds
setTimeout(() => {
ttoast.style.display = 'none';
}, 5000);
}
async function handleMessage(event) {
// Handle any WebRTC data channel messages if needed
const eventJson = JSON.parse(event.data);
if (eventJson.type === "error") {
showError(eventJson.message);
} else if (eventJson.type === "send_input") {
const response = await fetch('/send_input', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ webrtc_id: webrtc_id, transcript: "" })
});
}
console.log('Received message:', event.data);
}
function updateButtonState() {
// Remove existing mute listener if present
const existingMuteButton = startButton.querySelector('.mute-toggle');
if (existingMuteButton) {
existingMuteButton.removeEventListener('click', toggleMute);
existingMuteButton.remove();
}
if (peerConnection && (peerConnection.connectionState === 'connecting' || peerConnection.connectionState === 'new')) {
startButton.innerHTML =
`
`;
startButton.disabled = true;
} else if (peerConnection && peerConnection.connectionState === 'connected') {
startButton.innerHTML =
`
${isMuted ? micMutedIconSVG : micIconSVG}
`;
startButton.disabled = false;
const muteButton = startButton.querySelector('.mute-toggle');
if (muteButton) {
muteButton.addEventListener('click', toggleMute);
}
} else {
startButton.innerHTML = 'Start Recording';
startButton.disabled = false;
}
}
function toggleMute(event) {
event.stopPropagation();
if (!peerConnection || peerConnection.connectionState !== 'connected') return;
isMuted = !isMuted;
console.log("Mute toggled:", isMuted);
peerConnection.getSenders().forEach(sender => {
if (sender.track && sender.track.kind === 'audio') {
sender.track.enabled = !isMuted;
console.log(\`Audio track ${sender.track.id} enabled: ${!isMuted}\`)
}
});
updateButtonState();
}
function setupAudioVisualization(stream) {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
audioSource = audioContext.createMediaStreamSource(stream);
audioSource.connect(analyser);
analyser.fftSize = 64;
const dataArray = new Uint8Array(analyser.frequencyBinCount);
function updateAudioLevel() {
analyser.getByteFrequencyData(dataArray);
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
audioLevel = average / 255;
const pulseCircle = document.querySelector('.pulse-circle');
if (pulseCircle) {
pulseCircle.style.setProperty('--audio-level', 1 + audioLevel);
}
animationFrame = requestAnimationFrame(updateAudioLevel);
}
updateAudioLevel();
}
async function setupWebRTC() {
const config = __RTC_CONFIGURATION__;
peerConnection = new RTCPeerConnection(config);
const timeoutId = setTimeout(() => {
const toast = document.getElementById('error-toast');
ttoast.textContent = "Connection is taking longer than usual. Are you on a VPN?";
ttoast.className = 'toast warning';
ttoast.style.display = 'block';
// Hide warning after 5 seconds
setTimeout(() => {
ttoast.style.display = 'none';
}, 5000);
}, 5000);
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
setupAudioVisualization(stream);
stream.getTracks().forEach(track => {
peerConnection.addTrack(track, stream);
});
// Add connection state change listener
peerConnection.addEventListener('connectionstatechange', () => {
consol
```
--------------------------------
### Install Dependencies
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_smolagents/README.md
Install Python 3.9+ and create a virtual environment. Then, install the required Python packages using pip.
```bash
python -m venv .venv
source .venv/bin/activate # On Windows: .venv\Scripts\activate
```
```bash
pip install -r requirements.txt
```
--------------------------------
### Setup and Establish WebRTC Connection
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_sambanova/index.html
Initiates the WebRTC connection process, including getting user media, setting up the peer connection, creating a data channel, and exchanging SDP offers/answers. Handles potential connection errors and server responses.
```javascript
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
setupAudioVisualization(stream);
stream.getTracks().forEach(track => {
peerConnection.addTrack(track, stream);
});
const dataChannel = peerConnection.createDataChannel('text');
dataChannel.onmessage = handleMessage;
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
peerConnection.onicecandidate = ({ candidate }) => {
if (candidate) {
console.debug("Sending ICE candidate", candidate);
fetch('/webrtc/offer', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
candidate: candidate.toJSON(),
webrtc_id: webrtc_id,
type: "ice-candidate",
})
})
}
};
peerConnection.addEventListener('connectionstatechange', () => {
console.log('connectionstatechange', peerConnection.connectionState);
if (peerConnection.connectionState === 'connected') {
clearTimeout(timeoutId);
const toast = document.getElementById('error-toast');
toast.style.display = 'none';
} else if (['closed', 'failed', 'disconnected'].includes(peerConnection.connectionState)) {
stop();
}
updateButtonState();
});
webrtc_id = Math.random().toString(36).substring(7);
const response = await fetch('/webrtc/offer', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
sdp: peerConnection.localDescription.sdp,
type: peerConnection.localDescription.type,
webrtc_id: webrtc_id
})
});
const serverResponse = await response.json();
if (serverResponse.status === 'failed') {
showError(serverResponse.meta.error === 'concurrency_limit_reached' ? `Too many connections. Maximum limit is ${serverResponse.meta.limit}` : serverResponse.meta.error);
stop();
return;
}
await peerConnection.setRemoteDescription(serverResponse);
eventSource = new EventSource('/outputs?webrtc_id=' + webrtc_id);
eventSource.addEventListener("output", (event) => {
const eventJson = JSON.parse(event.data);
console.log(eventJson);
messages.push(eventJson.message);
addMessage(eventJson.message.role, eventJson.audio ?? eventJson.message.content);
});
} catch (err) {
clearTimeout(timeoutId);
console.error('Error setting up WebRTC:', err);
showError('Failed to establish connection. Please try again.');
stop();
}
```
--------------------------------
### Initialize WebRTC and Audio Visualization
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_openai/index.html
Sets up the RTCPeerConnection, gets user media for audio, and initializes audio visualization. Includes error handling for connection timeouts.
```javascript
let peerConnection;
let webrtc_id;
let isMuted = false;
const audioOutput = document.getElementById('audio-output');
const startButton = document.getElementById('start-button');
const chatMessages = document.getElementById('chat-messages');
let audioLevel = 0;
let animationFrame;
let audioContext, analyser, audioSource;
// SVG Icons
const micIconSVG =
`
`;
const micMutedIconSVG =
`
`;
function updateButtonState() {
const button = document.getElementById('start-button');
// Clear previous content
button.innerHTML = '';
if (peerConnection && (peerConnection.connectionState === 'connecting' || peerConnection.connectionState === 'new')) {
const spinner = document.createElement('div');
spinner.className = 'spinner';
const text = document.createElement('span');
text.textContent = 'Connecting...';
button.appendChild(spinner);
button.appendChild(text);
} else if (peerConnection && peerConnection.connectionState === 'connected') {
// Create pulse circle
const pulseCircle = document.createElement('div');
pulseCircle.className = 'pulse-circle';
// Create mic icon
const micIcon = document.createElement('div');
micIcon.className = 'mute-toggle';
micIcon.innerHTML = isMuted ? micMutedIconSVG : micIconSVG;
micIcon.addEventListener('click', toggleMute);
// Create text
const text = document.createElement('span');
text.textContent = 'Stop Conversation';
// Add elements in correct order
button.appendChild(pulseCircle);
button.appendChild(micIcon);
button.appendChild(text);
} else {
const text = document.createElement('span');
text.textContent = 'Start Conversation';
button.appendChild(text);
}
}
function toggleMute(event) {
event.stopPropagation();
if (!peerConnection || peerConnection.connectionState !== 'connected') return;
isMuted = !isMuted;
console.log("Mute toggled:", isMuted);
peerConnection.getSenders().forEach(sender => {
if (sender.track && sender.track.kind === 'audio') {
sender.track.enabled = !isMuted;
console.log(`Audio track ${sender.track.id} enabled: ${!isMuted}`);
}
});
updateButtonState();
}
function setupAudioVisualization(stream) {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
audioSource = audioContext.createMediaStreamSource(stream);
audioSource.connect(analyser);
analyser.fftSize = 64;
const dataArray = new Uint8Array(analyser.frequencyBinCount);
function updateAudioLevel() {
analyser.getByteFrequencyData(dataArray);
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
audioLevel = average / 255; // Update CSS variable instead of rebuilding the button
const pulseCircle = document.querySelector('.pulse-circle');
if (pulseCircle) {
pulseCircle.style.setProperty('--audio-level', 1 + audioLevel);
}
animationFrame = requestAnimationFrame(updateAudioLevel);
}
updateAudioLevel();
}
function showError(message) {
const toast = document.getElementById('error-toast');
toast.textContent = message;
toast.style.display = 'block';
// Hide toast after 5 seconds
setTimeout(() => {
toast.style.display = 'none';
}, 5000);
}
async function setupWebRTC() {
isConnecting = true;
const config = __RTC_CONFIGURATION__;
peerConnection = new RTCPeerConnection(config);
const timeoutId = setTimeout(() => {
const toast = document.getElementById('error-toast');
toast.textContent = "Connection is taking longer than usual. Are you on a VPN?";
toast.className = 'toast warning';
toast.style.display = 'block';
// Hide warning after 5 seconds
setTimeout(() => {
toast.style.display = 'none';
}, 5000);
}, 5000);
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
setupAudioVisualization(stream);
stream.getTracks().forEach(track => {
peerConnection.addTrack(track, stream);
});
peerConnection.addEventListener('track', (evt) => {
if (audioOutput.srcObject !== evt.streams[0]) {
audioOutput.srcObject = evt.streams[0];
audioOutput.play();
}
});
peerConnection.onicecandidate = ({ candidate }) => {
if (candidate) {
console.debug("Sending ICE candidate", candidate);
fetch('/webrtc/offer', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
cand
```
--------------------------------
### Install FastRTC
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/index.md
Install the FastRTC library using pip. For additional features like pause detection, speech-to-text, and text-to-speech, install the respective extras.
```bash
pip install fastrtc
```
```bash
pip install "fastrtc[vad, stt, tts]"
```
--------------------------------
### Setup WebRTC Connection
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/object_detection/index.html
Initiates the WebRTC connection, sends an initial confidence threshold, and sets up event listeners for connection state changes. Handles potential errors during setup, including concurrency limits.
```javascript
async function setupWebRTC() {
try {
const response = await fetch('/webrtc/offer', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ webrtc_id: webrtc_id }) });
const serverResponse = await response.json();
if (serverResponse.status === 'failed') {
showError(serverResponse.meta.error === 'concurrency_limit_reached' ? `Too many connections. Maximum limit is ${serverResponse.meta.limit}` : serverResponse.meta.error);
stop();
startButton.textContent = 'Start';
return;
}
await peerConnection.setRemoteDescription(serverResponse);
// Send initial confidence threshold update
updateConfThreshold(confThreshold.value);
peerConnection.addEventListener('connectionstatechange', () => {
if (peerConnection.connectionState === 'connected') {
clearTimeout(timeoutId);
const toast = document.getElementById('error-toast');
toast.style.display = 'none';
}
});
} catch (err) {
clearTimeout(timeoutId);
console.error('Error setting up WebRTC:', err);
showError('Failed to establish connection. Please try again.');
stop();
startButton.textContent = 'Start';
}
}
```
--------------------------------
### Run Development Server with Next.js
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/nextjs_voice_chat/frontend/fastrtc-demo/README.md
Use one of these commands to start the Next.js development server. Open http://localhost:3000 in your browser to view the application.
```bash
npm run dev
# or
yarn dev
# or
pnpm dev
# or
bun dev
```
--------------------------------
### WebRTC Setup and Message Handling
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/hello_computer/index.html
Sets up the WebRTC connection and handles incoming messages from the server. Includes error handling and connection timeouts.
```javascript
function setupWebRTC() {
try {
eventSource = new EventSource('/stream');
eventSource.onmessage = (event) => {
const eventJson = JSON.parse(event.data);
if (eventJson.audio) {
playAudio(eventJson.audio);
}
addMessage(eventJson.message.role, eventJson.audio ?? eventJson.message.content);
};
eventSource.onerror = (err) => {
console.error('EventSource failed:', err);
eventSource.close();
};
timeoutId = setTimeout(() => {
if (peerConnection.connectionState !== 'connected') {
showError('Connection timed out. Please try again.');
stop();
}
}, 10000);
peerConnection.oniceconnectionstatechange = () => {
if (peerConnection.iceConnectionState === 'failed' || peerConnection.iceConnectionState === 'disconnected' || peerConnection.iceConnectionState === 'closed') {
clearTimeout(timeoutId);
stop();
}
};
peerConnection.ontrack = (event) => {
clearTimeout(timeoutId);
remoteAudio.srcObject = event.streams[0];
updateButtonState();
};
peerConnection.onicecandidate = (event) => {
if (event.candidate) {
// Send ICE candidate to the other peer
}
};
peerConnection.createOffer().then(offer => {
return peerConnection.setLocalDescription(offer);
}).then(() => {
// Send offer to the other peer
});
} catch (err) {
clearTimeout(timeoutId);
console.error('Error setting up WebRTC:', err);
showError('Failed to establish connection. Please try again.');
stop();
}
}
```
--------------------------------
### Install Dependencies
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/nextjs_voice_chat/README.md
Installs project dependencies using pip. Ensure you are in a virtual environment.
```bash
python3 -m venv env
source env/bin/activate
pip install -r requirements.txt
```
--------------------------------
### Create and Configure a Video Stream
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/userguide/streams.md
Example of creating a video stream with a handler that flips the video vertically. Demonstrates setting modality, mode, and additional inputs.
```python
from fastrtc import Stream
import gradio as gr
import numpy as np
def detection(image, slider):
return np.flip(image, axis=0)
stream = Stream(
handler=detection, # (1)
modality="video", # (2)
mode="send-receive", # (3)
additional_inputs=[
gr.Slider(minimum=0, maximum=1, step=0.01, value=0.3) # (4)
],
additional_outputs=None, # (5)
additional_outputs_handler=None # (6)
)
```
--------------------------------
### Install FastRTC
Source: https://github.com/gradio-app/fastrtc/blob/main/README.md
Install the base FastRTC package using pip.
```bash
pip install fastrtc
```
--------------------------------
### Install distil-whisper-FastRTC
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/speech_to_text_gallery.md
Use this command to install the distil-whisper-FastRTC package for plug-and-play STT functionality.
```python
pip install distil-whisper-fastrtc
```
--------------------------------
### Setup WebRTC Connection
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_llama4/index.html
Initiates the WebRTC connection by sending an offer to the server and setting up an event listener for incoming messages. Handles potential connection failures and concurrency limits.
```javascript
async function setupWebRTC() {
try {
const response = await fetch('/webrtc/offer', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sdp: peerConnection.localDescription.sdp, type: peerConnection.localDescription.type, webrtc_id: webrtc_id }) }); const serverResponse = await response.json(); if (serverResponse.status === 'failed') { showError(serverResponse.meta.error === 'concurrency_limit_reached' ? `Too many connections. Maximum limit is ${serverResponse.meta.limit}` : serverResponse.meta.error); stop(); return; } await peerConnection.setRemoteDescription(serverResponse); eventSource = new EventSource('/outputs?webrtc_id=' + webrtc_id); eventSource.addEventListener("output", (event) => { const eventJson = JSON.parse(event.data); console.log(eventJson); messages.push(eventJson.message); addMessage(eventJson.message.role, eventJson.audio ?? eventJson.message.content); })
} catch (err) {
clearTimeout(timeoutId);
console.error('Error setting up WebRTC:', err);
showError('Failed to establish connection. Please try again.');
stop();
}
}
```
--------------------------------
### Run the App with Gradio UI
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_smolagents/README.md
Launch the application with the Gradio interface enabled by setting the MODE environment variable. This starts a local server for the web application.
```bash
MODE=UI python app.py
```
--------------------------------
### Setup WebRTC Peer Connection
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_sambanova/index.html
Initializes a new RTCPeerConnection with provided configuration. Includes a timeout to warn the user if the connection is taking too long.
```javascript
async function setupWebRTC() { const config = __RTC_CONFIGURATION__; peerConnection = new RTCPeerConnection(config); const timeoutId = setTimeout(() => { const toast = document.getElementById('error-toast'); toast.textContent = "Connection is taking longer than usual. Are you on a VPN?"; toast.className = 'toast warning'; toast.style.display = 'block'; // Hide warning after 5 seconds setTimeout(() => { toast.style.display = 'none'; }, 5000); }, 15000); }
```
--------------------------------
### Asynchronous Stream Handler Start Up Method
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/reference/stream_handlers.md
Optional asynchronous startup logic for stream handlers. Must be defined as a coroutine.
```python
async start_up()
```
--------------------------------
### Install FastRTC with VAD and TTS Extras
Source: https://github.com/gradio-app/fastrtc/blob/main/README.md
Install FastRTC with optional 'vad' and 'tts' extras for pause detection and text-to-speech capabilities.
```bash
pip install "fastrtc[vad, tts]"
```
--------------------------------
### Initialize WebRTC and Audio Context
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_gemini/index.html
Sets up the RTCPeerConnection, initializes the AudioContext, and gets user media. Connects the audio stream to an analyser for visualization. Handles connection state changes and potential delays.
```javascript
let peerConnection;
let audioContext;
let dataChannel;
let isRecording = false;
let webrtc_id;
let isMuted = false;
let analyser_input, dataArray_input;
let analyser, dataArray;
let source_input = null;
let source_output = null;
const startButton = document.getElementById('start-button');
const apiKeyInput = document.getElementById('api-key');
const voiceSelect = document.getElementById('voice');
const audioOutput = document.getElementById('audio-output');
const boxContainer = document.querySelector('.box-container');
const numBars = 32;
for (let i = 0; i < numBars; i++) {
const box = document.createElement('div');
box.className = 'box';
boxContainer.appendChild(box);
}
// SVG Icons
const micIconSVG =
`
`;
const micMutedIconSVG =
`
`;
function updateButtonState() {
startButton.innerHTML = '';
startButton.onclick = null;
if (peerConnection && (peerConnection.connectionState === 'connecting' || peerConnection.connectionState === 'new')) {
startButton.innerHTML =
`
`;
startButton.disabled = true;
} else if (peerConnection && peerConnection.connectionState === 'connected') {
const pulseContainer = document.createElement('div');
pulseContainer.className = 'pulse-container';
pulseContainer.innerHTML =
`
Stop Recording
`;
const muteToggle = document.createElement('div');
muteToggle.className = 'mute-toggle';
muteToggle.title = isMuted ? 'Unmute' : 'Mute';
muteToggle.innerHTML = isMuted ? micMutedIconSVG : micIconSVG;
muteToggle.addEventListener('click', toggleMute);
startButton.appendChild(pulseContainer);
startButton.appendChild(muteToggle);
startButton.disabled = false;
} else {
startButton.innerHTML = 'Start Recording';
startButton.disabled = false;
}
}
function showError(message) {
const toast = document.getElementById('error-toast');
toast.textContent = message;
toast.className = 'toast error';
toast.style.display = 'block'; // Hide toast after 5 seconds
setTimeout(() => {
toast.style.display = 'none';
}, 5000);
}
function toggleMute(event) {
event.stopPropagation();
if (!peerConnection || peerConnection.connectionState !== 'connected') return;
isMuted = !isMuted;
console.log("Mute toggled:", isMuted);
peerConnection.getSenders().forEach(sender => {
if (sender.track && sender.track.kind === 'audio') {
sender.track.enabled = !isMuted;
console.log(`Audio track ${sender.track.id} enabled: ${!isMuted}`);
}
});
updateButtonState();
}
async function setupWebRTC() {
const config = __RTC_CONFIGURATION__;
peerConnection = new RTCPeerConnection(config);
webrtc_id = Math.random().toString(36).substring(7);
const timeoutId = setTimeout(() => {
const toast = document.getElementById('error-toast');
toast.textContent = "Connection is taking longer than usual. Are you on a VPN?";
toast.className = 'toast warning';
toast.style.display = 'block'; // Hide warning after 5 seconds
setTimeout(() => {
toast.style.display = 'none';
}, 5000);
}, 5000);
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
stream.getTracks().forEach(track => peerConnection.addTrack(track, stream));
if (!audioContext || audioContext.state === 'closed') {
audioContext = new AudioContext();
}
if (source_input) {
try {
source_input.disconnect();
} catch (e) {
console.warn("Error disconnecting previous input source:", e);
}
source_input = null;
}
source_input = audioContext.createMediaStreamSource(stream);
analyser_input = audioContext.createAnalyser();
source_input.connect(analyser_input);
analyser_input.fftSize = 64;
dataArray_input = new Uint8Array(analyser_input.frequencyBinCount);
updateAudioLevel();
peerConnection.addEventListener('connectionstatechange', () => {
console.log('connectionstatechange', peerConnection.connectionState);
if (peerConnection.connectionState === 'connected') {
clearTimeout(timeoutId);
const toast = document.getElementById('error-toast');
toast.style.display = 'none';
if (analyser_input) updateAudioLevel();
if (analyser) updateVisualization();
} else if (['disconnected', 'failed', 'closed'].includes(peer
```
--------------------------------
### Setup and Handle WebRTC Connection
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_claude/index.html
Initiates a WebRTC connection, handles server responses, and sets up an event stream for incoming messages. Includes error handling for connection failures and concurrency limits.
```javascript
async function setupWebRTC() {
const webrtc_id = uuidv4();
try {
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
const response = await fetch('/offer', { method: 'POST', body: JSON.stringify({ offer, webrtc_id: webrtc_id }) });
const serverResponse = await response.json();
if (serverResponse.status === 'failed') {
showError(serverResponse.meta.error === 'concurrency_limit_reached' ? `Too many connections. Maximum limit is ${serverResponse.meta.limit}` : serverResponse.meta.error);
stop();
return;
}
await peerConnection.setRemoteDescription(serverResponse);
// Start visualization updateVisualization();
// create event stream to receive messages from /output
const eventSource = new EventSource('/outputs?webrtc_id=' + webrtc_id);
eventSource.addEventListener("output", (event) => {
const eventJson = JSON.parse(event.data);
addMessage(eventJson.role, eventJson.content);
});
} catch (err) {
clearTimeout(timeoutId);
console.error('Error setting up WebRTC:', err);
showError('Failed to establish connection. Please try again.');
stop();
}
}
```
--------------------------------
### Run Server
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/nextjs_voice_chat/README.md
Executes the FastAPI server using a shell script. This command starts the backend of the application.
```bash
./run.sh
```
--------------------------------
### Start/Stop Button Event Listener
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_claude/index.html
Handles the click event for the start/stop button. Calls `setupWebRTC` if the button text is 'Start', otherwise calls `stop`.
```javascript
startButton.addEventListener('click', () => {
if (startButton.textContent === 'Start') {
setupWebRTC();
} else {
stop();
}
});
```
--------------------------------
### Setup WebRTC Connection
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/userguide/webrtc_docs.md
Initializes an RTCPeerConnection and sets up the WebRTC connection, including handling media streams and data channels. This function should be called to establish communication with the server.
```javascript
const pc = new RTCPeerConnection();
const audio_output_component = document.getElementById("audio_output_component_id");
const video_output_component = document.getElementById("video_output_component_id");
async function setupWebRTC(peerConnection) {
// Get audio stream from webcam
const stream = await navigator.mediaDevices.getUserMedia({
audio: true,
})
// Send audio stream to server
stream.getTracks().forEach(async (track) => {
const sender = pc.addTrack(track, stream);
})
peerConnection.addEventListener("track", (evt) => {
if (audio_output_component &&
audio_output_component.srcObject !== evt.streams[0]) {
audio_output_component.srcObject = evt.streams[0];
}
});
// Create data channel (needed!)
const dataChannel = peerConnection.createDataChannel("text");
// Create and send offer
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
// Send offer to server
const response = await fetch('/webrtc/offer', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sdp: offer.sdp,
type: offer.type,
webrtc_id: Math.random().toString(36).substring(7)
})
});
// Handle server response
const serverResponse = await response.json();
await peerConnection.setRemoteDescription(serverResponse);
}
```
```javascript
const pc = new RTCPeerConnection();
const audio_output_component = document.getElementById("audio_output_component_id");
const video_output_component = document.getElementById("video_output_component_id");
async function setupWebRTC(peerConnection) {
// Get audio stream from webcam
const stream = await navigator.mediaDevices.getUserMedia({
audio: true,
})
// Receive audio stream from server
pc.addTransceiver(audio, { direction: "recvonly" })
peerConnection.addEventListener("track", (evt) => {
if (audio_output_component &&
audio_output_component.srcObject !== evt.streams[0]) {
audio_output_component.srcObject = evt.streams[0];
}
});
// Create data channel (needed!)
const dataChannel = peerConnection.createDataChannel("text");
// Create and send offer
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
// Send offer to server
const response = await fetch('/webrtc/offer', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sdp: offer.sdp,
type: offer.type,
webrtc_id: Math.random().toString(36).substring(7)
})
});
// Handle server response
const serverResponse = await response.json();
await peerConnection.setRemoteDescription(serverResponse);
}
```
```javascript
const pc = new RTCPeerConnection();
const audio_output_component = document.getElementById("audio_output_component_id");
const video_output_component = document.getElementById("video_output_component_id");
async function setupWebRTC(peerConnection) {
// Get video stream from webcam
const stream = await navigator.mediaDevices.getUserMedia({
video: true,
})
// Send video stream to server
stream.getTracks().forEach(async (track) => {
const sender = pc.addTrack(track, stream);
})
peerConnection.addEventListener("track", (evt) => {
if (video_output_component &&
video_output_component.srcObject !== evt.streams[0]) {
video_output_component.srcObject = evt.streams[0];
}
});
// Create data channel (needed!)
const dataChannel = peerConnection.createDataChannel("text");
// Create and send offer
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
// Send offer to server
const response = await fetch('/webrtc/offer', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sdp: offer.sdp,
type: offer.type,
webrtc_id: Math.random().toString(36).substring(7)
})
});
// Handle server response
const serverResponse = await response.json();
await peerConnection.setRemoteDescription(serverResponse);
}
```
```javascript
const pc = new RTCPeerConnection();
const audio_output_component = document.getElementById("audio_output_component_id");
const video_output_component = document.getElementById("video_output_component_id");
async function setupWebRTC(peerConnection) {
// Get video stream from webcam
const stream = await navigator.mediaDevices.getUserMedia({
video: true,
})
// Receive video stream from server
pc.addTransceiver(video, { direction: "recvonly" })
peerConnection.addEventListener("track", (evt) => {
if (video_output_component &&
video_output_component.srcObject !== evt.streams[0]) {
video_output_component.srcObject = evt.streams[0];
}
});
// Create data channel (needed!)
const dataChannel = peerConnection.createDataChannel("text");
// Create and send offer
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
// Send offer to server
const response = await fetch('/webrtc/offer', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sdp: offer.sdp,
type: offer.type,
webrtc_id: Math.random().toString(36).substring(7)
})
});
// Handle server response
const serverResponse = await response.json();
await peerConnection.setRemoteDescription(serverResponse);
}
```
--------------------------------
### Audio Visualization Setup
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/send_text_or_audio/index.html
Configures the AudioContext and analyser to visualize audio levels from a media stream. Updates the UI element's style based on the detected audio level.
```javascript
function setupAudioVisualization(stream) {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
audioSource = audioContext.createMediaStreamSource(stream);
audioSource.connect(analyser);
analyser.fftSize = 64;
const dataArray = new Uint8Array(analyser.frequencyBinCount);
function updateAudioLevel() {
analyser.getByteFrequencyData(dataArray);
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
audioLevel = average / 255;
const pulseCircle = document.querySelector('.pulse-circle');
if (pulseCircle) {
pulseCircle.style.setProperty('--audio-level', 1 + audioLevel);
}
animationFrame = requestAnimationFrame(updateAudioLevel);
}
updateAudioLevel();
}
```
--------------------------------
### Configure Gradio WebRTC
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/deployment.md
Example of configuring the Gradio WebRTC component with TURN server details. Replace placeholders with your actual TURN server IP, username, and password.
```python
from fastrtc import Stream
rtc_configuration = {
"iceServers": [
{
"urls": "turn:35.173.254.80:80",
"username": "",
"credential": ""
},
]
}
Stream(
handler=...,
rtc_configuration=rtc_configuration,
modality="audio",
mode="send-receive"
)
```
--------------------------------
### Launch FastPhone Service
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/reference/stream.md
Launch the FastPhone service for telephone integration. This starts a local server, mounts the stream, creates a public tunnel, and registers it to provide a phone number for interaction.
```python
fastphone(token="YOUR_HF_TOKEN", host="127.0.0.1", port=8000)
```
--------------------------------
### StreamHandler.start_up
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/reference/stream_handlers.md
Optional synchronous startup logic.
```APIDOC
## start_up
### Description
Optional synchronous startup logic.
### Method
`start_up`
### Parameters
#### Path Parameters
None
#### Query Parameters
None
#### Request Body
None
### Request Example
```python
start_up()
```
### Response
#### Success Response
None
#### Response Example
None
```
--------------------------------
### Start/Stop Button Event Listener
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/send_text_or_audio/index.html
Handles the click event for the start button. It either initiates the WebRTC setup if not connected or stops the current connection.
```javascript
startButton.addEventListener('click', () => { if (!peerConnection || peerConnection.connectionState !== 'connected') { setupWebRTC(); } else { stop(); } });
```
--------------------------------
### Setup Input Audio Visualization (JavaScript)
Source: https://github.com/gradio-app/fastrtc/blob/main/demo/talk_to_llama4/index.html
Initializes an AudioContext and AnalyserNode to visualize input audio levels. Connects a media stream source and configures FFT size. Updates a visual element based on average audio frequency data.
```javascript
audioContext_input = new (window.AudioContext || window.webkitAudioContext)();
analyser_input = audioContext_input.createAnalyser();
audioSource_input = audioContext_input.createMediaStreamSource(stream);
audioSource_input.connect(analyser_input);
analyser_input.fftSize = 64;
dataArray_input = new Uint8Array(analyser_input.frequencyBinCount);
function updateAudioLevel() {
// Update input audio visualization (pulse circle)
analyser_input.getByteFrequencyData(dataArray_input);
const average = Array.from(dataArray_input).reduce((a, b) => a + b, 0) / dataArray_input.length;
audioLevel = average / 255;
const pulseCircle = document.querySelector('.pulse-circle');
if (pulseCircle) {
pulseCircle.style.setProperty('--audio-level', 1 + audioLevel);
}
animationFrame_input = requestAnimationFrame(updateAudioLevel);
}
updateAudioLevel();
```
--------------------------------
### Get TTS Model (Synchronous)
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/userguide/audio.md
Import and use the `get_tts_model` function for synchronous text-to-speech generation. Ensure the 'tts' extra is installed. This returns the complete audio data at once.
```python
from fastrtc import get_tts_model
model = get_tts_model(model="kokoro")
audio = model.tts("Hello, world!")
```
--------------------------------
### start_up
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/reference/reply_on_pause.md
Executes the startup function `startup_fn` if provided. This method waits for additional arguments if needed before calling `startup_fn`.
```APIDOC
## start_up
### Description
Executes the startup function `startup_fn` if provided. Waits for additional arguments if needed before calling `startup_fn`.
### Method
```python
start_up()
```
```
--------------------------------
### Get and Stream TTS Model (Async)
Source: https://github.com/gradio-app/fastrtc/blob/main/docs/userguide/audio.md
Import and use the `get_tts_model` function for asynchronous streaming of text-to-speech. Ensure the 'tts' extra is installed. This method is suitable for non-blocking operations.
```python
from fastrtc import get_tts_model
model = get_tts_model(model="kokoro")
async for audio in model.stream_tts("Hello, world!"):
yield audio
```