### Install ImageBind with Conda

Source: https://github.com/facebookresearch/imagebind/blob/main/README.md

Set up a conda environment and install ImageBind and its dependencies. Ensure PyTorch 2.0+ is installed.

```shell
conda create --name imagebind python=3.10 -y
conda activate imagebind

pip install .
```

--------------------------------

### Install soundfile for Windows Users

Source: https://github.com/facebookresearch/imagebind/blob/main/README.md

If you are using Windows, you may need to install the 'soundfile' library for audio file operations.

```shell
pip install soundfile
```

--------------------------------

### Extract and Compare Features Across Modalities

Source: https://github.com/facebookresearch/imagebind/blob/main/README.md

This Python script demonstrates how to load the ImageBind model, process text, image, and audio data, extract embeddings, and compute similarity scores between modalities. Ensure you have the necessary data files (e.g., .assets/dog_image.jpg) and PyTorch installed.

```python
from imagebind import data
import torch
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType

text_list=["A dog.", "A car", "A bird"]
image_paths=[ ".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
audio_paths=[ ".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]

device = "cuda:0" if torch.cuda.is_available() else "cpu"

# Instantiate model
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()
model.to(device)

# Load data
inputs = {
    ModalityType.TEXT: data.load_and_transform_text(text_list, device),
    ModalityType.VISION: data.load_and_transform_vision_data(image_paths, device),
    ModalityType.AUDIO: data.load_and_transform_audio_data(audio_paths, device),
}

with torch.no_grad():
    embeddings = model(inputs)

print(
    "Vision x Text: ",
    torch.softmax(embeddings[ModalityType.VISION] @ embeddings[ModalityType.TEXT].T, dim=-1),
)
print(
    "Audio x Text: ",
    torch.softmax(embeddings[ModalityType.AUDIO] @ embeddings[ModalityType.TEXT].T, dim=-1),
)
print(
    "Vision x Audio: ",
    torch.softmax(embeddings[ModalityType.VISION] @ embeddings[ModalityType.AUDIO].T, dim=-1),
)

# Expected output:
#
# Vision x Text:
# tensor([[9.9761e-01, 2.3694e-03, 1.8612e-05],
#         [3.3836e-05, 9.9994e-01, 2.4118e-05],
#         [4.7997e-05, 1.3496e-02, 9.8646e-01]])
#
# Audio x Text:
# tensor([[1., 0., 0.],
#         [0., 1., 0.],
#         [0., 0., 1.]])
#
# Vision x Audio:
# tensor([[0.8070, 0.1088, 0.0842],
#         [0.1036, 0.7884, 0.1079],
#         [0.0018, 0.0022, 0.9960]])

```

--------------------------------

### Initialize the ImageBind Model

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Loads the pretrained huge model configuration and moves it to the appropriate device.

```python
import torch
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType

# Load the pretrained model
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()

# Move to GPU if available
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model.to(device)

# Model architecture details:
# - Vision encoder: 1280 dim, 32 transformer blocks, 16 attention heads
# - Text encoder: 1024 dim, 24 transformer blocks, 16 attention heads
# - Audio encoder: 768 dim, 12 transformer blocks, 12 attention heads
# - Output embedding dimension: 1024 (shared across all modalities)

print(f"Model loaded on {device}")
print(f"Output embedding dimension: 1024")
```

--------------------------------

### Initialize ImageBind Model

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Initializes the ImageBind model for inference on a specified device. This is a prerequisite for computing cross-modal similarities.

```python
import torch
from imagebind import data
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType

# Setup model
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()
model.to(device)
```

--------------------------------

### Compose Embeddings: Image + Audio for Text Matching

Source: https://context7.com/facebookresearch/imagebind/llms.txt

This snippet demonstrates how to compose embeddings from different modalities (image and audio) to find the best matching text description. Ensure you have the necessary data files (e.g., beach.jpg, waves.wav) and text candidates.

```python
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()
model.to(device)

# Components for composition
image_path = ["beach.jpg"]  # Visual concept: beach
audio_path = ["waves.wav"]   # Audio concept: ocean waves

# Text candidates to match against composed embedding
text_candidates = [
    "A quiet mountain landscape",
    "A beach with ocean waves",
    "A busy city street",
    "A forest with birds singing"
]

# Load all modalities
inputs = {
    ModalityType.VISION: data.load_and_transform_vision_data(image_path, device),
    ModalityType.AUDIO: data.load_and_transform_audio_data(audio_path, device),
    ModalityType.TEXT: data.load_and_transform_text(text_candidates, device),
}

with torch.no_grad():
    embeddings = model(inputs)

    # Compose embeddings: image + audio
    composed_embedding = embeddings[ModalityType.VISION] + embeddings[ModalityType.AUDIO]

    # Normalize the composed embedding
    composed_embedding = torch.nn.functional.normalize(composed_embedding, dim=-1)

    # Find best matching text
    similarity = composed_embedding @ embeddings[ModalityType.TEXT].T
    probs = torch.softmax(similarity, dim=-1)

    top_prob, top_idx = probs[0].max(dim=0)

print("Embedding Composition: Image(beach) + Audio(waves)")
print(f"\nBest match: {text_candidates[top_idx]}")
print(f"Confidence: {top_prob:.2%}")
print("\nAll similarities:")
for text, prob in zip(text_candidates, probs[0]):
    print(f"  {text}: {prob:.2%}")
```

--------------------------------

### Execute Audio-to-Image Retrieval

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Retrieve images from a database based on their similarity to an audio query.

```python
import torch
from imagebind import data
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType

# Setup
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()
model.to(device)

# Database of images to search
image_database = [
    "beach_sunset.jpg",
    "dog_playing.jpg",
    "city_traffic.jpg",
    "forest_birds.jpg",
    "concert_crowd.jpg"
]

# Audio query
audio_query = ["dog_barking.wav"]

# Load inputs
inputs = {
    ModalityType.VISION: data.load_and_transform_vision_data(image_database, device),
    ModalityType.AUDIO: data.load_and_transform_audio_data(audio_query, device),
}

# Compute embeddings and find best matching image
with torch.no_grad():
    embeddings = model(inputs)

    # Compute audio-to-image similarity
    similarity = embeddings[ModalityType.AUDIO] @ embeddings[ModalityType.VISION].T

    # Rank images by similarity to audio query
    scores, indices = similarity[0].sort(descending=True)

print("Audio-to-Image Retrieval Results:")
print(f"Query audio: {audio_query[0]}")
print("\nRanked images:")
for rank, (score, idx) in enumerate(zip(scores, indices), 1):
    print(f"  {rank}. {image_database[idx]} (score: {score:.4f})")
```

--------------------------------

### Preprocess Vision Data

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Loads images from paths and applies standard resizing, cropping, and normalization transforms.

```python
from imagebind import data
import torch

device = "cuda:0" if torch.cuda.is_available() else "cpu"

# Load multiple images from file paths
image_paths = [
    "path/to/dog.jpg",
    "path/to/cat.jpg",
    "path/to/car.jpg"
]

# Transform images to model input format
# Output shape: (batch_size, 3, 224, 224)
vision_data = data.load_and_transform_vision_data(image_paths, device)

print(f"Vision tensor shape: {vision_data.shape}")
# Expected output: Vision tensor shape: torch.Size([3, 3, 224, 224])

# Preprocessing applied:
# 1. Resize to 224px (shortest side) with bicubic interpolation
# 2. Center crop to 224x224
# 3. Convert to tensor (0-1 range)
# 4. Normalize with mean=(0.48145466, 0.4578275, 0.40821073)
#                 std=(0.26862954, 0.26130258, 0.27577711)
```

--------------------------------

### Batch Processing Multiple Samples for Image Retrieval

Source: https://context7.com/facebookresearch/imagebind/llms.txt

This snippet shows how to process a large batch of image samples efficiently using batching to manage memory. It then builds a simple image retrieval index to find similar images based on their embeddings. Ensure image files (image_0.jpg to image_99.jpg) are available.

```python
import torch
from imagebind import data
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType

# Setup
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()
model.to(device)

# Large batch of images
image_paths = [f"image_{i}.jpg" for i in range(100)]

# Process in batches to manage memory
batch_size = 16
all_embeddings = []

for i in range(0, len(image_paths), batch_size):
    batch_paths = image_paths[i:i + batch_size]

    inputs = {
        ModalityType.VISION: data.load_and_transform_vision_data(batch_paths, device)
    }

    with torch.no_grad():
        embeddings = model(inputs)
        all_embeddings.append(embeddings[ModalityType.VISION].cpu())

# Concatenate all batch embeddings
image_embeddings = torch.cat(all_embeddings, dim=0)
print(f"Processed {len(image_paths)} images")
print(f"Embedding matrix shape: {image_embeddings.shape}")
# Expected: torch.Size([100, 1024])

# Build a simple image retrieval index
def retrieve_similar_images(query_idx, embeddings, top_k=5):
    query = embeddings[query_idx:query_idx+1]
    similarities = (query @ embeddings.T).squeeze(0)
    similarities[query_idx] = -float('inf')  # Exclude self
    top_indices = similarities.argsort(descending=True)[:top_k]
    return top_indices, similarities[top_indices]

# Find images similar to image 0
similar_indices, scores = retrieve_similar_images(0, image_embeddings)
print(f"\nImages most similar to image_0.jpg:")
for idx, score in zip(similar_indices, scores):
    print(f"  image_{idx}.jpg (similarity: {score:.4f})")
```

--------------------------------

### Load and Transform Audio Data

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Processes audio files by resampling to 16kHz and extracting mel spectrograms. Requires a list of file paths and a target device.

```python
from imagebind import data
import torch

device = "cuda:0" if torch.cuda.is_available() else "cpu"

# List of audio file paths
audio_paths = [
    "path/to/dog_barking.wav",
    "path/to/car_engine.wav",
    "path/to/music.wav"
]

# Load and transform audio data with default parameters
# - Resamples to 16kHz
# - Extracts 128 mel bins
# - Creates 3 clips of 2 seconds each per audio
audio_data = data.load_and_transform_audio_data(
    audio_paths,
    device,
    num_mel_bins=128,      # Number of mel frequency bins
    target_length=204,      # Target spectrogram length in frames
    sample_rate=16000,      # Target sample rate
    clip_duration=2,        # Duration of each clip in seconds
    clips_per_video=3,      # Number of clips to extract
    mean=-4.268,            # Normalization mean
    std=9.138               # Normalization std
)

print(f"Audio tensor shape: {audio_data.shape}")
# Expected output: Audio tensor shape: torch.Size([3, 3, 1, 128, 204])
# Shape: (batch, num_clips, channels, mel_bins, time_frames)
```

--------------------------------

### Load and Transform Video Data

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Loads video files and applies temporal clipping and spatial augmentation. Uses the decord decoder for frame sampling.

```python
from imagebind import data
import torch

device = "cuda:0" if torch.cuda.is_available() else "cpu"

# List of video file paths
video_paths = [
    "path/to/video1.mp4",
    "path/to/video2.mp4"
]

# Load and transform video data
# - Extracts 5 temporal clips of 2 seconds each
# - Applies 3 spatial crops per temporal clip (left, center, right)
# - Results in 15 views per video (5 temporal x 3 spatial)
video_data = data.load_and_transform_video_data(
    video_paths,
    device,
    clip_duration=2,       # Duration of each clip in seconds
    clips_per_video=5,     # Number of temporal clips
    sample_rate=16000      # Audio sample rate (used for video loading)
)

print(f"Video tensor shape: {video_data.shape}")
# Expected output: Video tensor shape: torch.Size([2, 15, 3, 224, 224])
# Shape: (batch, num_views, channels, height, width)
# num_views = clips_per_video * num_spatial_crops = 5 * 3 = 15
```

--------------------------------

### Use ModalityType Constants

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Defines the supported input modalities for model interaction and dictionary-based input mapping.

```python
from imagebind.models.imagebind_model import ModalityType

# Available modality types
print(ModalityType.VISION)   # "vision" - for images and video frames
print(ModalityType.TEXT)     # "text" - for text strings
print(ModalityType.AUDIO)    # "audio" - for audio waveforms
print(ModalityType.THERMAL)  # "thermal" - for thermal/infrared images
print(ModalityType.DEPTH)    # "depth" - for depth maps
print(ModalityType.IMU)      # "imu" - for IMU sensor data (accelerometer/gyroscope)

# Use as dictionary keys for model inputs
inputs = {
    ModalityType.VISION: vision_tensor,
    ModalityType.TEXT: text_tensor,
    ModalityType.AUDIO: audio_tensor,
}
```

--------------------------------

### Preprocess Text Data

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Tokenizes text strings into tensors using BPE encoding with a fixed context length.

```python
from imagebind import data
import torch

device = "cuda:0" if torch.cuda.is_available() else "cpu"

# List of text descriptions
text_list = [
    "A photo of a dog",
    "A red sports car driving fast",
    "A beautiful sunset over the ocean",
    "A person playing guitar"
]

# Tokenize text inputs
# Output shape: (batch_size, 77) - 77 is the context length
text_data = data.load_and_transform_text(text_list, device)

print(f"Text tensor shape: {text_data.shape}")
# Expected output: Text tensor shape: torch.Size([4, 77])

# Tokenization process:
```

--------------------------------

### Compute Cross-Modal Similarities

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Calculate similarity matrices between different modalities using matrix multiplication and softmax normalization.

```python
text_list = ["A dog.", "A car", "A bird"]
image_paths = ["dog_image.jpg", "car_image.jpg", "bird_image.jpg"]
audio_paths = ["dog_audio.wav", "car_audio.wav", "bird_audio.wav"]

inputs = {
    ModalityType.TEXT: data.load_and_transform_text(text_list, device),
    ModalityType.VISION: data.load_and_transform_vision_data(image_paths, device),
    ModalityType.AUDIO: data.load_and_transform_audio_data(audio_paths, device),
}

with torch.no_grad():
    embeddings = model(inputs)

# Compute cross-modal similarities using matrix multiplication
# Vision-Text similarity: which text best matches each image?
vision_text_sim = torch.softmax(
    embeddings[ModalityType.VISION] @ embeddings[ModalityType.TEXT].T,
    dim=-1
)
print("Vision x Text similarity matrix:")
print(vision_text_sim)

# Audio-Text similarity: which text best matches each audio?
audio_text_sim = torch.softmax(
    embeddings[ModalityType.AUDIO] @ embeddings[ModalityType.TEXT].T,
    dim=-1
)
print("\nAudio x Text similarity matrix:")
print(audio_text_sim)

# Vision-Audio similarity: which audio best matches each image?
vision_audio_sim = torch.softmax(
    embeddings[ModalityType.VISION] @ embeddings[ModalityType.AUDIO].T,
    dim=-1
)
print("\nVision x Audio similarity matrix:")
print(vision_audio_sim)
```

--------------------------------

### Compute Cross-Modal Embeddings

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Generates embeddings for multiple modalities by passing transformed inputs through the ImageBind model. Embeddings are L2-normalized and exist in a shared 1024-dimensional space.

```python
import torch
from imagebind import data
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType

# Setup
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()
model.to(device)

# Prepare inputs for multiple modalities
text_list = ["A dog", "A car", "A bird"]
image_paths = ["dog.jpg", "car.jpg", "bird.jpg"]
audio_paths = ["dog_bark.wav", "car_engine.wav", "bird_chirp.wav"]

# Load and transform all modalities
inputs = {
    ModalityType.TEXT: data.load_and_transform_text(text_list, device),
    ModalityType.VISION: data.load_and_transform_vision_data(image_paths, device),
    ModalityType.AUDIO: data.load_and_transform_audio_data(audio_paths, device),
}

# Compute embeddings (disable gradients for inference)
with torch.no_grad():
    embeddings = model(inputs)

# Access embeddings for each modality
text_embeddings = embeddings[ModalityType.TEXT]      # Shape: (3, 1024)
vision_embeddings = embeddings[ModalityType.VISION]  # Shape: (3, 1024)
audio_embeddings = embeddings[ModalityType.AUDIO]    # Shape: (3, 1024)

print(f"Text embeddings shape: {text_embeddings.shape}")
print(f"Vision embeddings shape: {vision_embeddings.shape}")
print(f"Audio embeddings shape: {audio_embeddings.shape}")
# All embeddings are L2-normalized and in the same 1024-dim space
```

--------------------------------

### Perform Zero-Shot Classification

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Classify inputs by computing the similarity between vision embeddings and text embeddings of class labels.

```python
import torch
from imagebind import data
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType

# Setup
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval()
model.to(device)

# Define class labels as text
class_labels = [
    "a photo of a dog",
    "a photo of a cat",
    "a photo of a car",
    "a photo of a plane",
    "a photo of a bird",
    "a photo of a person"
]

# Image to classify
image_paths = ["unknown_image.jpg"]

# Load inputs
inputs = {
    ModalityType.TEXT: data.load_and_transform_text(class_labels, device),
    ModalityType.VISION: data.load_and_transform_vision_data(image_paths, device),
}

# Compute embeddings and classify
with torch.no_grad():
    embeddings = model(inputs)

    # Compute similarity between image and all class labels
    similarity = embeddings[ModalityType.VISION] @ embeddings[ModalityType.TEXT].T
    probs = torch.softmax(similarity, dim=-1)

    # Get top prediction
    top_prob, top_idx = probs[0].max(dim=0)
    predicted_class = class_labels[top_idx]

print(f"Predicted: {predicted_class} (confidence: {top_prob:.2%})")
print(f"\nAll class probabilities:")
for label, prob in zip(class_labels, probs[0]):
    print(f"  {label}: {prob:.2%}")
```

--------------------------------

### Perform Embedding Arithmetic

Source: https://context7.com/facebookresearch/imagebind/llms.txt

Combine embeddings arithmetically to create composed queries across modalities.

```python
import torch
from imagebind import data
from imagebind.models import imagebind_model
from imagebind.models.imagebind_model import ModalityType
```

=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.