### Load PhraseCut Dataset

Source: https://context7.com/timojl/clipseg/llms.txt

Demonstrates loading the PhraseCut and PhraseCutPlus datasets for training or evaluation. Configure parameters like image size, negative probability, and augmentation. The DataLoader setup is also shown.

```python
from datasets.phrasecut import PhraseCut, PhraseCutPlus
from torch.utils.data import DataLoader

# Basic PhraseCut for text-only conditioning
dataset = PhraseCut(
    split='train',           # 'train', 'val', or 'test'
    image_size=352,          # Output image size
    negative_prob=0.0,       # Probability of negative samples
    aug_crop=True,           # Enable random cropping
    aug_color=False,         # Enable color augmentation
    mask='text'              # Conditioning type: 'text', 'crop_blur_highlight352'
)

# PhraseCut with visual support for one-shot learning
dataset_visual = PhraseCut(
    split='train',
    image_size=352,
    with_visual=True,        # Include visual support images
    only_visual=False,       # If True, only samples with visual support
    mask='text_and_separate' # Provide text and visual separately
)

# PhraseCutPlus - extended dataset with visual prompts and negatives
dataset_plus = PhraseCutPlus(
    split='train',
    image_size=352,
    mask='text_and_crop_blur_highlight352'
)

# DataLoader setup
loader = DataLoader(dataset, batch_size=64, num_workers=4, shuffle=True)

for data_x, data_y in loader:
    img = data_x[0]           # [B, 3, H, W] normalized image
    phrase = data_x[1]        # List of text phrases or tensor
    seg = data_y[0]           # [B, 1, H, W] ground truth segmentation

    # For visual conditioning (with_visual=True, mask='text_and_separate')
    # data_x = (img, phrase, support_img, support_mask, is_valid)
    break
```

--------------------------------

### Visual Prompt Segmentation

Source: https://context7.com/timojl/clipseg/llms.txt

Performs segmentation using a query image and a reference image with its mask. This variant supports one-shot learning from visual examples.

```python
import torch
from models.clipseg import CLIPDensePredTMasked

# Initialize masked model for visual prompts
model = CLIPDensePredTMasked(
    version='ViT-B/16',
    reduce_dim=64,
    extract_layers=(3, 6, 9)
)
model.eval()
model.load_state_dict(torch.load('weights/rd64-uni.pth'), strict=False)

# Prepare query image (image to segment)
query_image = transform(Image.open('query.jpg')).unsqueeze(0)

# Prepare support image with mask (reference showing what to segment)
support_image = transform(Image.open('support.jpg')).unsqueeze(0)
support_mask = torch.tensor(load_mask('support_mask.png')).float()  # Binary mask

# Segment query using visual reference
with torch.no_grad():
    # img_q: query image, img_s: support image, seg_s: support mask
    pred = model(query_image, support_image, support_mask)[0]
    segmentation = torch.sigmoid(pred[0][0])

```

--------------------------------

### Initialize CLIP and CLIPSeg Models

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Loads the CLIP model and the CLIPSeg masked model. Ensure CUDA is available for GPU acceleration. Models are set to evaluation mode.

```python
%load_ext autoreload
%autoreload 2

import clip
from evaluation_utils import norm, denorm
from general_utils import *
from datasets.lvis_oneshot3 import LVIS_OneShot3

clip_device = 'cuda'
clip_model, preprocess = clip.load("ViT-B/16", device=clip_device)
clip_model.eval();

from models.clipseg import CLIPDensePredTMasked

clip_mask_model = CLIPDensePredTMasked(version='ViT-B/16').to(clip_device)
clip_mask_model.eval();
```

--------------------------------

### Training Configuration

Source: https://context7.com/timojl/clipseg/llms.txt

Sets up training for custom CLIPSeg models using YAML experiment files or programmatically. Defines model, dataset, optimizer, and loss parameters.

```python
# Command line training
# python training.py phrasecut.yaml 0
```

```yaml
# Experiment configuration (experiments/phrasecut.yaml)
configuration:
  batch_size: 64
  optimizer: torch.optim.AdamW
  lr: 0.001
  lr_scheduler: cosine
  T_max: 20000
  eta_min: 0.0001
  max_iterations: 20000

  model: models.clipseg.CLIPDensePredT
  dataset: datasets.phrasecut.PhraseCut
  split: train
  image_size: 352

  # Model configuration
  version: 'ViT-B/16'
  extract_layers: [3, 7, 9]
  reduce_dim: 64
  prompt: shuffle+

  loss: torch.nn.functional.binary_cross_entropy_with_logits
  amp: True  # Automatic mixed precision

individual_configurations:
  - name: rd64-uni
    version: 'ViT-B/16'
    reduce_dim: 64
    with_visual: True
    negative_prob: 0.2
    mix: True
    mix_text_max: 0.5
```

```python
# Programmatic training setup
from training import main, validate
from general_utils import training_config_from_cli_args, get_attribute, filter_args
import inspect

config = training_config_from_cli_args()

# Initialize model
model_cls = get_attribute(config.model)
_, model_args, _ = filter_args(config, inspect.signature(model_cls).parameters)
model = model_cls(**model_args).cuda()

# Initialize dataset
dataset_cls = get_attribute(config.dataset)
_, dataset_args, _ = filter_args(config, inspect.signature(dataset_cls).parameters)
dataset = dataset_cls(**dataset_args)
```

--------------------------------

### Load CLIPDensePredT Model and Weights

Source: https://github.com/timojl/clipseg/blob/master/Quickstart.ipynb

Initializes the CLIPDensePredT model and loads pre-trained decoder weights. Ensure weights are downloaded and unzipped before running.

```python
import torch
import requests

! wget https://owncloud.gwdg.de/index.php/s/ioHbRzFx6th32hn/download -O weights.zip
! unzip -d weights -j weights.zip
from models.clipseg import CLIPDensePredT
from PIL import Image
from torchvision import transforms
from matplotlib import pyplot as plt

# load model
model = CLIPDensePredT(version='ViT-B/16', reduce_dim=64)
model.eval();

# non-strict, because we only stored decoder weights (not CLIP weights)
model.load_state_dict(torch.load('weights/rd64-uni.pth', map_location=torch.device('cpu')), strict=False);
```

--------------------------------

### Prompt Template Configuration

Source: https://context7.com/timojl/clipseg/llms.txt

Configures text prompt templates for the CLIPSeg model. Demonstrates retrieving available prompt lists and setting prompt modes during initialization or modification.

```python
from models.clipseg import get_prompt_list, CLIPDensePredT

# Available prompt modes
plain_prompts = get_prompt_list('plain')      # ['{}']
fixed_prompts = get_prompt_list('fixed')      # ['a photo of a {}.']
shuffle_prompts = get_prompt_list('shuffle')  # Multiple variations
shuffle_plus = get_prompt_list('shuffle+')    # Extended variations

# shuffle+ includes:
# ['a photo of a {}.', 'a photograph of a {}.', 'an image of a {}.',
#  '{}.', 'a cropped photo of a {}.', 'a good photo of a {}.',
#  'a photo of one {}.', 'a bad photo of a {}.', 'a photo of the {}.']
```

```python
# Set prompt mode during initialization
model = CLIPDensePredT(
    version='ViT-B/16',
    reduce_dim=64,
    prompt='shuffle+'  # Use extended prompt templates
)
```

```python
# Or modify after initialization
model.prompt_list = ['a photo of a {}.', 'an image of a {}.']
```

```python
# Sample prompts for a batch of labels
labels = ['cat', 'dog', 'bird', 'car']
prompts = model.sample_prompts(labels)  # Randomly samples from prompt_list
# Example output: ['a photo of a cat.', 'an image of a dog.', ...]
```

--------------------------------

### Initialize CLIPSeg Model

Source: https://context7.com/timojl/clipseg/llms.txt

Initializes the CLIPDensePredT model with specified parameters. Load pre-trained weights using load_state_dict with strict=False. For fine-grained predictions, use complex_trans_conv=True.

```python
import torch
from models.clipseg import CLIPDensePredT

# Initialize standard model with ViT-B/16 backbone
model = CLIPDensePredT(
    version='ViT-B/16',      # CLIP backbone version: 'ViT-B/16' or 'ViT-B/32'
    reduce_dim=64,           # Decoder dimension (64 or 128)
    extract_layers=(3, 6, 9), # Transformer layers to extract features from
    n_heads=4,               # Number of attention heads in decoder
    prompt='fixed'           # Prompt template: 'plain', 'fixed', 'shuffle', 'shuffle+'
)
model.eval()

# Load pre-trained weights (non-strict because CLIP weights are loaded separately)
model.load_state_dict(
    torch.load('weights/rd64-uni.pth', map_location=torch.device('cpu')),
    strict=False
)

# For fine-grained predictions with refined decoder
model_refined = CLIPDensePredT(
    version='ViT-B/16',
    reduce_dim=64,
    complex_trans_conv=True  # Enable refined transposed convolution
)
model_refined.load_state_dict(
    torch.load('weights/rd64-uni-refined.pth', map_location=torch.device('cpu')),
    strict=False
)
```

--------------------------------

### Load CLIP and Utilities

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Imports necessary libraries for CLIPSeg, including CLIP model and utility functions for normalization and experiment management.

```python
%load_ext autoreload
%autoreload 2

import clip
from evaluation_utils import norm, denorm
from general_utils import *

```

--------------------------------

### Initialize and Evaluate Metrics

Source: https://context7.com/timojl/clipseg/llms.txt

Initializes FixedIntervalMetrics for evaluation and processes data through a DataLoader to calculate segmentation scores. Ensure model and data are on the correct device.

```python
metric = FixedIntervalMetrics(
    resize_pred=True,    # Resize predictions to match ground truth
    sigmoid=True,        # Apply sigmoid to predictions
    n_values=51,         # Number of threshold values for curves
    custom_threshold=0.5 # Optional fixed threshold
)

# Evaluate on dataset
from torch.utils.data import DataLoader
from datasets.phrasecut import PhraseCut

dataset = PhraseCut('test', image_size=352, mask='text')
loader = DataLoader(dataset, batch_size=32, shuffle=False)

with torch.no_grad():
    for data_x, data_y in loader:
        data_x = [v.cuda() for v in data_x if isinstance(v, torch.Tensor)]
        data_y = [v.cuda() for v in data_y if isinstance(v, torch.Tensor)]

        pred, visual_q, _, _ = model(data_x[0], data_x[1], return_features=True)
        metric.add([pred], data_y)

# Get comprehensive scores
scores = metric.value()
print(f"Best FG-IoU: {scores['fgiou_best']:.3f}")
print(f"FG-IoU@0.5: {scores['fgiou_0.5']:.3f}")
print(f"Mean IoU@0.5: {scores['miou_0.5']:.3f}")
print(f"Average Precision: {scores['ap']:.3f}")
```

--------------------------------

### Load PhraseCut Experiment Data

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Loads data for the PhraseCut experiment using a specified configuration file and a range of samples.

```python
pc = experiment('experiments/phrasecut.yaml', nums=':6').dataframe()

```

--------------------------------

### Load and Use CLIPSeg with HuggingFace Transformers

Source: https://context7.com/timojl/clipseg/llms.txt

Load the CLIPSeg processor and model from HuggingFace Hub. Prepare image and text prompts, process them, and generate segmentation masks. Use `torch.no_grad()` for inference. The output logits are then passed through a sigmoid function to obtain masks.

```python
from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation
from PIL import Image
import torch

# Load from HuggingFace Hub
processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")

# Prepare inputs
image = Image.open("example.jpg")
prompts = ["a cat", "a dog", "the background"]

# Process and predict
inputs = processor(
    text=prompts,
    images=[image] * len(prompts),
    padding=True,
    return_tensors="pt"
)

with torch.no_grad():
    outputs = model(**inputs)

# Get segmentation masks
logits = outputs.logits  # [batch, height, width]
masks = torch.sigmoid(logits)

# For each prompt, get the corresponding mask
for i, prompt in enumerate(prompts):
    mask = masks[i].numpy()
    print(f"{prompt}: mask shape {mask.shape}")
```

--------------------------------

### Load and Normalize Image for CLIPSeg

Source: https://github.com/timojl/clipseg/blob/master/Quickstart.ipynb

Prepares an image for model input by loading, normalizing, and resizing it. Supports loading from a local file or a URL.

```python
# load and normalize image
input_image = Image.open('example_image.jpg')

# or load from URL...
# image_url = 'https://farm5.staticflickr.com/4141/4856248695_03475782dc_z.jpg'
# input_image = Image.open(requests.get(image_url, stream=True).raw)

transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    transforms.Resize((352, 352)),
])
img = transform(input_image).unsqueeze(0)
```

--------------------------------

### Load LVIS OneShot3 Dataset

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Initializes the LVIS OneShot3 dataset with specific parameters for training. This includes mask handling, normalization, and image size.

```python
lvis = LVIS_OneShot3('train_fixed', mask='separate', normalize=True, with_class_label=True, add_bar=False, 
                     text_class_labels=True, image_size=352, min_area=0.1,
                     min_frac_s=0.05, min_frac_q=0.05, fix_find_crop=True)
```

--------------------------------

### Load Pascal One-Shot Experiment Data

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Loads data for the Pascal one-shot experiment using a specified configuration file and a range of samples.

```python
pas = experiment('experiments/pascal_1shot.yaml', nums=':19').dataframe()

```

--------------------------------

### Download and Unzip Weights

Source: https://github.com/timojl/clipseg/blob/master/README.md

Download and unzip model weights. The MIT license does not apply to these weights. Ensure weights are extracted into a 'weights' directory.

```bash
wget https://owncloud.gwdg.de/index.php/s/ioHbRzFx6th32hn/download -O weights.zip
unzip -d weights -j weights.zip
```

--------------------------------

### Load Pre-trained CLIPSeg Models

Source: https://context7.com/timojl/clipseg/llms.txt

Utility functions for loading pre-trained CLIPSeg models from checkpoints. Supports loading with saved configurations, custom arguments, or retrieving the configuration alongside the model.

```python
from general_utils import load_model, get_attribute, filter_args
import inspect
import json

# Load model with saved configuration
model = load_model(
    checkpoint_id='rd64-uni',    # Directory name in logs/
    weights_file='weights.pth',   # Weight file name
    strict=False,                 # Allow partial weight loading
    model_args='from_config'      # Use saved config.json
)

# Load model with custom arguments
model = load_model(
    checkpoint_id='rd64-uni',
    model_args={
        'version': 'ViT-B/16',
        'reduce_dim': 64,
        'complex_trans_conv': True  # Override architecture
    }
)

# Load model with configuration returned
model, config = load_model(
    checkpoint_id='rd64-uni',
    with_config=True
)
print(f"Model: {config['model']}")
print(f"Trained on: {config['dataset']}")

# Manual checkpoint loading
config = json.load(open('logs/rd64-uni/config.json'))
model_cls = get_attribute(config['model'])
_, model_args, _ = filter_args(config, inspect.signature(model_cls).parameters)
model = model_cls(**model_args)
model.load_state_dict(torch.load('logs/rd64-uni/weights.pth'), strict=False)
```

--------------------------------

### Load and Preprocess Sample Images

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Defines a function to load two sample images, apply standard PyTorch image transformations including normalization, resizing, and cropping. It prepares the images for input into a neural network model.

```python
from os.path import join
from torchvision import transforms
from PIL import Image

def load_sample(filename, filename2):
    bp = expanduser('~/cloud/resources/sample_images')
    tf = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
        transforms.Resize(224),
        transforms.CenterCrop(224)
    ])
    tf2 = transforms.Compose([
        transforms.ToTensor(),
        transforms.Resize(224),
        transforms.CenterCrop(224)
    ])
    inp1 = [None, tf(Image.open(join(bp, filename))), tf2(Image.open(join(bp, filename2)))]
    inp1[1] = inp1[1].unsqueeze(0)
    inp1[2] = inp1[2][:1]   
    return inp1
```

--------------------------------

### Clone PFENet Repository

Source: https://github.com/timojl/clipseg/blob/master/README.md

Clone the PFENet repository to the root folder to use its dataset and model wrappers. This is a prerequisite for utilizing PFENet-related functionalities.

```bash
git clone https://github.com/Jia-Research-Lab/PFENet.git 
```

--------------------------------

### Train CLIPSeg Model

Source: https://github.com/timojl/clipseg/blob/master/README.md

Train a CLIPSeg model using the 'training.py' script. Specify the experiment file and experiment ID as parameters. Model weights are saved in the 'logs/' directory.

```python
python training.py phrasecut.yaml 0
```

--------------------------------

### Clone Third-Party Dependencies

Source: https://github.com/timojl/clipseg/blob/master/README.md

Clone repositories for third-party dependencies required for some datasets. Ensure these are cloned into the 'third_party' folder.

```bash
git clone https://github.com/cvlab-yonsei/JoEm
git clone https://github.com/Jia-Research-Lab/PFENet.git
git clone https://github.com/ChenyunWu/PhraseCutDataset.git
git clone https://github.com/juhongm999/hsnet.git
```

--------------------------------

### Print Generalization Results

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Prints the generalization performance metrics in a formatted string suitable for tables. This compares CLIPSeg variants and ViTSeg across different generalization abilities.

```python
print(
    'CLIPSeg (PC+) & ' + ' & '.join(f'{x*100:.1f}' for x in gen[1]) + ' \\ \n' + 
    'CLIPSeg (LVIS)  & ' + ' & '.join(f'{x*100:.1f}' for x in gen[0]) + ' \\ \n' + 
    'CLIP-Deconv & ' + ' & '.join(f'{x*100:.1f}' for x in gen[2]) + ' \\ \n' + 
    'VITSeg & ' + ' & '.join(f'{x*100:.1f}' for x in gen[3]) + ' \\'
)
```

--------------------------------

### Import Evaluation Utilities

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Imports necessary utility functions for normalizing and denormalizing image data, which are crucial for consistent image processing and analysis.

```python
from evaluation_utils import denorm, norm
```

--------------------------------

### Load PASCAL VOC Zero-Shot Experiment Data

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Loads PASCAL VOC zero-shot experiment results. This code prepares data for comparing CLIPSeg, CLIP-Deconv, and ViTSeg on unseen classes.

```python
zs = experiment('experiments/pascal_0shot.yaml', nums=':11').dataframe()

tab1 = zs[['pas_zs_seen', 'pas_zs_unseen']]
print('CLIPSeg (PC+) & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[8:9].values[0].tolist() + tab1[10:11].values[0].tolist()), '\\')
print('CLIP-Deconv & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[2:3].values[0].tolist()  + tab1[3:4].values[0].tolist()), '\\')
print('ViTSeg & ImageNet-1K &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:5].values[0].tolist()  + tab1[5:6].values[0].tolist()), '\\')
```

--------------------------------

### Load PASCAL VOC Experiment Data

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Loads PASCAL VOC dataset experiment results for one-shot learning. Use this to analyze performance metrics like mIoU, bIoU, and AP.

```python
pas = experiment('experiments/pascal_1shot.yaml', nums=':8').dataframe()
tab1 = pas[['pas_t_best_miou', 'pas_t_best_biniou', 'pas_t_ap']]
print('CLIPSeg (PC+) & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')
print('CLIPSeg (PC) & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:8].mean(0).values), '\\')

pas = experiment('experiments/pascal_1shot.yaml', nums='12:16').dataframe()
tab1 = pas[['pas_t_best_miou', 'pas_t_best_biniou', 'pas_t_ap']]
print('CLIP-Deconv (PC+) & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')
```

--------------------------------

### Apply All Preprocessing Functions

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Applies a comprehensive set of preprocessing techniques to an input image batch. This function is used to generate diverse versions of an image for analysis.

```python
def all_preprocessing(inp1):
    return [
        img_preprocess(inp1),
        img_preprocess(inp1, colorize=True),
        img_preprocess(inp1, outline=True),
        img_preprocess(inp1, blur=3),
        img_preprocess(inp1, bg_fac=0.1),
        img_preprocess(inp1, blur=3, bg_fac=0.5, center_context=0.5),
    ]
```

--------------------------------

### Select Pascal One-Shot Columns

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Selects specific columns related to IoU and AP metrics for the Pascal one-shot experiment.

```python
pas[['name', 'pas_h2_miou_0.3', 'pas_h2_biniou_0.3', 'pas_h2_ap', 'pas_h2_fgiou_ct']]

```

--------------------------------

### Load Generalization Experiment Data

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Loads data from the generalization experiment. This snippet prepares the results for comparing CLIPSeg with other models on generalization tasks.

```python
generalization = experiment('experiments/generalize.yaml').dataframe()

gen = generalization[['aff_best_fgiou', 'aff_ap', 'ability_best_fgiou', 'ability_ap', 'part_best_fgiou', 'part_ap']].values
```

--------------------------------

### Format and Print PhraseCut Table (Threshold 0.1)

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Formats and prints a LaTeX table for PhraseCut results with a threshold of 0.1. It scales metrics to percentages and sets custom row names.

```python
cols = ['pc_miou_0.1',  'pc_fgiou_0.1', 'pc_ap']
tab1 = pc[['name'] + cols]
for k in cols:
    tab1.loc[:, k] = (100 * tab1.loc[:, k]).round(1)
tab1.loc[:, 'name'] = ['CLIPSeg (PC+)', 'CLIPSeg (PC, $D=128$)', 'CLIPSeg (PC)', 'CLIP-Deconv', 'ViTSeg (PC+)', 'ViTSeg (PC)']
tab1.insert(1, 't', [0.1]*tab1.shape[0])
print(tab1.to_latex(header=False, index=False))

```

--------------------------------

### Calculate and Print Pascal One-Shot Metrics

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Calculates and prints average metrics for different models and thresholds from the Pascal one-shot dataset. It processes data in chunks and formats output for LaTeX.

```python
pas = experiment('experiments/pascal_1shot.yaml', nums=':8').dataframe()
tab1 = pas[['pas_h2_miou_0.3', 'pas_h2_biniou_0.3', 'pas_h2_ap']]
print('CLIPSeg (PC+) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')
print('CLIPSeg (PC)  & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:8].mean(0).values), '\\')

pas = experiment('experiments/pascal_1shot.yaml', nums='12:16').dataframe()
tab1 = pas[['pas_h2_miou_0.2', 'pas_h2_biniou_0.2', 'pas_h2_ap']]
print('CLIP-Deconv (PC+) & 0.2 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')

pas = experiment('experiments/pascal_1shot.yaml', nums='16:20').dataframe()
tab1 = pas[['pas_t_miou_0.2', 'pas_t_biniou_0.2', 'pas_t_ap']]
print('ViTSeg (PC+) & 0.2 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')

```

--------------------------------

### Multi-Label Semantic Segmentation with CLIPSeg

Source: https://context7.com/timojl/clipseg/llms.txt

Initializes and uses the CLIPSegMultiLabel model for zero-shot semantic segmentation across all Pascal VOC classes. Input images are processed, and logits for each class are obtained.

```python
import torch
from models.clipseg import CLIPSegMultiLabel

# Initialize multi-label model (wraps CLIPSeg for Pascal VOC classes)
model = CLIPSegMultiLabel(model='rd64-uni')
model.eval()
model.cuda()

# Input image batch
images = torch.randn(2, 3, 352, 352).cuda()  # [B, C, H, W]

# Get predictions for all 21 Pascal VOC classes
with torch.no_grad():
    logits = model(images)  # [B, 21, 352, 352]

# Get predicted class per pixel
predicted_classes = logits.argmax(dim=1)  # [B, 352, 352]

# Class names from Pascal VOC
# ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle',
#  'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog',
#  'horse', 'motorbike', 'person', 'pottedplant', 'sheep',
#  'sofa', 'train', 'tvmonitor']
```

--------------------------------

### Calculate and Print Pascal Zero-Shot Metrics

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Calculates and prints average metrics for different models and thresholds from the Pascal zero-shot dataset (in a one-shot setting). It processes data in chunks and formats output for LaTeX.

```python
pas = experiment('experiments/pascal_1shot.yaml', nums=':8').dataframe()
tab1 = pas[['pas_t_miou_0.3', 'pas_t_biniou_0.3', 'pas_t_ap']]
print('CLIPSeg (PC+) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')
print('CLIPSeg (PC) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:8].mean(0).values), '\\')

pas = experiment('experiments/pascal_1shot.yaml', nums='12:16').dataframe()
tab1 = pas[['pas_t_miou_0.3', 'pas_t_biniou_0.3', 'pas_t_ap']]
print('CLIP-Deconv (PC+) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')

pas = experiment('experiments/pascal_1shot.yaml', nums='16:20').dataframe()
tab1 = pas[['pas_t_miou_0.2', 'pas_t_biniou_0.2', 'pas_t_ap']]
print('ViTSeg (PC+) & 0.2 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\')

```

--------------------------------

### Format and Print PhraseCut Table (Threshold 0.3)

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Formats and prints a LaTeX table for PhraseCut results with a threshold of 0.3. It scales metrics to percentages and sets custom row names.

```python
cols = ['pc_miou_0.3',  'pc_fgiou_0.3', 'pc_ap']
tab1 = pc[['name'] + cols]
for k in cols:
    tab1.loc[:, k] = (100 * tab1.loc[:, k]).round(1)
tab1.loc[:, 'name'] = ['CLIPSeg (PC+)', 'CLIPSeg (PC, $D=128$)', 'CLIPSeg (PC)', 'CLIP-Deconv', 'ViTSeg (PC+)', 'ViTSeg (PC)']
tab1.insert(1, 't', [0.3]*tab1.shape[0])
print(tab1.to_latex(header=False, index=False))

```

--------------------------------

### Calculate Image-Text Similarities

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Computes similarity scores between image features and text prompts using CLIP and CLIPSeg models. Handles both pre-loaded batches and datasets. Requires a preprocessing function and optionally a mask function.

```python
from general_utils import get_batch
from functools import partial
from evaluation_utils import img_preprocess
import torch

def get_similarities(batches_or_dataset, process, mask=lambda x: None, clipmask=False):

    # base_words = [f'a photo of {x}' for x in ['a person', 'an animal', 'a knife', 'a cup']]

    all_prompts = []
    
    with torch.no_grad():
        valid_sims = []
        torch.manual_seed(571)
        
        if type(batches_or_dataset) == list:
            loader = batches_or_dataset  # already loaded
            max_iter = float('inf')
        else:
            loader = DataLoader(batches_or_dataset, shuffle=False, batch_size=32)
            max_iter = 50
        
        global batch
        for i_batch, (batch, batch_y) in enumerate(loader):
            
            if i_batch >= max_iter: break
                
            processed_batch = process(batch)
            if type(processed_batch) == dict:
                
                # processed_batch =  {k: v.to(clip_device) for k, v in processed_batch.items()}
                image_features = clip_mask_model.visual_forward(**processed_batch)[0].to(clip_device).half()
            else:
                processed_batch = process(batch).to(clip_device)
                processed_batch = nnf.interpolate(processed_batch, (224, 224), mode='bilinear')
                #image_features = clip_model.encode_image(processed_batch.to(clip_device)) 
                image_features = clip_mask_model.visual_forward(processed_batch)[0].to(clip_device).half()
                
            image_features = image_features / image_features.norm(dim=-1, keepdim=True)
            bs = len(batch[0])
            for j in range(bs):
            
                c, _, sid, qid = lvis.sample_ids[bs * i_batch + j]
                support_image = basename(lvis.samples[c][sid])
                
                img_objs = [o for o in objects_per_image[int(support_image)]]
                img_objs = [o.replace('_', ' ') for o in img_objs]
                
                other_words = [f'a photo of a {o.replace("_", " ")}' for o in img_objs 
                               if o != batch_y[2][j]]
            
                prompts = [f'a photo of a {batch_y[2][j]}'] + other_words
                all_prompts += [prompts]
                
                text_cond = clip_model.encode_text(clip.tokenize(prompts).to(clip_device))
                text_cond = text_cond / text_cond.norm(dim=-1, keepdim=True)            

                global logits
                logits = clip_model.logit_scale.exp() * image_features[j] @ text_cond.T

                global sim
                sim = torch.softmax(logits, dim=-1)
            
                valid_sims += [sim]
                
        #valid_sims = torch.stack(valid_sims)
        return valid_sims, all_prompts
    

def new_img_preprocess(x):
    return {'x_inp': x[1], 'mask': (11, 'cls_token', x[2])}
    
#get_similarities(lvis, partial(img_preprocess, center_context=0.5));
get_similarities(lvis, lambda x: x[1]);
```

--------------------------------

### Generate Similarity Table with Pandas

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Creates a Pandas DataFrame to store and display the mean similarity differences for various preprocessing functions. The results are formatted for LaTeX output.

```python
from pandas import DataFrame
tab = dict()
for j, (name, _) in enumerate(preprocessing_functions):
    tab[name] =  np.mean([outs[j][0][i][0].cpu() - base[i][0].cpu() for i in range(len(base)) if len(base_p[i]) >= 3])
    
    
print('\n'.join(f'{k} & {v*100:.2f} \\' for k,v in tab.items()))
```

--------------------------------

### Model Evaluation and Scoring

Source: https://context7.com/timojl/clipseg/llms.txt

Evaluates trained CLIPSeg models using command line or programmatic interfaces. Loads models and utilizes metrics for scoring.

```python
# Command line evaluation
# python score.py phrasecut.yaml 0 0
```

```python
# Programmatic evaluation
from score import score, load_model
from metrics import FixedIntervalMetrics
import torch

# Load trained model
model = load_model(
    'rd64-uni',           # Checkpoint directory name in logs/
    weights_file=None,     # Uses weights.pth by default
    strict=False,
    model_args='from_config'  # Load architecture from saved config
)
model.eval()
model.cuda()
```

--------------------------------

### Define Preprocessing Functions

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

A list of preprocessing functions for image data, including options for CLIP masking with different strategies and colorization.

```python
preprocessing_functions = [
#     ['clip mask CLS L11', lambda x: {'x_inp': x[1].cuda(), 'mask': (11, 'cls_token', x[2].cuda())}],
#     ['clip mask CLS all', lambda x: {'x_inp': x[1].cuda(), 'mask': ('all', 'cls_token', x[2].cuda())}],
#     ['clip mask all all', lambda x: {'x_inp': x[1].cuda(), 'mask': ('all', 'all', x[2].cuda())}],
#     ['colorize object red', partial(img_preprocess, colorize=True)],

```

--------------------------------

### Calculate Similarities with Preprocessing

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Calculates image similarities using predefined preprocessing functions and compares them against a base similarity. This is useful for evaluating the impact of different preprocessing steps.

```python
base, base_p = get_similarities(lvis, lambda x: x[1])
outs = [get_similarities(lvis, fun) for _, fun in preprocessing_functions]
outs2 = [get_similarities(lvis, fun) for _, fun in  [['BG brightness 0%', partial(img_preprocess, bg_fac=0.0)]]]
```

--------------------------------

### Generate LaTeX Table for Ablation Study

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Generates a LaTeX table from the ablation study results. This is used for presenting a concise summary of the ablation experiments.

```python
print(tab1.loc[[0,1,4,5,6,7],:].to_latex(header=False, index=False))
```

```python
print(tab1.loc[[0,1,4,5,6,7],:].to_latex(header=False, index=False))
```

--------------------------------

### Evaluate CLIPSeg Model

Source: https://github.com/timojl/clipseg/blob/master/README.md

Evaluate a trained CLIPSeg model using the 'score.py' script. Provide the experiment file, experiment ID, and configuration ID as arguments.

```python
python score.py phrasecut.yaml 0 0
```

--------------------------------

### Reuse Conditional Embeddings

Source: https://context7.com/timojl/clipseg/llms.txt

Demonstrates how to compute visual conditional embeddings from a support image and mask, and then reuse these embeddings for multiple query images.

```python
with torch.no_grad():
    # Compute visual conditional from support image+mask
    cond, _, _ = model.visual_forward_masked(support_image, support_mask)
    # Reuse conditional for multiple query images
    pred = model(query_image, cond)[0]
```

--------------------------------

### CLIP Model Image Feature Extraction and Similarity Calculation

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Loads a pre-trained CLIP model, extracts image features from preprocessed images, and calculates similarity scores against text prompts. This is used for object recognition and classification tasks.

```python
from torchvision import transforms
from PIL import Image
from matplotlib import pyplot as plt
from evaluation_utils import img_preprocess, denorm
import clip
import torch

images_queries = [
    [load_sample('things1.jpg', 'things1_jar.png'), ['jug', 'knife', 'car', 'animal', 'sieve', 'nothing']],
    [load_sample('own_photos/IMG_2017s_square.jpg', 'own_photos/IMG_2017s_square_trash_can.png'), ['trash bin', 'house', 'car', 'bike', 'window', 'nothing']],
]


_, ax = plt.subplots(2 * len(images_queries), 6, figsize=(14, 4.5 * len(images_queries)))

for j, (images, objects) in enumerate(images_queries):
    
    joint_image = all_preprocessing(images)
    
    joint_image = torch.stack(joint_image)[:,0]
    clip_model, preprocess = clip.load("ViT-B/16", device='cpu')
    image_features = clip_model.encode_image(joint_image)
    image_features = image_features / image_features.norm(dim=-1, keepdim=True)
    
    prompts = [f'a photo of a {obj}'for obj in objects]
    text_cond = clip_model.encode_text(clip.tokenize(prompts))
    text_cond = text_cond / text_cond.norm(dim=-1, keepdim=True)
    logits = clip_model.logit_scale.exp() * image_features @ text_cond.T
    sim = torch.softmax(logits, dim=-1).detach().cpu()

    for i, img in enumerate(joint_image):
        ax[2*j, i].axis('off')
        
        ax[2*j, i].imshow(torch.clamp(denorm(joint_image[i]).permute(1,2,0), 0, 1))
        ax[2*j+ 1, i].grid(True)
        
        ax[2*j + 1, i].set_ylim(0,1)
        ax[2*j + 1, i].set_yticklabels([])
        ax[2*j + 1, i].set_xticks([])  # set_xticks(range(len(prompts)))
#         ax[1, i].set_xticklabels(objects, rotation=90)
        for k in range(len(sim[i])):
            ax[2*j + 1, i].bar(k, sim[i][k], color=plt.cm.tab20(1) if k!=0 else plt.cm.tab20(3))
            ax[2*j + 1, i].text(k, 0.07, objects[k], rotation=90, ha='center', fontsize=15)

plt.tight_layout()
plt.savefig('figures/prompt_engineering.pdf', bbox_inches='tight')
```

--------------------------------

### Load Ablation Study Data

Source: https://github.com/timojl/clipseg/blob/master/Tables.ipynb

Loads data for an ablation study on CLIPSeg. This snippet prepares a DataFrame with various performance metrics and model configurations for analysis.

```python
ablation = experiment('experiments/ablation.yaml', nums=':8').dataframe()

tab1 = ablation[['name', 'pc_miou_best', 'pc_ap', 'pc-vis_miou_best', 'pc-vis_ap']]
for k in ['pc_miou_best', 'pc_ap', 'pc-vis_miou_best', 'pc-vis_ap']:
    tab1.loc[:, k] = (100 * tab1.loc[:, k]).round(1)
tab1.loc[:, 'name'] = ['CLIPSeg', 'no CLIP pre-training', 'no-negatives', '50% negatives', 'no visual', '$D=16$', 'only layer 3', 'highlight mask']
```

--------------------------------

### Load Fine-grained CLIPSeg Model

Source: https://github.com/timojl/clipseg/blob/master/README.md

Load a CLIPSeg model with fine-grained prediction capabilities using new weights. This requires specifying 'complex_trans_conv=True' during model initialization.

```python
model = CLIPDensePredT(version='ViT-B/16', reduce_dim=64, complex_trans_conv=True)
model.load_state_dict(torch.load('weights/rd64-uni-refined.pth'), strict=False)
```

--------------------------------

### Predict and Visualize Segmentation Masks

Source: https://github.com/timojl/clipseg/blob/master/Quickstart.ipynb

Performs segmentation prediction using the loaded model and input image, then visualizes the results alongside the original image and prompts.

```python
prompts = ['a glass', 'something to fill', 'wood', 'a jar']

# predict
with torch.no_grad():
    preds = model(img.repeat(4,1,1,1), prompts)[0]

# visualize prediction
_, ax = plt.subplots(1, 5, figsize=(15, 4))
[a.axis('off') for a in ax.flatten()]
ax[0].imshow(input_image)
[ax[i+1].imshow(torch.sigmoid(preds[i][0])) for i in range(4)];
[ax[i+1].text(0, -15, prompts[i]) for i in range(4)];
```

--------------------------------

### Plot Dataset Information

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Visualizes the loaded dataset information. This is useful for understanding the distribution and characteristics of the dataset.

```python
plot_data(lvis)
```

--------------------------------

### Text-Prompted Segmentation

Source: https://context7.com/timojl/clipseg/llms.txt

Performs segmentation using an image tensor and text prompts. Handles single and multiple prompts for batch inference. Masks are obtained by applying sigmoid and can be thresholded for binary segmentation.

```python
import torch
from PIL import Image
from torchvision import transforms
from models.clipseg import CLIPDensePredT

# Load and preprocess image
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    transforms.Resize((352, 352)),
])

input_image = Image.open('example_image.jpg')
img_tensor = transform(input_image).unsqueeze(0)  # Shape: [1, 3, 352, 352]

# Initialize model
model = CLIPDensePredT(version='ViT-B/16', reduce_dim=64)
model.eval()
model.load_state_dict(torch.load('weights/rd64-uni.pth'), strict=False)

# Single text prompt
with torch.no_grad():
    preds = model(img_tensor, 'a cat')[0]  # Returns tuple, [0] is prediction
    mask = torch.sigmoid(preds[0][0])       # Apply sigmoid, shape: [352, 352]

# Multiple text prompts (batch processing)
prompts = ['a glass', 'something to fill', 'wood', 'a jar']
with torch.no_grad():
    # Repeat image for each prompt
    batch_img = img_tensor.repeat(len(prompts), 1, 1, 1)
    preds = model(batch_img, prompts)[0]  # Shape: [4, 1, 352, 352]
    masks = torch.sigmoid(preds[:, 0])     # Shape: [4, 352, 352]

# Threshold masks for binary segmentation
binary_masks = (masks > 0.5).float()
```

--------------------------------

### Analyze Preprocessing Impact on Similarities

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Iterates through a list of preprocessing results and prints the mean difference in similarity scores compared to the base. This helps quantify the effect of specific preprocessing steps.

```python
for j in range(1):
    print(np.mean([outs2[j][0][i][0].cpu() - base[i][0].cpu() for i in range(len(base)) if len(base_p[i]) >= 3]))
```

--------------------------------

### Process LVIS Dataset Annotations

Source: https://github.com/timojl/clipseg/blob/master/Visual_Feature_Engineering.ipynb

Loads raw LVIS annotations and processes them to create a mapping of image IDs to their corresponding object category IDs. This involves iterating through both training and validation sets.

```python
from collections import defaultdict
import json

lvis_raw = json.load(open(expanduser('~/datasets/LVIS/lvis_v1_train.json')))
lvis_val_raw = json.load(open(expanduser('~/datasets/LVIS/lvis_v1_val.json')))

objects_per_image = defaultdict(lambda : set())
for ann in lvis_raw['annotations']:
    objects_per_image[ann['image_id']].add(ann['category_id'])
    
for ann in lvis_val_raw['annotations']:
    objects_per_image[ann['image_id']].add(ann['category_id'])    
    
objects_per_image = {o: [lvis.category_names[o] for o in v] for o, v in objects_per_image.items()}

del lvis_raw, lvis_val_raw
```