### Initialize and Execute MangaNinja Inference Pipeline

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

This snippet demonstrates how to load the required pre-trained models, initialize the MangaNinjiaPipeline class, and perform colorization on target line art using a reference image. It includes configuration for guidance scales and input resolution settings.

```python
from inference.manganinjia_pipeline import MangaNinjiaPipeline
from diffusers import ControlNetModel, DDIMScheduler, AutoencoderKL
from transformers import CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection
from src.models.unet_2d_condition import UNet2DConditionModel
from src.models.refunet_2d_condition import RefUNet2DConditionModel
from src.point_network import PointNet
from PIL import Image
import torch
import numpy as np

# Load pre-trained models
pretrained_path = './checkpoints/StableDiffusion'
clip_path = './checkpoints/models/clip-vit-large-patch14'
controlnet_path = './checkpoints/models/control_v11p_sd15_lineart'

# Initialize scheduler and VAE
noise_scheduler = DDIMScheduler.from_pretrained(pretrained_path, subfolder='scheduler')
vae = AutoencoderKL.from_pretrained(pretrained_path, subfolder='vae')

# Initialize UNets
denoising_unet = UNet2DConditionModel.from_pretrained(
    pretrained_path, subfolder="unet",
    in_channels=4, low_cpu_mem_usage=False, ignore_mismatched_sizes=True
)
reference_unet = RefUNet2DConditionModel.from_pretrained(
    pretrained_path, subfolder="unet",
    in_channels=4, low_cpu_mem_usage=False, ignore_mismatched_sizes=True
)

# Initialize CLIP encoders
tokenizer = CLIPTokenizer.from_pretrained(clip_path)
text_encoder = CLIPTextModel.from_pretrained(clip_path)
image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_path)

# Initialize ControlNet
controlnet = ControlNetModel.from_pretrained(
    controlnet_path, in_channels=4, low_cpu_mem_usage=False, ignore_mismatched_sizes=True
)

# Initialize PointNet
point_net = PointNet()

# Load MangaNinja weights
controlnet.load_state_dict(torch.load('./checkpoints/MangaNinjia/controlnet.pth', map_location="cpu"), strict=False)
point_net.load_state_dict(torch.load('./checkpoints/MangaNinjia/point_net.pth', map_location="cpu"), strict=False)
reference_unet.load_state_dict(torch.load('./checkpoints/MangaNinjia/reference_unet.pth', map_location="cpu"), strict=False)
denoising_unet.load_state_dict(torch.load('./checkpoints/MangaNinjia/denoising_unet.pth', map_location="cpu"), strict=False)

# Create pipeline
pipe = MangaNinjiaPipeline(
    reference_unet=reference_unet,
    controlnet=controlnet,
    denoising_unet=denoising_unet,
    vae=vae,
    refnet_tokenizer=tokenizer,
    refnet_text_encoder=text_encoder,
    refnet_image_encoder=image_encoder,
    controlnet_tokenizer=tokenizer,
    controlnet_text_encoder=text_encoder,
    controlnet_image_encoder=image_encoder,
    scheduler=noise_scheduler,
    point_net=point_net
)
pipe = pipe.to(torch.device("cuda"))

# Prepare inputs
ref_image = Image.open('reference.png').resize((512, 512))
target_image = Image.open('lineart.png').resize((512, 512))

# Empty point matrices (no point guidance)
point_ref = torch.zeros((1, 1, 512, 512))
point_main = torch.zeros((1, 1, 512, 512))

# Run colorization
output = pipe(
    is_lineart=False,
    ref1=ref_image,
    raw2=target_image,
    edit2=target_image,
    denosing_steps=50,
    processing_res=512,
    match_input_res=True,
    batch_size=1,
    show_progress_bar=True,
    guidance_scale_ref=9.0,
    guidance_scale_point=15.0,
    preprocessor=preprocessor,
    generator=torch.cuda.manual_seed(42),
    point_ref=point_ref,
    point_main=point_main
)

# Save results
output.img_pil.save('colorized_output.png')
output.to_save_dict['edge2_black'].save('extracted_lineart.png')
```

--------------------------------

### Initialize and Use PointNet

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Initializes the PointNet architecture to encode point correspondence matrices into multi-scale embeddings. It demonstrates loading weights and processing a sample point matrix.

```python
from src.point_network import PointNet
import torch

point_net = PointNet(conditioning_channels=1, out_channels=(320, 640, 1280, 1280), downsamples=(6, 2, 2, 2))
point_net.load_state_dict(torch.load('./checkpoints/MangaNinjia/point_net.pth', map_location='cpu'), strict=False)
point_net = point_net.to('cuda')

point_matrix = torch.zeros((1, 1, 512, 512), dtype=torch.float32, device='cuda')
point_coords = [(100, 100), (200, 150), (300, 200), (400, 300), (250, 400)]
for i, (x, y) in enumerate(point_coords, start=1):
    point_matrix[0, 0, y, x] = i

embeddings = point_net(point_matrix)
```

--------------------------------

### Point Guidance System for Precise Color Control

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Utilize the point guidance system to define exact color correspondences between reference and target images. This involves creating 512x512 numpy arrays (point matrices) where matching coordinates share the same integer value, allowing for precise color mapping during inference.

```python
import numpy as np
import torch

# Create point matrices for matching 3 color points
matrix_ref = np.zeros((512, 512), dtype=np.uint8)
matrix_target = np.zeros((512, 512), dtype=np.uint8)

# Define matching point pairs (x, y coordinates)
# Point 1: Hair color - ref(100, 150) matches target(120, 160)
matrix_ref[150, 100] = 1
matrix_target[160, 120] = 1

# Point 2: Skin color - ref(200, 250) matches target(180, 240)
matrix_ref[250, 200] = 2
matrix_target[240, 180] = 2

# Point 3: Clothing color - ref(300, 350) matches target(310, 340)
matrix_ref[350, 300] = 3
matrix_target[340, 310] = 3

# Save for command-line inference
np.save('point_ref.npy', matrix_ref)
np.save('point_target.npy', matrix_target)

# Or convert to tensors for pipeline use
point_ref = torch.from_numpy(matrix_ref).unsqueeze(0).unsqueeze(0)
point_main = torch.from_numpy(matrix_target).unsqueeze(0).unsqueeze(0)
```

--------------------------------

### Command-Line Inference for Manga Colorization

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Perform batch inference for colorizing multiple images using the infer.py script. Supports automatic line art extraction or direct line art input, with options for point guidance for precise color control. Requires specifying various model paths and inference parameters.

```bash
python infer.py \
    --pretrained_model_name_or_path './checkpoints/StableDiffusion' \
    --image_encoder_path './checkpoints/models/clip-vit-large-patch14' \
    --controlnet_model_name_or_path './checkpoints/models/control_v11p_sd15_lineart' \
    --annotator_ckpts_path './checkpoints/models/Annotators' \
    --manga_reference_unet_path './checkpoints/MangaNinjia/reference_unet.pth' \
    --manga_main_model_path './checkpoints/MangaNinjia/denoising_unet.pth' \
    --manga_controlnet_model_path './checkpoints/MangaNinjia/controlnet.pth' \
    --point_net_path './checkpoints/MangaNinjia/point_net.pth' \
    --output_dir 'output' \
    --seed 0 \
    --denoise_steps 50 \
    --guidance_scale_ref 9 \
    --guidance_scale_point 15 \
    --input_reference_paths './test_cases/hz0.png' './test_cases/hz1.png' \
    --input_lineart_paths './test_cases/hz1.png' './test_cases/hz0.png'
```

```bash
python infer.py \
    --pretrained_model_name_or_path './checkpoints/StableDiffusion' \
    --image_encoder_path './checkpoints/models/clip-vit-large-patch14' \
    --controlnet_model_name_or_path './checkpoints/models/control_v11p_sd15_lineart' \
    --annotator_ckpts_path './checkpoints/models/Annotators' \
    --manga_reference_unet_path './checkpoints/MangaNinjia/reference_unet.pth' \
    --manga_main_model_path './checkpoints/MangaNinjia/denoising_unet.pth' \
    --manga_controlnet_model_path './checkpoints/MangaNinjia/controlnet.pth' \
    --point_net_path './checkpoints/MangaNinjia/point_net.pth' \
    --output_dir 'output' \
    --denoise_steps 50 \
    --is_lineart \
    --input_reference_paths './reference.png' \
    --input_lineart_paths './lineart.png'
```

```bash
python infer.py \
    --pretrained_model_name_or_path './checkpoints/StableDiffusion' \
    --image_encoder_path './checkpoints/models/clip-vit-large-patch14' \
    --controlnet_model_name_or_path './checkpoints/models/control_v11p_sd15_lineart' \
    --annotator_ckpts_path './checkpoints/models/Annotators' \
    --manga_reference_unet_path './checkpoints/MangaNinjia/reference_unet.pth' \
    --manga_main_model_path './checkpoints/MangaNinjia/denoising_unet.pth' \
    --manga_controlnet_model_path './checkpoints/MangaNinjia/controlnet.pth' \
    --point_net_path './checkpoints/MangaNinjia/point_net.pth' \
    --output_dir 'output' \
    --denoise_steps 50 \
    --guidance_scale_ref 9 \
    --guidance_scale_point 15 \
    --input_reference_paths './test_cases/hz0.png' \
    --input_lineart_paths './test_cases/hz1.png' \
    --point_ref_paths './test_cases/hz01_0.npy' \
    --point_lineart_paths './test_cases/hz01_1.npy'
```

--------------------------------

### MangaNinjiaPipeline Inference

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

This snippet demonstrates how to load the MangaNinjiaPipeline and perform line art colorization using a reference image and target line art.

```APIDOC
## MangaNinjiaPipeline Inference

### Description
This section details the process of initializing and using the MangaNinjiaPipeline for automatic colorization of line art. It covers loading pre-trained models, setting up the pipeline with various components like UNets, schedulers, and encoders, and finally running the inference with sample inputs.

### Method
Python Script

### Endpoint
N/A (Local Inference)

### Parameters
N/A (Code-based initialization and execution)

### Request Example
```python
from inference.manganinjia_pipeline import MangaNinjiaPipeline
from diffusers import ControlNetModel, DDIMScheduler, AutoencoderKL
from transformers import CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection
from src.models.unet_2d_condition import UNet2DConditionModel
from src.models.refunet_2d_condition import RefUNet2DConditionModel
from src.point_network import PointNet
from PIL import Image
import torch
import numpy as np

# Load pre-trained models
pretrained_path = './checkpoints/StableDiffusion'
clip_path = './checkpoints/models/clip-vit-large-patch14'
controlnet_path = './checkpoints/models/control_v11p_sd15_lineart'

# Initialize scheduler and VAE
noise_scheduler = DDIMScheduler.from_pretrained(pretrained_path, subfolder='scheduler')
vae = AutoencoderKL.from_pretrained(pretrained_path, subfolder='vae')

# Initialize UNets
denoisng_unet = UNet2DConditionModel.from_pretrained(
    pretrained_path, subfolder="unet",
    in_channels=4, low_cpu_mem_usage=False, ignore_mismatched_sizes=True
)
reference_unet = RefUNet2DConditionModel.from_pretrained(
    pretrained_path, subfolder="unet",
    in_channels=4, low_cpu_mem_usage=False, ignore_mismatched_sizes=True
)

# Initialize CLIP encoders
tokenizer = CLIPTokenizer.from_pretrained(clip_path)
text_encoder = CLIPTextModel.from_pretrained(clip_path)
image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_path)

# Initialize ControlNet
controlnet = ControlNetModel.from_pretrained(
    controlnet_path, in_channels=4, low_cpu_mem_usage=False, ignore_mismatched_sizes=True
)

# Initialize PointNet
point_net = PointNet()

# Load MangaNinja weights
controlnet.load_state_dict(torch.load('./checkpoints/MangaNinjia/controlnet.pth', map_location="cpu"), strict=False)
point_net.load_state_dict(torch.load('./checkpoints/MangaNinjia/point_net.pth', map_location="cpu"), strict=False)
reference_unet.load_state_dict(torch.load('./checkpoints/MangaNinjia/reference_unet.pth', map_location="cpu"), strict=False)
denoisng_unet.load_state_dict(torch.load('./checkpoints/MangaNinjia/denoisng_unet.pth', map_location="cpu"), strict=False)

# Create pipeline
pipe = MangaNinjiaPipeline(
    reference_unet=reference_unet,
    controlnet=controlnet,
    denoisng_unet=denoisng_unet,
    vae=vae,
    refnet_tokenizer=tokenizer,
    refnet_text_encoder=text_encoder,
    refnet_image_encoder=image_encoder,
    controlnet_tokenizer=tokenizer,
    controlnet_text_encoder=text_encoder,
    controlnet_image_encoder=image_encoder,
    scheduler=noise_scheduler,
    point_net=point_net
)
pipe = pipe.to(torch.device("cuda"))

# Prepare inputs
ref_image = Image.open('reference.png').resize((512, 512))
target_image = Image.open('lineart.png').resize((512, 512))

# Empty point matrices (no point guidance)
point_ref = torch.zeros((1, 1, 512, 512))
point_main = torch.zeros((1, 1, 512, 512))

# Run colorization
output = pipe(
    is_lineart=False,           # Set True if input is already line art
    ref1=ref_image,
    raw2=target_image,
    edit2=target_image,
    denosing_steps=50,
    processing_res=512,
    match_input_res=True,
    batch_size=1,
    show_progress_bar=True,
    guidance_scale_ref=9.0,     # Reference guidance strength
    guidance_scale_point=15.0,  # Point guidance strength
    preprocessor=preprocessor, # Assuming 'preprocessor' is defined elsewhere
    generator=torch.cuda.manual_seed(42),
    point_ref=point_ref,
    point_main=point_main
)

# Save results
output.img_pil.save('colorized_output.png')
output.to_save_dict['edge2_black'].save('extracted_lineart.png')
```

### Response
#### Success Response (200)
- **img_pil** (PIL.Image.Image) - The final colorized image.
- **to_save_dict** (dict) - A dictionary containing additional images to save, such as extracted line art.

#### Response Example
```json
{
  "img_pil": "<PIL.Image.Image object>",
  "to_save_dict": {
    "edge2_black": "<PIL.Image.Image object>"
  }
}
```
```

--------------------------------

### Image Utility Functions

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Provides helper functions for resizing images while maintaining aspect ratio and converting tensor formats between CHW and HWC.

```python
from utils.image_util import resize_max_res, chw2hwc
from PIL import Image
import numpy as np
import torch

image = Image.open('large_image.png')
resized = resize_max_res(image, max_edge_resolution=512)

chw_tensor = torch.randn(3, 512, 512)
hwc_tensor = chw2hwc(chw_tensor)
```

--------------------------------

### Batch Line Art Extraction with BatchLineartDetector

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Employ the BatchLineartDetector for neural network-based line art extraction from RGB images. This detector processes images in batches, converting them into clean line art using a generator network trained on the Informative Drawings dataset. Requires PyTorch and Pillow.

```python
from src.annotator.lineart import BatchLineartDetector
import torch
from PIL import Image
import numpy as np

# Initialize detector
detector = BatchLineartDetector('./checkpoints/models/Annotators')
detector.to('cuda', dtype=torch.float32)

# Prepare input image (normalized to [-1, 1])
image = Image.open('input_rgb.png').convert('RGB').resize((512, 512))
image_np = np.array(image)
image_tensor = torch.from_numpy(image_np.transpose(2, 0, 1)).float() / 255.0 * 2.0 - 1.0
image_tensor = image_tensor.unsqueeze(0).to('cuda')
```

--------------------------------

### Inference Configuration Format

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Defines the YAML structure required for configuring model paths and inference settings for the MangaNinjia pipeline.

```yaml
model_path:
  pretrained_model_name_or_path: ./checkpoints/StableDiffusion
  clip_vision_encoder_path: ./checkpoints/models/clip-vit-large-patch14
  controlnet_model_name: './checkpoints/models/control_v11p_sd15_lineart'
  annotator_ckpts_path: ./checkpoints/models/Annotators
  manga_control_model_path: ./checkpoints/MangaNinjia/controlnet.pth
  manga_reference_model_path: ./checkpoints/MangaNinjia/reference_unet.pth
  manga_main_model_path: ./checkpoints/MangaNinjia/denoising_unet.pth
  point_net_path: ./checkpoints/MangaNinjia/point_net.pth

inference_config:
  output_path: output
  device: cuda
```

--------------------------------

### CHW to HWC Tensor/Array Conversion (Python)

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Demonstrates converting a CHW (Channels-Height-Width) formatted PyTorch tensor or NumPy array to HWC (Height-Width-Channels) format. This is a common operation in image processing pipelines.

```python
import torch
import numpy as np

# Example with PyTorch tensor
chw_tensor = torch.randn(3, 512, 512)
hwc_tensor = chw_tensor.permute(1, 2, 0)
print(f"CHW: {chw_tensor.shape}, HWC: {hwc_tensor.shape}")
# Output: CHW: torch.Size([3, 512, 512]), HWC: torch.Size([512, 512, 3])

# Example with NumPy array (assuming chw2hwc is defined elsewhere)
# chw_array = np.random.rand(3, 256, 256)
# hwc_array = chw2hwc(chw_array)
# print(f"CHW: {chw_array.shape}, HWC: {hwc_array.shape}")
# Output: CHW: (3, 256, 256), HWC: (256, 256, 3)
```

--------------------------------

### Extract Line Art from Images

Source: https://context7.com/ali-vilab/manganinjia/llms.txt

Uses a pre-trained detector to extract line art from an input tensor. The output is thresholded to remove noise and converted to a standard image format.

```python
with torch.no_grad():
    lineart = detector(image_tensor)
    lineart[lineart <= 0.24] = 0

lineart_np = ((lineart[0, 0].cpu().numpy() + 1.) / 2 * 255).astype(np.uint8)
lineart_image = Image.fromarray(lineart_np)
lineart_image.save('extracted_lineart.png')
```

=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.