### Configure and start training

Source: https://github.com/unslothai/notebooks/blob/main/nb/Kaggle-Qwen2_5_7B_VL_GRPO.ipynb

Sets up the training arguments and initiates the fine-tuning process using the Unsloth `Trainer`.

```python
from trl import SFTTrainer
from transformers import TrainingArguments

training_args = TrainingArguments(
    output_dir="./results",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    num_train_epochs=3,
    logging_steps=10,
    save_steps=50,
    fp16=True, # Use mixed precision training
    optim="adamw_torch", # Optimizer
    warmup_ratio=0.03, # Warmup ratio
    lr_scheduler_type="cosine", # Learning rate scheduler
    report_to="tensorboard", # Reporting to TensorBoard
)

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=data["train"],
    dataset_text_field="text", # Field containing the text data
    max_seq_length=2048,
    formatting_func=formatting_prompts_func,
    args=training_args,
)

trainer.train()

```

--------------------------------

### Install Unsloth and Transformers

Source: https://github.com/unslothai/notebooks/blob/main/nb/Advanced_Llama3_1_(3B)_GRPO_LoRA.ipynb

Install the Unsloth library and the Transformers library to get started.

```bash
pip install -q unsloth transformers accelerate peft bitsandbytes
```

--------------------------------

### NeMo Gym Setup Script

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-NeMo-Gym-Multi-Environment.ipynb

This script clones the NeMo Gym repository, sets up a virtual environment, installs dependencies, creates datasets, and starts the resource servers.

```python
import subprocess
import os
import time
import atexit
import requests

GYM_DIR = os.path.expanduser("~/Gym")

# Detect Google Colab
try:
    import google.colab
    _on_colab = True
except ImportError:
    _on_colab = False

# Step 1: Clone NeMo Gym
if not os.path.exists(GYM_DIR):
    print("Cloning NeMo Gym...")
    subprocess.run(
        ["git", "clone", "https://github.com/NVIDIA-NeMo/Gym.git", GYM_DIR],
        check = True,
    )

# Step 2: Create venv and install dependencies
if not os.path.exists(os.path.join(GYM_DIR, ".venv", "bin", "python")):
    print("Setting up NeMo Gym environment (this may take a few minutes)...")
    subprocess.run(["uv", "venv", "--python", "3.12"], cwd = GYM_DIR, check = True)
    subprocess.run(
        ["bash", "-c", "source .venv/bin/activate && uv sync"],
        cwd = GYM_DIR, check = True,
    )
    subprocess.run(
        ["bash", "-c", "source .venv/bin/activate && uv pip install reasoning-gym"],
        cwd = GYM_DIR, check = True,
    )
# Ensure matplotlib is installed (required by reasoning-gym via cellpylib)
subprocess.run(
    ["bash", "-c", "source .venv/bin/activate && uv pip install matplotlib"],
    cwd = GYM_DIR, check = True, stdout = subprocess.DEVNULL,
)
# Step 3: Create sudoku dataset
sudoku_path = os.path.join(
    GYM_DIR, "resources_servers/reasoning_gym/data/train_mini_sudoku.jsonl"
)
if not os.path.exists(sudoku_path):
    print("Creating mini_sudoku dataset (2000 examples)...")
    subprocess.run(
        [
            "bash", "-c",
            "source .venv/bin/activate && python "
            "resources_servers/reasoning_gym/scripts/create_dataset.py "
            "--task mini_sudoku --size 2000 --seed 42 "
            f"--output {sudoku_path}",
        ],
        cwd = GYM_DIR, check = True,
    )

# Step 4: Download instruction_following dataset
import shutil
from huggingface_hub import hf_hub_download

if_path = os.path.join(
    GYM_DIR,
    "resources_servers/instruction_following/data/instruction_following.jsonl",
)
if not os.path.exists(if_path):
    print("Downloading instruction_following dataset...")
    src = hf_hub_download(
        repo_id = "nvidia/Nemotron-RL-instruction_following",
        filename = "instruction_following.jsonl",
        repo_type = "dataset",
    )
    os.makedirs(os.path.dirname(if_path), exist_ok = True)
    shutil.copy(src, if_path)
# Step 5: Create resources_only.yaml for instruction_following if missing
_if_resources_only = os.path.join(
    GYM_DIR, "resources_servers/instruction_following/configs/resources_only.yaml"
)
if not os.path.exists(_if_resources_only):
    with open(_if_resources_only, "w") as _f:
        _f.write(
            "instruction_following:\n"
            "  resources_servers:\n"
            "    instruction_following:\n"
            "      entrypoint: app.py\n"
            "      domain: instruction_following\n"
            "      verified: false\n"
        )

```

--------------------------------

### Create Trainer and Start Training

Source: https://github.com/unslothai/notebooks/blob/main/nb/Qwen2_5_7B_VL_GRPO.ipynb

Initializes the Trainer with the model, tokenizer, training arguments, and dataset, then starts the fine-tuning process.

```python
from trl import SFTTrainer

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=None, # Replace with your actual training dataset
    eval_dataset=None, # Replace with your actual evaluation dataset
    peft_config=None, # PEFT config is already applied to the model
    dataset_text_field="text", # Specify the column containing the text data
    max_seq_length=2048,
    args=args,
    packing=False,
)

# Start training
trainer.train()

```

--------------------------------

### Install Dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen3_(4B)_Instruct-QAT.ipynb

Installs necessary libraries including accelerate, peft, trl, sentencepiece, protobuf, datasets, huggingface_hub, hf_transfer, transformers, and fbgemm-gpu-genai. The versions are pinned for compatibility.

```python
!uv pip install --system -qqq --no-deps accelerate peft "trl==0.22.2"
!uv pip install --system -qqq sentencepiece protobuf "datasets==4.3.0" "huggingface_hub>=0.34.0" hf_transfer "transformers==4.55.4"
!uv pip install --system -qqq --upgrade --force-reinstall fbgemm-gpu-genai=={_qat_fbgemm}
```

--------------------------------

### NeMo Gym Setup and Server Start

Source: https://github.com/unslothai/notebooks/blob/main/nb/NeMo-Gym-Sudoku.ipynb

This code block handles the cloning of the NeMo Gym repository, setting up a virtual environment using 'uv', installing dependencies including 'reasoning-gym' and 'matplotlib', creating a mini_sudoku dataset, and starting the NeMo Gym server. It includes error handling and cleanup mechanisms for the server process.

```python
if not os.path.exists(GYM_DIR):
    print("Cloning NeMo Gym...")
    subprocess.run(
        ["git", "clone", "https://github.com/NVIDIA-NeMo/Gym.git", GYM_DIR],
        check = True,
    )

# Step 2: Create venv and install dependencies
if not os.path.exists(os.path.join(GYM_DIR, ".venv", "bin", "python")):
    print("Setting up NeMo Gym environment (this may take a few minutes)...")
    subprocess.run(["uv", "venv", "--python", "3.12"], cwd = GYM_DIR, check = True)
    subprocess.run(
        ["bash", "-c", "source .venv/bin/activate && uv sync"],
        cwd = GYM_DIR, check = True,
    )
    subprocess.run(
        ["bash", "-c", "source .venv/bin/activate && uv pip install reasoning-gym"],
        cwd = GYM_DIR, check = True,
    )
# Ensure matplotlib is installed (required by reasoning-gym via cellpylib)
subprocess.run(
    ["bash", "-c", "source .venv/bin/activate && uv pip install matplotlib"],
    cwd = GYM_DIR, check = True, stdout = subprocess.DEVNULL,
)
# Step 3: Create dataset
_sudoku_ds = os.path.join(
    GYM_DIR, "resources_servers/reasoning_gym/data/train_mini_sudoku.jsonl"
)
if not os.path.exists(_sudoku_ds):
    print("Creating mini_sudoku dataset (2000 examples)...")
    subprocess.run(
        [
            "bash", "-c",
            "source .venv/bin/activate && python "
            "resources_servers/reasoning_gym/scripts/create_dataset.py "
            "--task mini_sudoku --size 2000 --seed 42 "
            f"--output {_sudoku_ds}",
        ],
        cwd = GYM_DIR, check = True,
    )
# Start NeMo Gym server if not already running
try:
    requests.get("http://127.0.0.1:11000/global_config_dict_yaml", timeout = 2)
    print("NeMo Gym server already running on port 11000.")
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
    _colab_flag = " +uv_pip_set_python=true"
    print("Starting NeMo Gym server...")
    _ng_log = open(os.path.join(GYM_DIR, "ng_run.log"), "w")
    ng_process = subprocess.Popen(
        [
            "bash", "-c",
            "source .venv/bin/activate && ng_run "
            '"+config_paths=[resources_servers/reasoning_gym/configs/resources_only.yaml]"'
            + _colab_flag,
        ],
        cwd = GYM_DIR,
        stdout = _ng_log,
        stderr = subprocess.STDOUT,
    )

    def _cleanup_ng():
        if ng_process.poll() is None:
            ng_process.terminate()
            try:
                ng_process.wait(timeout = 10)
            except subprocess.TimeoutExpired:
                ng_process.kill()
        _ng_log.close()
    atexit.register(_cleanup_ng)

    print("Waiting for server", end = "", flush = True)
    for _ in range(120):
        try:
            requests.get(
                "http://127.0.0.1:11000/global_config_dict_yaml", timeout = 2
            )
            break
        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
            if ng_process.poll() is not None:
                raise RuntimeError(
                    "Server process exited unexpectedly. "
                    f"Check {GYM_DIR}/ng_run.log for details."
                )
            print(".", end = "", flush = True)
            time.sleep(3)
    else:
        raise RuntimeError(
            "NeMo Gym server did not start within 6 minutes."
        )
    print("\nHead server ready!")
```

--------------------------------

### NeMo Gym Setup and Server Start

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-NeMo-Gym-Sudoku.ipynb

This Python script automates the cloning of NeMo Gym, sets up the virtual environment, installs dependencies including reasoning-gym, creates a mini Sudoku training dataset, and starts the resources server in the background. It includes logic to detect Google Colab and adjust the command accordingly.

```python
import subprocess
import os
import time
import atexit
import requests

GYM_DIR = os.path.expanduser("~/Gym")

# Detect Google Colab
try:
    import google.colab
    _on_colab = True
except ImportError:
    _on_colab = False

# Step 1: Clone NeMo Gym
if not os.path.exists(GYM_DIR):
    print("Cloning NeMo Gym...")
    subprocess.run(
        ["git", "clone", "https://github.com/NVIDIA-NeMo/Gym.git", GYM_DIR],
        check = True,
    )

# Step 2: Create venv and install dependencies
if not os.path.exists(os.path.join(GYM_DIR, ".venv", "bin", "python")):
    print("Setting up NeMo Gym environment (this may take a few minutes)...")
    subprocess.run(["uv", "venv", "--python", "3.12"], cwd = GYM_DIR, check = True)
    subprocess.run(
        ["bash", "-c", "source .venv/bin/activate && uv sync"],
        cwd = GYM_DIR, check = True,
    )
    subprocess.run(
        ["bash", "-c", "source .venv/bin/activate && uv pip install reasoning-gym"],
        cwd = GYM_DIR, check = True,
    )
# Ensure matplotlib is installed (required by reasoning-gym via cellpylib)
subprocess.run(
    ["bash", "-c", "source .venv/bin/activate && uv pip install matplotlib"],
    cwd = GYM_DIR, check = True, stdout = subprocess.DEVNULL,
)
# Step 3: Create dataset
_sudoku_ds = os.path.join(
    GYM_DIR, "resources_servers/reasoning_gym/data/train_mini_sudoku.jsonl"
)
if not os.path.exists(_sudoku_ds):
    print("Creating mini_sudoku dataset (2000 examples)...")
    subprocess.run(
        [
            "bash", "-c",
            "source .venv/bin/activate && python "
            "resources_servers/reasoning_gym/scripts/create_dataset.py "
            "--task mini_sudoku --size 2000 --seed 42 "
            f"--output {_sudoku_ds}",
        ],
        cwd = GYM_DIR, check = True,
    )

```

--------------------------------

### Start training

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen3_(14B).ipynb

Initiates the training process using the configured trainer. It also mentions how to resume a training run.

```python
# Let's train the model! To resume a training run, set trainer.train(resume_from_checkpoint = True)

trainer_stats = trainer.train()
```

--------------------------------

### Install Unsloth and Dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/Orpheus_(3B)-TTS.ipynb

Installs Unsloth and related libraries. For Google Colab, it installs specific versions of dependencies and xformers based on the PyTorch version. For local or cloud setups, it installs Unsloth directly.

```python
%%capture
import os, re
if "COLAB_" not in "".join(os.environ.keys()):
    !pip install unsloth  # Do this in local & cloud setups
else:
    import torch; v = re.match(r'[\d]{1,}\.[\d]{1,}', str(torch.__version__)).group(0)
    xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, "0.0.34")
    !pip install sentencepiece protobuf "datasets==4.3.0" "huggingface_hub>=0.34.0" hf_transfer
    !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth
    !pip install --no-deps --upgrade "torchao>=0.16.0"
!pip install transformers==4.56.2
!pip install --no-deps trl==0.22.2
!pip install snac torchcodec "datasets>=3.4.1,<4.0.0"
```

--------------------------------

### Start Training

Source: https://github.com/unslothai/notebooks/blob/main/nb/Kaggle-Qwen2.5_(3B)-GRPO.ipynb

Start the fine-tuning process using the Unsloth `Trainer`.

```python
from trl import SFTTrainer

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=data["train"],
    dataset_text_field="text",
    max_seq_length=2048,
    args=TrainingArguments(
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        warmup_steps=5,
        max_steps=50,
        learning_rate=2e-4,
        fp16=True,
        logging_steps=1,
        output_dir="outputs",
        optim="adamw_torch",
        lr_scheduler_type="cosine",
        disable_tqdm=False, # Set to True to disable progress bar
    ),
    formatting_func=formatting_prompts_func,
    lora_config=lora_config,
    data_collator=None,
    # data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),
)

trainer.train()

```

--------------------------------

### Start Training

Source: https://github.com/unslothai/notebooks/blob/main/nb/Qwen3_5_MoE.ipynb

Initiate the training process using the configured model and dataset.

```python
from trl import SFTTrainer

trainer = SFTTrainer(
    model=model,
    train_dataset=data,
    dataset_text_field="text",
    max_seq_length=2048,
    tokenizer=tokenizer,
    args=TrainingArguments(
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        warmup_steps=5,
        max_steps=500,
        learning_rate=2e-4,
        fp16=False,
        bf16=True,
        logging_steps=1,
        output_dir="outputs",
        optim="paged_adamw_8bit",
        lr_scheduler_type="cosine",
        disable_tqdm=True,
    ),
)

trainer.train()
```

--------------------------------

### Install Unsloth and Dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/Sesame_CSM_(1B)-TTS.ipynb

Installs Unsloth and necessary libraries. For Google Colab, it detects the environment and installs specific versions of dependencies like xformers, torch, and datasets. For local setups, it installs the base unsloth package.

```python
%%capture
import os, re
if "COLAB_" not in "".join(os.environ.keys()):
    !pip install unsloth  # Do this in local & cloud setups
else:
    import torch; v = re.match(r'[\d]{1,}[.][\d]{1,}', str(torch.__version__)).group(0)
    xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, "0.0.34")
    !pip install sentencepiece protobuf "datasets==4.3.0" "huggingface_hub>=0.34.0" hf_transfer
    !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth
    !pip install --no-deps --upgrade "torchao>=0.16.0"
!pip install transformers==4.52.3
!pip install --no-deps trl==0.22.2
!pip install torchcodec "datasets>=3.4.1,<4.0.0"
```

--------------------------------

### Install Unsloth and dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/Magistral_(24B)-Reasoning-Conversational.ipynb

Installs Unsloth and necessary libraries. It includes conditional installation for Google Colab and local/cloud setups, along with specific versions for transformers and trl.

```python
%%capture
import os, re
if "COLAB_" not in "".join(os.environ.keys()):
    !pip install unsloth  # Do this in local & cloud setups
else:
    import torch; v = re.match(r'[\d]{1,}\[.][\d]{1,}', str(torch.__version__)).group(0)
    xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, "0.0.34")
    !pip install sentencepiece protobuf "datasets==4.3.0" "huggingface_hub>=0.34.0" hf_transfer
    !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth
    !pip install --no-deps --upgrade "torchao>=0.16.0"
!pip install transformers==4.56.2
!pip install --no-deps trl==0.22.2
```

--------------------------------

### Example Reasoning and Solution

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen2_5_7B_VL_GRPO.ipynb

Demonstrates the expected output format for reasoning and solution, specifically for a measurement question.

```text
<REASONING>
To measure the length of the nail, we need to align it with the ruler and observe where it ends relative to the markings on the ruler.

1. Place the nail on the ruler so that the tip of the nail is at the 0-inch mark.
2. Observe where the back end of the nail falls on the ruler.
3. The back end of the nail appears to be just past the 3-inch mark but not quite reaching the 4-inch mark.

Since the question asks for the length to the nearest inch, we need to determine if the nail is closer to 3 inches or 4 inches in length. In this case, the nail is closer to 3 inches than to 4 inches because the back end of the nail is closer to the 3-inch mark than the 4-inch mark.

Therefore, the nail is about 3 inches long.
</REASONING>

<SOLUTION>
3</SOLUTION>
```

--------------------------------

### Install Dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen3_(32B)_A100-Reasoning-Conversational.ipynb

Installs necessary libraries for Unsloth and Hugging Face Transformers.

```bash
!uv pip install --system -qqq sentencepiece protobuf "datasets==4.3.0" "huggingface_hub>=0.34.0" hf_transfer "transformers==4.56.2" 
!uv pip install --system -qqq --no-deps accelerate peft "trl==0.22.2"
```

--------------------------------

### Install and Setup Oute TTS Dependencies

Source: https://github.com/unslothai/notebooks/blob/main/original_template/Oute_TTS_(1B).ipynb

Installs Unsloth and other required Python packages. It includes conditional installation for Google Colab and removes specific files from the cloned repository. Use this to prepare your environment for Oute TTS.

```python
%%capture
import os
if "COLAB_" not in "".join(os.environ.keys()):
    !pip install unsloth
else:
    # Do this only in Colab notebooks! Otherwise use pip install unsloth
    !pip install --no-deps bitsandbytes accelerate xformers==0.0.29.post3 peft trl==0.15.2 triton cut_cross_entropy unsloth_zoo
    !pip install sentencepiece protobuf "datasets>=3.4.1" huggingface_hub hf_transfer
    !pip install --no-deps unsloth
!pip install omegaconf einx
!rm -rf OuteTTS && git clone https://github.com/edwko/OuteTTS
import os
os.remove("/content/OuteTTS/outetts/models/gguf_model.py")
os.remove("/content/OuteTTS/outetts/interface.py")
os.remove("/content/OuteTTS/outetts/__init__.py")
!pip install pyloudnorm openai-whisper uroman MeCab loguru flatten_dict ffmpy randomname argbind tiktoken ftfy
!pip install descript-audio-codec descript-audiotools julius openai-whisper --no-deps
%env UNSLOTH_DISABLE_FAST_GENERATION = 1
```

--------------------------------

### Example: Get Weather in Sydney

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-FunctionGemma_(270M)-Multi-Turn-Tool-Calling.ipynb

Sends a user query to the model and processes the response to get the weather in Sydney, Australia.

```python
messages.append({"role" : "user", "content" : "What's the weather like in Sydney, Australia?"})
messages = do_inference(model, messages, max_new_tokens = 128)
```

--------------------------------

### Install Dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen_3_5_27B_A100(80GB).ipynb

Installs necessary libraries for Unsloth and model fine-tuning.

```shell
!uv pip install --system -qqq --no-deps "torchcodec==0.7.0"
!uv pip install --system -qqq --upgrade --no-deps "trl==0.22.2"
!uv pip install --system -qqq "transformers==5.3.0"
!uv pip install --system -qqq --no-build-isolation flash-linear-attention "causal_conv1d==1.6.0"
```

--------------------------------

### Start Training

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen2.5_(3B)-GRPO.ipynb

Initiates the fine-tuning process using the configured model, tokenizer, and prepared dataset with specified training arguments.

```python
from transformers import TrainingArguments

# Define training arguments
training_args = TrainingArguments(
    output_dir="./results", # Directory to save training results
    num_train_epochs=1, # Number of training epochs
    per_device_train_batch_size=2, # Batch size per device during training
    gradient_accumulation_steps=2, # Number of updates steps to accumulate before performing a backward pass
    optim="adamw_torch", # Optimizer to use
    learning_rate=2e-4, # Initial learning rate
    fp16=False, # Whether to use mixed precision training
    bf16=True, # Whether to use bfloat16 mixed precision training
    logging_steps=1, # Number of steps between logging
    save_steps=50, # Number of steps to save the model checkpoint
    warmup_steps=5, # Number of steps for a linear warmup from from 0 to learning_rate
    lr_scheduler_type="cosine", # Learning rate scheduler type
    report_to="tensorboard", # Reporting destination
)

# Start the training process
trainer = Trainer(
    model=model,
    train_dataset=dataset,
    args=training_args,
    data_collator=data_collator,
    # Use a custom tokenizer_func to ensure correct tokenization
    tokenizer_func=None,
)

trainer.train()

```

--------------------------------

### Example: Get Weather in San Francisco

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-FunctionGemma_(270M)-Multi-Turn-Tool-Calling.ipynb

Sends a user query to the model and processes the response to get the weather in San Francisco.

```python
messages.append({"role" : "user", "content" : "What's the weather like in San Francisco?"})
messages = do_inference(model, messages, max_new_tokens = 128)
```

--------------------------------

### Install Unsloth

Source: https://github.com/unslothai/notebooks/blob/main/nb/Qwen3_5_MoE.ipynb

Install the Unsloth library for faster training.

```bash
from unsloth import FastLanguageModel
import torch

model, tokenizer = FastLanguageModel.from_pretrained(
    model = "Qwen/Qwen3.5-0.5B-Chat",
    torch_dtype = torch.bfloat16,
    load_in_4bit = True,
)
```

--------------------------------

### Example: Get Today's Date

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-FunctionGemma_(270M)-Multi-Turn-Tool-Calling.ipynb

Sends a user query to the model and processes the response to get today's date.

```python
messages = []
messages.append({"role": "user", "content": "What's today's date?"})
messages = do_inference(model, messages, max_new_tokens = 128)
```

--------------------------------

### Install Unsloth and Dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/Gemma3N_(4B)-Audio.ipynb

Installs Unsloth and essential libraries like sentencepiece, protobuf, datasets, huggingface_hub, hf_transfer, bitsandbytes, accelerate, peft, trl, and triton. It includes conditional installation for xformers based on the PyTorch version and upgrades torchao. This is for local and cloud setups.

```python
%%capture
import os, re
if "COLAB_" not in "".join(os.environ.keys()):
    !pip install unsloth  # Do this in local & cloud setups
else:
    import torch; v = re.match(r'[\d]{1,}[.][\d]{1,}', str(torch.__version__)).group(0)
    xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, "0.0.34")
    !pip install sentencepiece protobuf "datasets==4.3.0" "huggingface_hub>=0.34.0" hf_transfer
    !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth
    !pip install --no-deps --upgrade "torchao>=0.16.0"
!pip install transformers==4.56.2
!pip install --no-deps trl==0.22.2
!pip install torchcodec
import torch; torch._dynamo.config.recompile_limit = 64;
```

--------------------------------

### Print First Example

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-gpt-oss-(120B)_A100-Fine-tuning.ipynb

Prints the formatted text of the first example in the dataset to inspect the output.

```python
print(dataset[0]['text'])
```

--------------------------------

### Install necessary libraries

Source: https://github.com/unslothai/notebooks/blob/main/nb/HuggingFace Course-Qwen3_VL_(8B)-Vision-GRPO.ipynb

Installs Unsloth and other required libraries for the project.

```bash
!uv pip install -qqq --no-deps --upgrade "torchao>=0.16.0"
!uv pip install transformers==4.56.2
!uv pip install --no-deps trl==0.22.2
```

--------------------------------

### Unsloth Model Loading Output

Source: https://github.com/unslothai/notebooks/blob/main/nb/Gemma4_(12B)_Audio.ipynb

Example output shown during Unsloth model loading, indicating patching, installation checks, and hardware/software configurations.

```text
Output:
🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.
Unsloth: Your Flash Attention 2 installation seems to be broken. Using Xformers instead. No performance changes will be seen.
🦥 Unsloth Zoo will now patch everything to make training faster!
==((====))==  Unsloth 2026.5.10: Fast Gemma4_Unified patching. Transformers: 5.10.0.dev0.
   \   /|    NVIDIA A100-SXM4-40GB. Num GPUs = 1. Max memory: 39.494 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.11.0+cu128. CUDA: 8.0. CUDA Toolkit: 12.8. Triton: 3.6.0
\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.34. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
Unsloth: QLoRA and full finetuning all not selected. Switching to 16bit LoRA.


```

--------------------------------

### Set up Training Arguments

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen3_VL_(8B)-Vision-GRPO.ipynb

Configure the training arguments for the Trainer.

```python
from transformers import TrainingArguments

args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=1,
    per_device_train_batch_size=2, # Adjust based on your GPU memory
    gradient_accumulation_steps=4, # Adjust based on your GPU memory
    gradient_checkpointing=True,
    optim="adamw_torch",
    logging_steps=10,
    save_steps=50,
    learning_rate=2e-5,
    fp16=False, # Set to True if using float16
    bf16=True, # Set to True if using bfloat16
    max_grad_norm=0.3,
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    report_to="tensorboard",
)

```

--------------------------------

### Re-prompting with Tool Results

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen2.5_Coder_(1.5B)-Tool_Calling.ipynb

Example of re-prompting the model with the results of tool calls to get a final answer.

```python
messages = []

original_prompt = user_query['content']

prompt_with_context = f"""You are a super helpful AI assistant.
You are asked to answer a question based on the following context information.
Question:
{original_prompt}"""

messages.append({
    "role": "user",
    "content": prompt_with_context
})

tool_call_id = generate_alphanumeric()
tool_calls = [{
    "id": tool_call_id,
    "type": "function",
    "function": {
        "name": "inventory_check",
        "arguments": arguments
    }
}]

messages.append({
    "role": "assistant",
    "tool_calls": tool_calls
})
messages.append({
    "role": "tool",
    "name": "inventory_check",
    "content": result_total  # pass the result total
})

messages.append({
    "role": "assistant",
    "content": "Answer:\n"
})

tokenizer = copy.deepcopy(tokenizer_orig)
tool_prompt = tokenizer.apply_chat_template(
    messages,
    continue_final_message = True,
    add_special_tokens = True,
    return_tensors = "pt",
    return_dict = True,
    tools = None,
)
tool_prompt = tool_prompt.to(model.device)

print(tokenizer.decode(tool_prompt['input_ids'][0]))
```

--------------------------------

### Inference Example

Source: https://github.com/unslothai/notebooks/blob/main/nb/Gemma2_(2B)-Alpaca.ipynb

A placeholder for inference code, indicating where to start running the trained model. It suggests modifying the instruction and input, leaving the output blank.

```python
```python
```

--------------------------------

### Start Fine-Tuning

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen2_5_7B_VL_GRPO.ipynb

Initiates the fine-tuning process using the configured trainer and dataset. The trainer will save checkpoints and logs during training.

```python
# Start the fine-tuning process
trainer.train()

# Save the fine-tuned model and tokenizer
trainer.save_model("/content/drive/MyDrive/unsloth_models/Qwen2-5B-VL-Instruct")
tokenizer.save_pretrained("/content/drive/MyDrive/unsloth_models/Qwen2-5B-VL-Instruct")

print("Fine-tuning complete. Model saved.")

```

--------------------------------

### Initialize Trainer and Train

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen2.5_(3B)-GRPO.ipynb

Initialize the Trainer with the model, tokenizer, dataset, and training arguments, then start the training process.

```python
from trl import SFTTrainer

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=data,
    dataset_text_field="text",
    max_seq_length=2048,
    formatting_func=formatting_prompts_func,
    args=args,
)

trainer.train()

```

--------------------------------

### Inference Setup

Source: https://github.com/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb

This code block provides a starting point for running inference with the fine-tuned model, including prompt and image file path configurations.

```python
prompt = "<image>\nFree OCR. "
image_file = 'your_image.jpg'
output_path = 'your/output/dir'

# Tiny: base_size = 512, image_size = 512, crop_mode = False
# Small: base_size = 640, image_size = 640, crop_mode = False
# Base: base_size = 1024, image_size = 1024, crop_mode = False
# Large: base_size = 1280, image_size = 1280, crop_mode = False
```

--------------------------------

### Install Unsloth and dependencies

Source: https://github.com/unslothai/notebooks/blob/main/nb/Kaggle-Qwen3_(4B)_Instruct-QAT.ipynb

Installs Unsloth and necessary libraries, with specific handling for Google Colab environments to ensure compatibility with different PyTorch versions.

```python
%%capture
import os, re
if "COLAB_" not in "".join(os.environ.keys()):
    !pip install unsloth
else:
    # Do this only in Colab notebooks! Otherwise use pip install unsloth
    import torch; v = re.match(r"[0-9]{1,}\.[0-9]{1,}", str(torch.__version__)).group(0)
    xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, "0.0.34")
    !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth
    !pip install --no-deps --upgrade "torchao>=0.16.0"
    !pip install sentencepiece protobuf "datasets==4.3.0" "huggingface_hub>=0.34.0" hf_transfer
try:
    import torch; _qat_torch_minor = re.match(r"[0-9]{1,}\.[0-9]{1,}", str(torch.__version__)).group(0)
except Exception:
    _qat_torch_minor = ""
_qat_torchao_map = {"2.10":"0.16.0","2.8":"0.14.1","2.9":"0.15.0"}
_qat_torchao = _qat_torchao_map.get(_qat_torch_minor, "0.16.0")
_qat_fbgemm_map = {"2.10":"1.5.0","2.8":"1.3.0","2.9":"1.4.2"}
_qat_fbgemm = _qat_fbgemm_map.get(_qat_torch_minor, "1.5.0")
!pip install --upgrade --force-reinstall torchao=={_qat_torchao} fbgemm-gpu-genai=={_qat_fbgemm}
!pip install transformers==4.55.4 && pip install --no-deps trl==0.22.2
```

--------------------------------

### Example of Tool Use with Gemma

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-FunctionGemma_(270M).ipynb

This snippet shows how a model might use a tool to get Amazon product details based on an ASIN.

```python
messages_4 = [
    {
        "role": "assistant",
        "content": (
            "<think>User is asking for an opinion, but I need factual product details first "
            "such as price, features, and reviews. I should call the Amazon product "
            "details tool with the provided ASIN."
            "</think>"
        ),
        "tool_calls": [
            {
                "id": "call_1",
                "type": "function",
                "function": {
                    "name": "get_amazon_product_details",
                    "arguments": {
                        "asin": "B0XYZ12345"
                    },
                },
            }
        ],
    },
    {
        "role": "tool",
        "name": "get_amazon_product_details",
        "tool_call_id": "call_1",
        "content": (
            '{"title": "Home Pro Espresso 3000", '
            '"price": 199.99, '
            '"pressure_bar": 15, '
            '"features": ["steam wand", "single and double shot baskets"], '
            '"pros": ["good crema", "compact"], '
            '"cons": ["a bit noisy"]}'
        ),
    },
    {
        "role": "assistant",
        "content": (
            "<think>"
            "Tool response shows a mid-range price and standard 15 bar pressure. "
            "Features and pros/cons indicate it’s fine for home espresso but not "
            "a high-end machine for enthusiasts."
            "</think>\n"
            "Based on the product details, the Home Pro Espresso 3000 (ASIN B0XYZ12345) "
            "is a solid option for home use. It offers 15-bar pressure, a steam wand, "
            "and both single and double shot baskets, which are enough for everyday "
            "lattes and cappuccinos. It’s compact and produces good crema, but it can "
            "be a bit noisy. If you want a convenient, reasonably priced home machine, "
            "it should work well; if you’re very picky about espresso or plan to upgrade "
            "grinders and accessories, you might eventually want something more advanced."
        ),
    },
]

rendered_prompt = tokenizer.apply_chat_template(
    messages_4,
    tools = tools_4,
    add_generation_prompt = False,  # True if you want to open a fresh model turn for generation
    tokenize = False,
)

print("=== Thinking + Tools ===")
print(rendered_prompt)
```

```

--------------------------------

### Unsloth Fine-tuning

Source: https://github.com/unslothai/notebooks/blob/main/original_template/Phi_4_(14B)-GRPO.ipynb

This code snippet shows the basic setup for fine-tuning a model with Unsloth, including installing the library, loading the model and tokenizer, and configuring training arguments.

```python
from unsloth import FastLanguageModel
import torch

# Load the model and tokenizer
model, tokenizer = FastLanguageModel.from_pretrained(
    model="unsloth/phi-3-mini-4k-instruct",
    torch_dtype=torch.bfloat16, # Memory optimization
    load_in_4bit=True, # Memory optimization
)

# Configure training arguments
model = FastLanguageModel.get(
    args=TrainingArguments(
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        warmup_ratio=0.03,
        max_steps=500,
        num_train_epochs=3,
        learning_rate=2e-4,
        fp16=False, # Use bf16 for better performance on Ampere GPUs
        bf16=True,
        logging_steps=1,
        optim="adamw_torch",
        weight_decay=0.001,
        lr_scheduler_type="cosine",
        seed=42, # random seed
    ),
)

# Add LoRA adapter
model = model.add_lora_adapter(
    lora_config=LoraConfig(
        r=16, # Rank
        lora_alpha=32, # Alpha
                target_modules="all", # Target modules
        lora_dropout=0.05,
        bias=False,
        task_type="CAUSAL_LM",
    )
)

# Prepare dataset
dataset = load_dataset("json", data_files="alpaca_data.json")

# Start training
trainer = Trainer(
    model=model,
    train_dataset=dataset["train"],
    args=model.args,
    data_collator=data_collator,
)

# Start training
trainer.train()

# Save the model
model.save_model("my_finetuned_model")
```

--------------------------------

### Start Training

Source: https://github.com/unslothai/notebooks/blob/main/nb/Qwen2.5_(3B)-GRPO.ipynb

Initialize and start the training process using the configured model and dataset.

```python
from trl import SFTTrainer

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=data["train"],
    dataset_text_field="text", # or "content" or "column"
    max_seq_length=1024, # max sequence length
    args=TrainingArguments(
        per_device_train_batch_size=2, # batch size per device
        gradient_accumulation_steps=4, # gradient accumulation steps
        warmup_steps=5, # warmup steps
        max_steps=50, # max training steps
        learning_rate=2e-4, # learning rate
        fp16=True, # enable fp16 training
        logging_steps=1, # log every step
        output_dir="outputs", # output directory
        optim="adamw_torch", # optimizer
        # lr_scheduler_type="cosine", # learning rate scheduler type
        # disable_tqdm=True, # disable tqdm progress bar
    ),
    # formatting_func=formatting_prompts_func, # use formatting_func if your dataset is not in instruction format
)

trainer.train()

```

--------------------------------

### Unsloth Studio Chat Setup

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Unsloth_Studio.ipynb

This code block clones the Unsloth Studio repository and executes the chat module to start the Unsloth Studio Chat for Llama-3.1 8b.

```python
# @title ↙️ Press ▶ to start 🦥 Unsloth Studio Chat for Llama-3.1 8b

# Unsloth Studio
# Copyright (C) 2024-present the Unsloth AI team. All rights reserved.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
!git clone https://github.com/unslothai/studio > /dev/null 2>&1
with open("studio/unsloth_studio/chat.py", "r") as chat_module:
    code = chat_module.read()
exec(code)
```

--------------------------------

### Initialize SFTTrainer

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Qwen3_(4B)_Instruct-QAT.ipynb

Configures and initializes the SFTTrainer for fine-tuning the model. It includes settings for batch size, gradient accumulation, learning rate, and logging.

```python
from trl import SFTTrainer, SFTConfig
trainer = SFTTrainer(
    model = model,
    tokenizer = tokenizer,
    train_dataset = dataset,
    eval_dataset = None, # Can set up evaluation!
    args = SFTConfig(
        dataset_text_field = "text",
        per_device_train_batch_size = 1,
        gradient_accumulation_steps = 4, # Use GA to mimic batch size!
        warmup_steps = 5,
        # num_train_epochs = 1, # Set this for 1 full training run.
        max_steps = 30,
        learning_rate = 2e-4, # Reduce to 2e-5 for long training runs
        logging_steps = 1,
        optim = "adamw_8bit",
        weight_decay = 0.001,
        lr_scheduler_type = "linear",
        seed = 3407,
        report_to = "none", # Use this for WandB etc
    ),
)
```

--------------------------------

### Model Inference Setup for Base Model

Source: https://github.com/unslothai/notebooks/blob/main/nb/AMD-Gemma4_(E2B)_Reinforcement_Learning_Sudoku_Game.ipynb

This snippet shows the code for preparing the input text and running the model's generation process to get an initial output before reinforcement learning.

```python
text = tokenizer.apply_chat_template(
    [{"role": "user", "content": prompt.strip()}],
    tokenize = False,
    add_generation_prompt = True,
)

from transformers import TextStreamer
print("=" * 50)
print("BASE MODEL OUTPUT (before RL training):")
print("=" * 50)

inputs = tokenizer(
    text = text,
    add_special_tokens = False,
    return_tensors = "pt",
).to("cuda")

text_streamer = TextStreamer(tokenizer, skip_prompt = True)
result = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 128,
                        use_cache = True, temperature = 1.0, top_p = 0.95, top_k = 64)
```

```text
==================================================
BASE MODEL OUTPUT (before RL training):
==================================================
This is a complex request. Implementing a full, robust Sudoku solver strategy using *only* native Python built-in functions (no imports like `collections.Counter` or complex data structures beyond standard lists/dicts) requires implementing the core logic of constraint checking and candidate generation.

Since the goal is to find *a* valid next move, we will use a simple backtracking/constraint propagation approach:
1. Identify all empty cells.
2. For each empty cell, determine the set of valid numbers (1-9) that can be placed there without violating Sudoku rules based on the current `board`.
3.
```