### Prepare Data with Python API

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Configure featurizers and task definitions, then execute scaffold splitting and dataset featurization programmatically.

```python
import deepchem as dc
from prepare_data import generate_deepchem_splits, featurize_datasets, FEATURIZER_DICT, task_dict

# Supported featurizers
FEATURIZER_DICT = {
    "dmpnn": dc.feat.DMPNNFeaturizer(),
    "dummy": dc.feat.DummyFeaturizer(),
    "grover": dc.feat.GroverFeaturizer(features_generator=dc.feat.CircularFingerprint()),
    "ecfp": dc.feat.CircularFingerprint(size=1024),
    "molgraphconv": dc.feat.MolGraphConvFeaturizer(use_edges=True),
    "rdkit_conformer": dc.feat.RDKitConformerFeaturizer(),
}

# Task definitions for classification datasets
task_dict = {
    'bbbp': ['p_np'],
    'bace_classification': ['Class'],
    'clintox': ['FDA_APPROVED', 'CT_TOX'],
    'hiv': ['HIV_active'],
    'tox21': ['NR-AR', 'NR-AR-LBD', 'NR-AhR', 'NR-Aromatase', 'NR-ER', 'NR-ER-LBD',
              'NR-PPAR-gamma', 'SR-ARE', 'SR-ATAD5', 'SR-HSE', 'SR-MMP', 'SR-p53'],
    'delaney': ['measured log solubility in mols per litre'],
    'freesolv': ['y'],
    'lipo': ['exp'],
}

# Generate DeepChem scaffold splits
generate_deepchem_splits(
    dataset_names=['bbbp', 'bace_classification', 'tox21'],
    output_dir='./data/deepchem_splits',
    clean_smiles=True,
    max_smiles_len=200  # Following MoLFormer paper recommendation
)

# Featurize datasets for model training
featurize_datasets(
    dataset_names=['bbbp', 'bace_classification'],
    featurizer_names=['ecfp', 'molgraphconv', 'dummy'],
    data_root='./data/deepchem_splits',
    save_root='./data/featurized_datasets',
    smiles_column='smiles'
)
```

--------------------------------

### Run Data Preparation via CLI

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Execute the data preparation pipeline using command-line arguments to specify split types, datasets, and featurizers.

```bash
# Generate DeepChem scaffold splits and featurize with ECFP fingerprints
python3 prepare_data.py \
    --split_type 'deepchem' \
    --datasets 'delaney,bbbp,bace_classification,tox21' \
    --featurizers 'ecfp' \
    --data_dir ./data/datasets/deepchem_splits \
    --feat_dir ./data/featurized_datasets/deepchem_splits

# Use pre-existing MoLFormer splits for consistency with published benchmarks
python3 prepare_data.py \
    --split_type 'molformer' \
    --datasets 'delaney,bbbp,bace' \
    --featurizers 'dummy,molgraphconv,dmpnn' \
    --data_dir ./data/datasets/molformer_splits \
    --feat_dir ./data/featurized_datasets/molformer_splits
```

--------------------------------

### Configure Distributed Pretraining with Ray

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Sets up a TorchTrainer for distributed model pretraining. Includes configuration for scaling, checkpointing, and learning rate schedules.

```python
import ray
from ray.train.torch import TorchTrainer
from ray.train import Checkpoint, CheckpointConfig, RunConfig, ScalingConfig
import deepchem as dc
from deepchem.models.optimizers import Lamb

# Model mappings for pretraining
MODEL_MAPPINGS = {
    'chemberta': dc.models.torch_models.chemberta.Chemberta,
    'grover': dc.models.torch_models.GroverModel,
    'molformer': dc.models.torch_models.MoLFormer
}

# Learning rate schedule mappings
LR_SCHEDULE_MAPPINGS = {
    'polynomial_decay': dc.models.optimizers.PolynomialDecay,
    'exponential_decay': dc.models.optimizers.ExponentialDecay,
    'linear_cosine_decay': dc.models.optimizers.LinearCosineDecay,
    'lambda_lr_with_warmup': dc.models.optimizers.LambdaLRWithWarmup,
}

# Training configuration
train_loop_config = {
    'num_epochs': 4,
    'batch_size': 512,
    'model_name': 'molformer',
    'init_kwargs': {
        'batch_size': 512,
        'learning_rate': 'polynomial_decay',
        'initial_rate': 1e-4,
        'final_rate': 1e-6,
        'decay_steps': 1000000,
        'total_steps': 1000000,
        'checkpoint_frequency': 10000
    }
}

# Configure distributed training
scaling_config = ScalingConfig(num_workers=8, use_gpu=True)
checkpoint_config = CheckpointConfig(
    num_to_keep=5,
    checkpoint_score_attribute='iteration_loss',
    checkpoint_score_order='min'
)
run_config = RunConfig(
    checkpoint_config=checkpoint_config,
    name='molformer_zinc1b_pretrain',
    storage_path='s3://my-bucket/experiments'
)

# Create and run trainer
trainer = TorchTrainer(
    train_loop_per_worker=_train_loop_per_worker,
    train_loop_config=train_loop_config,
    scaling_config=scaling_config,
    run_config=run_config
)
result = trainer.fit()
best_checkpoint = result.get_best_checkpoint(metric='iteration_loss', mode='min')
print(f"Best checkpoint: {best_checkpoint.path}")
```

--------------------------------

### Execute Multi-node Training with torchrun

Source: https://github.com/deepforestsci/chemberta3/blob/main/distributed/td/README.md

Commands to initiate distributed training across multiple nodes using the torchrun utility.

```bash
torchrun --nproc-per-node=1 --nnodes=2 --node-rank=0 --rdzv-id=456 --rdzv-backend=c10d --rdzv-endpoint=172.16.130.32:16000 dc_multinode.py
```

```bash
torchrun --nproc-per-node=1 --nnodes=2 --node-rank=1 --rdzv-id=456 --rdzv-backend=c10d --rdzv-endpoint=172.16.130.32:16000 dc_multinode.py
```

--------------------------------

### Run ChemBERTa Fine-tuning Benchmark via CLI

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Initiate a ChemBERTa fine-tuning benchmark for classification tasks. This includes specifying datasets, batch size, epochs, pretrained model path, split name, and learning rate.

```bash
python3 chemberta_finetune_classification.py \
    --datasets "bace,bbbp,tox21,hiv,sider,clintox" \
    --batch_size 32 \
    --epochs 100 \
    --pretrained_model_path 'data/pretrained_model_checkpoints/pretrained_chemberta/chemberta-100M-mlm-4epochs' \
    --splits_name 'molformer_splits' \
    --learning_rate 3e-5
```

--------------------------------

### Run GCN Benchmark

Source: https://github.com/deepforestsci/chemberta3/blob/main/README.md

Execute triplicate benchmarks for the GCN model. Navigate to the 'gcn_benchmark' directory and run the classification script.

```bash
cd gcn_benchmark
bash gcn_classification_script.sh
```

--------------------------------

### Locate Log Files

Source: https://github.com/deepforestsci/chemberta3/blob/main/chemberta3_benchmarking/data/data_preprocessing/readme.md

Shows the default file paths where execution logs are saved.

```text
./deepchem_splits/deepchem_split_log_<timestamp>.log

./featurized/featurization_log_<timestamp>.log
```

--------------------------------

### Featurize Pre-split Datasets

Source: https://github.com/deepforestsci/chemberta3/blob/main/chemberta3_benchmarking/data/data_preprocessing/readme.md

Use this command to process datasets that are already split, such as those formatted for Molformer.

```bash
        python3 prepare_data.py\
            --split_type 'molformer' \
            --datasets 'delaney' \
            --featurizers 'ecfp' \
            --data_dir ./../datasets/molformer_splits \
            --feat_dir ./../featurized_datasets/molformer_splits \
```

--------------------------------

### Run GCN Regression Benchmark with Normalization via CLI

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Perform a GCN regression benchmark with data normalization enabled. This command sets up datasets, batch size, epochs, split name, transformation, and learning rate.

```bash
python3 gcn_regression_benchmark.py \
    --datasets "esol,freesolv,lipo" \
    --batch_size 32 \
    --epochs 100 \
    --splits_name 'molformer_splits' \
    --transform \
    --learning_rate 3e-5
```

--------------------------------

### Run Triplicate Benchmarking with DeepChem

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Executes three independent training runs with different random seeds to calculate mean and standard deviation. Requires pre-featurized datasets in the specified directory structure.

```python
import numpy as np
import deepchem as dc
from typing import List, Tuple, Callable

def set_seed(seed: int) -> None:
    """Set random seeds for reproducibility."""
    np.random.seed(seed)
    random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

def triplicate_benchmark_dc(
    dataset: str,
    splits_name: str,
    model_fn: Callable,
    metric: dc.metrics.Metric,
    tasks: List,
    batch_size: int,
    learning_rate: float,
    nb_epoch: int = 50
) -> Tuple[float, float]:
    """Run triplicate benchmark with different random seeds."""
    scores = []

    train_dataset = dc.data.DiskDataset(f'data/featurized_datasets/{splits_name}/molgraphconv_featurized/{dataset}/train')
    valid_dataset = dc.data.DiskDataset(f'data/featurized_datasets/{splits_name}/molgraphconv_featurized/{dataset}/valid')
    test_dataset = dc.data.DiskDataset(f'data/featurized_datasets/{splits_name}/molgraphconv_featurized/{dataset}/test')

    for run_id in range(3):
        set_seed(run_id)
        model = model_fn(tasks=tasks, model_dir=f'./model_run_{run_id}',
                         batch_size=batch_size, learning_rate=learning_rate)

        # Training loop with early stopping on validation
        best_score = -np.inf
        for epoch in range(nb_epoch):
            model.fit(train_dataset, nb_epoch=1, restore=epoch > 0)
            val_scores = model.evaluate(valid_dataset, [metric])
            if val_scores[metric.name] > best_score:
                best_score = val_scores[metric.name]
                model.save_checkpoint()

        # Evaluate on test set
        model.restore()
        test_scores = model.evaluate(test_dataset, [metric])
        scores.append(test_scores[metric.name])

    avg_score = np.mean(scores)
    std_score = np.std(scores)
    print(f"Triplicate Results — Avg: {avg_score:.4f} ± {std_score:.4f}")
    return avg_score, std_score

# Run triplicate benchmark
avg, std = triplicate_benchmark_dc(
    dataset='bbbp',
    splits_name='molformer_splits',
    model_fn=model_fn,
    metric=dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean),
    tasks=['p_np'],
    batch_size=32,
    learning_rate=3e-5,
    nb_epoch=100
)
```

--------------------------------

### View Output Directory Structure

Source: https://github.com/deepforestsci/chemberta3/blob/main/chemberta3_benchmarking/data/data_preprocessing/readme.md

Displays the expected directory structure for raw splits and processed featurized datasets.

```text
./deepchem_splits/
└── dataset/
    ├── train.csv
    ├── valid.csv
    └── test.csv

./featurized/
└── dmpnn_featurized/
    └── dataset/
        ├── train/
        ├── valid/
        └── test/
```

--------------------------------

### Load and Fine-tune ChemBERTa Model

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Shows how to load a pretrained ChemBERTa model and fine-tune it for classification tasks. It uses raw SMILES strings as input, leveraging DeepChem's DummyFeaturizer.

```python
import deepchem as dc
from deepchem.models.torch_models import Chemberta

def model_fn(tasks, model_dir, learning_rate, batch_size, pretrained_model_path):
    """Create a ChemBERTa model for classification with pretrained weights."""
    finetune_model = Chemberta(
        task='classification',
        learning_rate=learning_rate,
        batch_size=batch_size,
        n_tasks=len(tasks),
        model_dir=model_dir
    )
    finetune_model.load_from_pretrained(pretrained_model_path)
    return finetune_model

# Load datasets (ChemBERTa uses DummyFeaturizer - raw SMILES)
train_dataset = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/dummy_featurized/bbbp/train')
valid_dataset = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/dummy_featurized/bbbp/valid')
test_dataset = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/dummy_featurized/bbbp/test')
```

--------------------------------

### Apply Data Transformers for Regression

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Demonstrates applying data transformations like normalization to datasets for regression tasks. It defines a helper function to fit transformers on training data and apply them to all splits.

```python
import deepchem as dc
from deepchem.models.torch_models import GCNModel
from deepchem.molnet.load_function.molnet_loader import TransformerGenerator

# Define available transformers
transformers_mapping = {
    'balancing': TransformerGenerator(dc.trans.BalancingTransformer),
    'normalization': TransformerGenerator(dc.trans.NormalizationTransformer, transform_y=True),
    'minmax': TransformerGenerator(dc.trans.MinMaxTransformer, transform_y=True),
    'log': TransformerGenerator(dc.trans.LogTransformer, transform_y=True)
}

def transform_splits(train_dataset, valid_dataset, test_dataset, transformer_generators):
    """Apply transformations fitted on training data to all splits."""
    transformers = [
        transformers_mapping[t.lower()] if isinstance(t, str) else t
        for t in transformer_generators
    ]
    transformers = [t.create_transformer(train_dataset) for t in transformers]

    for transformer in transformers:
        train_dataset = transformer.transform(train_dataset)
        valid_dataset = transformer.transform(valid_dataset)
        test_dataset = transformer.transform(test_dataset)

    return (train_dataset, valid_dataset, test_dataset), transformers

# Create regression model
model = GCNModel(
    mode='regression',
    n_tasks=1,
    batch_size=32,
    learning_rate=3e-5,
    graph_conv_layers=[128, 128],
    batchnorm=True,
    dropout=0.2,
    predictor_hidden_feats=256,
    predictor_dropout=0.2,
    model_dir='./gcn_regression_model'
)

# Load datasets
train_ds = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/molgraphconv_featurized/esol/train')
valid_ds = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/molgraphconv_featurized/esol/valid')
test_ds = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/molgraphconv_featurized/esol/test')

# Apply normalization transformation
(train_ds, valid_ds, test_ds), transformers = transform_splits(
    train_ds, valid_ds, test_ds, ['normalization']
)

# Train and evaluate with RMSE metric
metric = dc.metrics.Metric(dc.metrics.rms_score)
model.fit(train_ds, nb_epoch=100)
test_scores = model.evaluate(dataset=test_ds, metrics=[metric], transformers=transformers)
print(f"Test RMSE: {test_scores[metric.name]:.4f}")
```

--------------------------------

### Run GCN Classification Benchmark via CLI

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Execute a GCN classification benchmark across multiple MoleculeNet datasets. This command configures datasets, batch size, epochs, split names, and learning rate.

```bash
python3 gcn_classification_benchmark.py \
    --datasets "bace,bbbp,tox21,hiv,sider,clintox" \
    --batch_size 32 \
    --epochs 100 \
    --splits_name 'molformer_splits' \
    --learning_rate 3e-5
```

--------------------------------

### Implement FusedLAMB Optimizer for MoLFormer

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Create a custom FusedLAMB optimizer wrapper for DeepChem models. This optimizer is suitable for large-scale molecular transformers like MoLFormer.

```python
import deepchem as dc
from deepchem.models.torch_models import MoLFormer
from deepchem.models.optimizers import Optimizer, LearningRateSchedule
from apex.optimizers import FusedLAMB

class FusedLamb(Optimizer):
    """FusedLAMB optimizer wrapper for DeepChem."""
    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08):
        super().__init__(learning_rate)
        self.beta1, self.beta2, self.epsilon = beta1, beta2, epsilon

    def _create_pytorch_optimizer(self, params):
        lr = self.learning_rate.initial_rate if isinstance(self.learning_rate, LearningRateSchedule) else self.learning_rate
        return FusedLAMB(params, lr=lr, betas=(self.beta1, self.beta2), eps=self.epsilon)

# Create MoLFormer model with FusedLAMB optimizer
finetune_model = MoLFormer(
    task='classification',
    optimizer=FusedLamb(learning_rate=3e-5),
    batch_size=32,
    n_tasks=1,
    model_dir='./molformer_finetune'
)
finetune_model.load_from_pretrained('data/pretrained_model_checkpoints/pretrained_molformer/molformer-1.1B')

# Load and train
train_ds = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/dummy_featurized/bbbp/train')
valid_ds = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/dummy_featurized/bbbp/valid')

metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean)
finetune_model.fit(train_ds, nb_epoch=100)
scores = finetune_model.evaluate(valid_ds, [metric])
print(f"Validation ROC-AUC: {scores[metric.name]:.4f}")
```

--------------------------------

### Fine-tune ChemBERTa Model

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Fine-tune a pretrained ChemBERTa model on a specific task. Ensure the dataset and pretrained model path are correctly specified.

```python
model = model_fn(
    tasks=['p_np'],
    model_dir='./chemberta_finetune',
    learning_rate=3e-5,
    batch_size=32,
    pretrained_model_path='data/pretrained_model_checkpoints/pretrained_chemberta/chemberta-100M-mlm-4epochs'
)

# Fine-tune on BBBP dataset
metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean)
for epoch in range(100):
    loss = model.fit(train_dataset, nb_epoch=1, restore=epoch > 0, max_checkpoints_to_keep=1)
    scores = model.evaluate(valid_dataset, [metric])
    print(f"Epoch {epoch+1} | Loss: {loss:.4f} | Val ROC-AUC: {scores[metric.name]:.4f}")

# Evaluate
test_scores = model.evaluate(test_dataset, [metric])
print(f"Test ROC-AUC: {test_scores[metric.name]:.4f}")
```

--------------------------------

### Generate DeepChem Scaffold Splits

Source: https://github.com/deepforestsci/chemberta3/blob/main/chemberta3_benchmarking/data/data_preprocessing/readme.md

Use this command to generate train, validation, and test splits for datasets using DeepChem's scaffold splitting.

```bash
        python3 prepare_data.py\
            --split_type 'deepchem' \
            --datasets 'delaney' \
            --featurizers 'ecfp' \
            --data_dir ./../datasets/deepchem_splits \
            --feat_dir ./../featurized_datasets/deepchem_splits \
```

--------------------------------

### Train GCN Model for Classification

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Loads featurized datasets, defines metrics and tasks, and trains a GCN model for classification. It includes a training loop with validation and evaluation on a test set.

```python
train_dataset = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/molgraphconv_featurized/bbbp/train')
valid_dataset = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/molgraphconv_featurized/bbbp/valid')
test_dataset = dc.data.DiskDataset('data/featurized_datasets/molformer_splits/molgraphconv_featurized/bbbp/test')

# Define metric and tasks
metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean)
tasks = ['p_np']

# Create and train model
model = model_fn(tasks=tasks, model_dir='./gcn_model', batch_size=32, learning_rate=3e-5)

best_score = -np.inf
for epoch in range(100):
    loss = model.fit(train_dataset, nb_epoch=1, restore=epoch > 0, max_checkpoints_to_keep=1)
    scores = model.evaluate(valid_dataset, [metric])
    val_score = scores[metric.name]
    print(f"Epoch {epoch+1}/100 | Train Loss: {loss:.4f} | Val ROC-AUC: {val_score:.4f}")

    if val_score > best_score:
        best_score = val_score
        # Save best model checkpoint

# Evaluate on test set
test_scores = model.evaluate(test_dataset, [metric])
print(f"Test ROC-AUC: {test_scores[metric.name]:.4f}")
```

--------------------------------

### Define GCN Model for Classification

Source: https://context7.com/deepforestsci/chemberta3/llms.txt

Initialize a Graph Convolutional Network model using DeepChem's torch_models module for classification tasks.

```python
import numpy as np
import deepchem as dc
from deepchem.models.torch_models import GCNModel

def model_fn(tasks, model_dir, batch_size, learning_rate):
    """Create a GCN model for classification."""
    model = GCNModel(
        n_tasks=len(tasks),
        mode='classification',
        batch_size=batch_size,
        learning_rate=learning_rate,
        graph_conv_layers=[128, 128],
        batchnorm=True,
        dropout=0.2,
        predictor_hidden_feats=256,
        predictor_dropout=0.2,
        model_dir=model_dir
    )
    return model
```

=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.