Transfer Learning

1import torch.nn as nn
2from torchvision import models
3
4# Load pretrained ResNet
5model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V2)
6
7# Freeze ALL backbone parameters
8for param in model.parameters():
9    param.requires_grad = False
10
11# Replace classifier head
12num_features = model.fc.in_features
13model.fc = nn.Sequential(
14    nn.Dropout(0.5),
15    nn.Linear(num_features, 256),
16    nn.ReLU(),
17    nn.Dropout(0.3),
18    nn.Linear(256, num_classes)
19)
20
21# Only head parameters are trainable
22optimizer = torch.optim.Adam(model.fc.parameters(), lr=1e-3)

1# Different learning rates for different depths
2# Key insight: earlier = smaller LR, later = larger LR
3
4param_groups = [
5    # Earliest layers: almost frozen
6    {'params': model.conv1.parameters(), 'lr': 1e-6},
7    {'params': model.bn1.parameters(), 'lr': 1e-6},
8    {'params': model.layer1.parameters(), 'lr': 1e-6},
9
10    # Early-mid layers: very small updates
11    {'params': model.layer2.parameters(), 'lr': 5e-6},
12
13    # Mid layers: moderate updates
14    {'params': model.layer3.parameters(), 'lr': 1e-5},
15
16    # Late layers: larger updates
17    {'params': model.layer4.parameters(), 'lr': 5e-5},
18
19    # New head: normal learning rate
20    {'params': model.fc.parameters(), 'lr': 1e-3},
21]
22
23optimizer = torch.optim.AdamW(param_groups, weight_decay=0.01)

1def get_layer_groups(model):
2    """Get ResNet layer groups from shallow to deep."""
3    return [
4        [model.conv1, model.bn1],
5        [model.layer1],
6        [model.layer2],
7        [model.layer3],
8        [model.layer4],
9        [model.fc]
10    ]
11
12def unfreeze_layer_group(model, group_idx, layer_groups):
13    """Unfreeze a specific layer group."""
14    for param in layer_groups[group_idx].parameters():
15        param.requires_grad = True
16
17# Training schedule
18layer_groups = get_layer_groups(model)
19
20# Phase 1: Only head (5 epochs, lr=1e-3)
21for group in layer_groups[:-1]:
22    for param in group.parameters():
23        param.requires_grad = False
24train(model, epochs=5, lr=1e-3)
25
26# Phase 2: Unfreeze layer4 (5 epochs, lr=1e-4)
27unfreeze_layer_group(model, 4, layer_groups)
28train(model, epochs=5, lr=1e-4)
29
30# Phase 3: Unfreeze layer3 (10 epochs, lr=1e-5)
31unfreeze_layer_group(model, 3, layer_groups)
32train(model, epochs=10, lr=1e-5)

1# Medical imaging specific preprocessing
2medical_transforms = transforms.Compose([
3    transforms.Resize(256),
4    transforms.CenterCrop(224),
5    transforms.Grayscale(num_output_channels=3),  # Convert to RGB
6    transforms.ToTensor(),
7    transforms.Normalize(
8        mean=[0.485, 0.456, 0.406],  # Still use ImageNet stats
9        std=[0.229, 0.224, 0.225]
10    ),
11])
12
13# Consider specialized pretrained models
14# RadImageNet, CheXNet, or DenseNet-121 pretrained on chest X-rays

1# CORRECT: Use weights-specific transforms
2from torchvision.models import ResNet50_Weights
3
4weights = ResNet50_Weights.IMAGENET1K_V2
5preprocess = weights.transforms()
6
7# Or manually specify ImageNet statistics
8preprocess = transforms.Compose([
9    transforms.Resize(256),
10    transforms.CenterCrop(224),
11    transforms.ToTensor(),
12    transforms.Normalize(
13        mean=[0.485, 0.456, 0.406],
14        std=[0.229, 0.224, 0.225]
15    )
16])
17
18# WRONG: Using different normalization
19# transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])  # Don't do this!

1# Option 1: Keep BatchNorm in eval mode during training
2def train_with_frozen_bn(model, dataloader, optimizer, criterion):
3    model.train()
4
5    # But keep BatchNorm layers in eval mode
6    for module in model.modules():
7        if isinstance(module, nn.BatchNorm2d):
8            module.eval()
9
10    # Training loop as usual...
11
12# Option 2: Freeze BatchNorm parameters
13for module in model.modules():
14    if isinstance(module, nn.BatchNorm2d):
15        module.weight.requires_grad = False
16        module.bias.requires_grad = False

1from torch.optim.lr_scheduler import (
2    CosineAnnealingLR,
3    OneCycleLR,
4    ReduceLROnPlateau
5)
6
7# Cosine annealing - smooth decay
8scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-6)
9
10# One cycle - fast convergence
11scheduler = OneCycleLR(
12    optimizer,
13    max_lr=1e-3,
14    epochs=num_epochs,
15    steps_per_epoch=len(train_loader),
16    pct_start=0.3,  # Warmup for 30% of training
17)
18
19# Reduce on plateau - adaptive
20scheduler = ReduceLROnPlateau(
21    optimizer,
22    mode='min',
23    factor=0.5,
24    patience=5,
25    min_lr=1e-7
26)

1# WRONG: Using arbitrary input size
2model = models.resnet50(weights=ResNet50_Weights.IMAGENET1K_V2)
3x = torch.randn(1, 3, 128, 128)  # Too small!
4
5# CORRECT: Use model's expected size (224 for most, 299 for Inception, 380 for EfficientNet-B4)
6x = torch.randn(1, 3, 224, 224)

1# WRONG: Training with 1000-class output
2model = models.resnet50(weights=ResNet50_Weights.IMAGENET1K_V2)
3# Oops, model.fc outputs 1000 classes, not our num_classes!
4
5# CORRECT: Replace the classification head
6model.fc = nn.Linear(model.fc.in_features, num_classes)

1# WRONG: Same learning rate for everything
2optimizer = Adam(model.parameters(), lr=1e-3)  # Will destroy early features!
3
4# CORRECT: Lower LR for pretrained layers
5optimizer = Adam([
6    {'params': model.conv1.parameters(), 'lr': 1e-5},
7    {'params': model.fc.parameters(), 'lr': 1e-3}
8])

1from torchvision import transforms
2
3train_transforms = transforms.Compose([
4    transforms.RandomResizedCrop(224, scale=(0.8, 1.0)),
5    transforms.RandomHorizontalFlip(),
6    transforms.RandomRotation(15),
7    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
8    transforms.ToTensor(),
9    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
10    transforms.RandomErasing(p=0.1),  # Cutout-style augmentation
11])