训练技巧与调优实战

import torch.optim as optim

# SGD + Momentum：计算机视觉经典选择，配合 lr warmup 效果出色
# 调得好能比 Adam 精度更高，但调参更麻烦
optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9,
                      weight_decay=1e-4, nesterov=True)

# Adam：默认首选，自适应学习率，调参简单
optimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-5)

# AdamW：Adam + 正确的权重衰减（推荐替代 Adam）
# Adam 的 weight_decay 实现有 bug（作用在梯度上而非参数上），AdamW 修复了这个问题
optimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-2)

from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR

optimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-2)

# Warmup：前 5 轮 lr 从 1e-4 线性增加到 1e-3（防止初期参数乱跳）
warmup_scheduler = LinearLR(optimizer, start_factor=0.1, end_factor=1.0, total_iters=5)
# 主调度：之后余弦退火（lr 从 1e-3 平滑降至 1e-5）
cosine_scheduler = CosineAnnealingLR(optimizer, T_max=95, eta_min=1e-5)
# 按顺序执行
scheduler = SequentialLR(optimizer, [warmup_scheduler, cosine_scheduler],
                         milestones=[5])

for epoch in range(100):
    train_one_epoch(...)
    scheduler.step()
    print(f"Epoch {epoch}: lr = {optimizer.param_groups[0]['lr']:.2e}")

# OneCycleLR：先升后降的三角形学习率，通常 10-20 轮就能收敛
scheduler = optim.lr_scheduler.OneCycleLR(
    optimizer,
    max_lr=1e-2,               # 学习率峰值
    steps_per_epoch=len(train_loader),
    epochs=30,
    pct_start=0.3,             # 30% 时间用于 warmup
    anneal_strategy="cos",
)

# 注意：OneCycleLR 每个 batch 调用一次，不是每个 epoch
for epoch in range(30):
    for X, y in train_loader:
        ...
        optimizer.step()
        scheduler.step()   # 每 batch 调用

from torch_lr_finder import LRFinder

optimizer = optim.AdamW(model.parameters(), lr=1e-7)
lr_finder = LRFinder(model, optimizer, criterion, device=device)
lr_finder.range_test(train_loader, end_lr=10, num_iter=100)
lr_finder.plot()  # 找曲线损失下降最陡处对应的 lr
lr_finder.reset()
# 通常选「损失最低点前一个数量级」作为初始 lr

class RegularizedNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(256, 512),
            nn.ReLU(),
            nn.Dropout(p=0.3),    # 训练时随机关闭 30% 神经元

            nn.Linear(512, 256),
            nn.ReLU(),
            nn.Dropout(p=0.3),

            nn.Linear(256, 10),
        )

    def forward(self, x):
        return self.net(x)

# 关键：训练和评估模式要切换
model.train()  # Dropout 生效
model.eval()   # Dropout 关闭（用全部神经元）

# 普通 CrossEntropyLoss：正确类目标概率 1.0，其余 0.0
# Label Smoothing：正确类目标概率 0.9，其余均分剩余 0.1
criterion = nn.CrossEntropyLoss(label_smoothing=0.1)

class StochasticDepthResBlock(nn.Module):
    """训练时随机跳过整个残差块，相当于集成了不同深度的网络"""
    def __init__(self, block, survival_prob=0.8):
        super().__init__()
        self.block = block
        self.survival_prob = survival_prob

    def forward(self, x):
        if self.training:
            if torch.rand(1).item() > self.survival_prob:
                return x  # 跳过这个块
        return x + self.block(x) * (1 / self.survival_prob)

import torchvision.transforms as T
from torchvision.transforms import v2  # 新版 API，支持批量增强

# 图像分类的强增强组合
train_transform = T.Compose([
    T.RandomResizedCrop(224, scale=(0.08, 1.0)),
    T.RandomHorizontalFlip(),
    T.RandAugment(num_ops=2, magnitude=9),  # 自动策略搜索的增强
    T.ToTensor(),
    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
    T.RandomErasing(p=0.25),  # 随机擦除一块区域
])

# Mixup：把两张图像按比例混合（标签也混合）
mixup = v2.MixUp(num_classes=1000, alpha=0.2)

for X, y in train_loader:
    X, y = mixup(X, y)  # y 变成了软标签
    ...

import torch
from torch.cuda.amp import GradScaler, autocast

scaler = GradScaler()  # 用于防止 FP16 下溢（梯度太小变成 0）

for X, y in train_loader:
    X, y = X.to(device), y.to(device)
    optimizer.zero_grad()

    with autocast(dtype=torch.float16):  # 这个 with 块内自动用 FP16 计算
        output = model(X)
        loss   = criterion(output, y)

    scaler.scale(loss).backward()        # 放大梯度（防止下溢）
    scaler.unscale_(optimizer)
    nn.utils.clip_grad_norm_(model.parameters(), 1.0)
    scaler.step(optimizer)               # 缩放回 FP32 再更新参数
    scaler.update()

accumulation_steps = 8   # 等效 batch_size = 实际 batch × 8

for i, (X, y) in enumerate(train_loader):
    X, y = X.to(device), y.to(device)

    with autocast():
        loss = criterion(model(X), y) / accumulation_steps  # 每步 loss 除以累积步数

    scaler.scale(loss).backward()

    if (i + 1) % accumulation_steps == 0:
        scaler.unscale_(optimizer)
        nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        scaler.step(optimizer)
        scaler.update()
        optimizer.zero_grad()    # 每 accumulation_steps 步才清零

nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
# 或对单个参数组
nn.utils.clip_grad_value_(model.parameters(), clip_value=0.5)

def train_model(model, train_loader, val_loader, config):
    optimizer = optim.AdamW(model.parameters(),
                            lr=config["lr"], weight_decay=config["wd"])
    scheduler = optim.lr_scheduler.OneCycleLR(
        optimizer, max_lr=config["lr"],
        steps_per_epoch=len(train_loader), epochs=config["epochs"]
    )
    scaler    = GradScaler()
    criterion = nn.CrossEntropyLoss(label_smoothing=0.1)
    best_val_acc = 0.0

    for epoch in range(config["epochs"]):
        model.train()
        for X, y in train_loader:
            X, y = X.to(device), y.to(device)
            optimizer.zero_grad()
            with autocast():
                loss = criterion(model(X), y)
            scaler.scale(loss).backward()
            scaler.unscale_(optimizer)
            nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            scaler.step(optimizer)
            scaler.update()
            scheduler.step()

        val_acc = evaluate(model, val_loader)
        if val_acc > best_val_acc:
            best_val_acc = val_acc
            torch.save(model.state_dict(), "best_model.pt")

    return best_val_acc