Learning Rate Scheduler학습률 스케줄러

import torch
import torch.optim as optim
import math

model = torch.nn.Linear(10, 1)
optimizer = optim.Adam(model.parameters(), lr=1e-3)

# 1. StepLR: 일정 epoch마다 감소
scheduler1 = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)

# 2. CosineAnnealingLR: 코사인 곡선으로 감소
scheduler2 = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=100, eta_min=1e-6)

# 3. ReduceLROnPlateau: 검증 손실이 개선 없으면 감소
scheduler3 = optim.lr_scheduler.ReduceLROnPlateau(
    optimizer, mode='min', factor=0.5, patience=10, verbose=True)

# 4. OneCycleLR: 1 사이클 정책 (빠른 수렴)
scheduler4 = optim.lr_scheduler.OneCycleLR(
    optimizer, max_lr=1e-2, total_steps=1000)

# 5. 워밍업 + 코사인 감소 (직접 구현)
def get_lr_with_warmup(step, warmup_steps, total_steps, max_lr):
    if step < warmup_steps:
        return max_lr * step / warmup_steps
    progress = (step - warmup_steps) / (total_steps - warmup_steps)
    return max_lr * 0.5 * (1 + math.cos(math.pi * progress))

# 커스텀 람다 스케줄러
scheduler5 = optim.lr_scheduler.LambdaLR(
    optimizer,
    lr_lambda=lambda step: get_lr_with_warmup(step, 100, 1000, 1.0)
)

for epoch in range(num_epochs):
    for batch in dataloader:
        loss = train_step(batch)
        optimizer.step()
        scheduler4.step()  # OneCycleLR은 배치마다

    # ReduceLROnPlateau는 epoch마다 val_loss 제공
    val_loss = validate()
    scheduler3.step(val_loss)

    # 나머지는 epoch마다
    scheduler1.step()
    print(f"Epoch {epoch}, LR: {optimizer.param_groups[0]['lr']:.6f}")