Mixed precision training in PyTorch isn’t just about speed; it’s about unlocking a more efficient way for your GPU to manage its memory and arithmetic units, often leading to surprising performance gains even on older hardware.
Let’s see what this looks like in practice. Imagine we’re training a simple ResNet-18 on CIFAR-10.
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
from torch.cuda.amp import GradScaler, autocast
# --- Configuration ---
NUM_EPOCHS = 10
BATCH_SIZE = 128
LEARNING_RATE = 0.01
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# --- Data Loading ---
transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE,
shuffle=True, num_workers=4)
# --- Model Definition ---
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, in_planes, planes, stride=1):
super(BasicBlock, self).__init__()
self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
self.bn1 = nn.BatchNorm2d(planes)
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
self.bn2 = nn.BatchNorm2d(planes)
self.shortcut = nn.Sequential()
if stride != 1 or in_planes != self.expansion*planes:
self.shortcut = nn.Sequential(
nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
nn.BatchNorm2d(self.expansion*planes)
)
def forward(self, x):
out = torch.relu(self.bn1(self.conv1(x)))
out = self.bn2(self.conv2(out))
out += self.shortcut(x)
out = torch.relu(out)
return out
class ResNet(nn.Module):
def __init__(self, block, num_blocks, num_classes=10):
super(ResNet, self).__init__()
self.in_planes = 64
self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
self.linear = nn.Linear(512*BasicBlock.expansion, num_classes)
def _make_layer(self, block, planes, num_blocks, stride):
strides = [stride] + [1]*(num_blocks-1)
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
strides = [stride] + [1]*(num_blocks-1)
layers = []