Sign In

dinushasan / transfer-learning-pytorch 0 0
2 months ago

Notebook Files Collaborators Version 2 (2 months ago)

Transfer Learning for Image Classification in PyTorch

How a CNN learns (source):

Layer visualization (source):

Downloading the Dataset

We'll use the Oxford-IIIT Pets dataset from https://course.fast.ai/datasets . It is 37 category (breeds) pet dataset with
roughly 200 images for each class. The images have a large variations in scale, pose and lighting.

In [2]:

!pip install jovian --upgrade --quiet

In [3]:

from torchvision.datasets.utils import download_url

In [4]:

download_url('https://s3.amazonaws.com/fast-ai-imageclas/oxford-iiit-pet.tgz', '.')

Downloading https://s3.amazonaws.com/fast-ai-imageclas/oxford-iiit-pet.tgz to ./oxford-iiit-

pet.tgz
HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))

In [5]:

import tarfile

with tarfile.open('./oxford-iiit-pet.tgz', 'r:gz') as tar:

tar.extractall(path='./data')

In [6]:

from torch.utils.data import Dataset

In [7]:

import os

DATA_DIR = './data/oxford-iiit-pet/images'

files = os.listdir(DATA_DIR)
files[:5]

Out[7]:
['shiba_inu_181.jpg',
'scottish_terrier_180.jpg',
'american_pit_bull_terrier_161.jpg',
'wheaten_terrier_116.jpg',
'american_pit_bull_terrier_55.jpg']

In [8]:

def parse_breed(fname):
 parts = fname.split('_')
return ' '.join(parts[:-1])

In [9]:

parse_breed(files[4])

Out[9]:
'american pit bull terrier'

In [10]:

from PIL import Image

def open_image(path):
with open(path, 'rb') as f:
img = Image.open(f)
return img.convert('RGB')

Creating a Custom PyTorch Dataset

In [11]:

import os

class PetsDataset(Dataset):
def __init__(self, root, transform):
super().__init__()
self.root = root
self.files = [fname for fname in os.listdir(root) if fname.endswith('.jpg')]
self.classes = list(set(parse_breed(fname) for fname in files))
self.transform = transform

def __len__(self):
return len(self.files)

def __getitem__(self, i):

fname = self.files[i]
fpath = os.path.join(self.root, fname)
img = self.transform(open_image(fpath))
class_idx = self.classes.index(parse_breed(fname))
return img, class_idx

In [12]:

import torchvision.transforms as T

img_size = 224
imagenet_stats = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
dataset = PetsDataset(DATA DIR T Compose([T Resize(img size)
dataset = PetsDataset(DATA_DIR, T.Compose([T.Resize(img_size),
T.Pad(8, padding_mode='reflect'),
T.RandomCrop(img_size),
T.ToTensor(),
T.Normalize(*imagenet_stats)]))

In [13]:

len(dataset)

Out[13]:
7390

In [14]:

import torch
import matplotlib.pyplot as plt
%matplotlib inline

def denormalize(images, means, stds):

if len(images.shape) == 3:
images = images.unsqueeze(0)
means = torch.tensor(means).reshape(1, 3, 1, 1)
stds = torch.tensor(stds).reshape(1, 3, 1, 1)
return images * stds + means

def show_image(img_tensor, label):

print('Label:', dataset.classes[label], '(' + str(label) + ')')
img_tensor = denormalize(img_tensor, *imagenet_stats)[0].permute((1, 2, 0))
plt.imshow(img_tensor)

In [15]:

show_image(*dataset[2])

Label: american pit bull terrier (0)

CreatingTrainingandValidation Sets
Creating Training and Validation Sets
In [16]:

from torch.utils.data import random_split

In [17]:

val_pct = 0.1
val_size = int(val_pct * len(dataset))

train_ds, valid_ds = random_split(dataset, [len(dataset) - val_size, val_size])

In [18]:

from torch.utils.data import DataLoader

batch_size = 256

train_dl = DataLoader(train_ds, batch_size, shuffle=True, num_workers=4, pin_memory=True)

valid_dl = DataLoader(valid_ds, batch_size*2, num_workers=4, pin_memory=True)

In [19]:

from torchvision.utils import make_grid

def show_batch(dl):
for images, labels in dl:
fig, ax = plt.subplots(figsize=(16, 16))
ax.set_xticks([]); ax.set_yticks([])
images = denormalize(images[:64], *imagenet_stats)
ax.imshow(make_grid(images, nrow=8).permute(1, 2, 0))
break

In [20]:

show_batch(train_dl)
Modifying a Pretrained Model (ResNet34)
Transfer learning (source):

In [21]:

import torch.nn as nn
import torch.nn.functional as F

def accuracy(outputs, labels):

_, preds = torch.max(outputs, dim=1)
return torch.tensor(torch.sum(preds == labels).item() / len(preds))
 class ImageClassificationBase(nn.Module):
def training_step(self, batch):
images, labels = batch
out = self(images) # Generate predictions
loss = F.cross_entropy(out, labels) # Calculate loss
return loss

def validation_step(self, batch):

images, labels = batch
out = self(images) # Generate predictions
loss = F.cross_entropy(out, labels) # Calculate loss
acc = accuracy(out, labels) # Calculate accuracy
return {'val_loss': loss.detach(), 'val_acc': acc}

def validation_epoch_end(self, outputs):

batch_losses = [x['val_loss'] for x in outputs]
epoch_loss = torch.stack(batch_losses).mean() # Combine losses
batch_accs = [x['val_acc'] for x in outputs]
epoch_acc = torch.stack(batch_accs).mean() # Combine accuracies
return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}

def epoch_end(self, epoch, result):

print("Epoch [{}],{} train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}".format(
epoch, "last_lr: {:.5f},".format(result['lrs'][-1]) if 'lrs' in result else '',
result['train_loss'], result['val_loss'], result['val_acc']))

In [22]:

from torchvision import models

class PetsModel(ImageClassificationBase):
def __init__(self, num_classes, pretrained=True):
super().__init__()
# Use a pretrained model
self.network = models.resnet34(pretrained=pretrained)
# Replace last layer
self.network.fc = nn.Linear(self.network.fc.in_features, num_classes)

def forward(self, xb):

return self.network(xb)

In [24]:

models.resnet34(pretrained=True)

Downloading: "https://download.pytorch.org/models/resnet34-333f7ec4.pth" to
/root/.cache/torch/hub/checkpoints/resnet34-333f7ec4.pth
HBox(children=(FloatProgress(value=0 0 max=87306240 0) HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=87306240.0), HTML(value= )))

Out[24]:
ResNet(
(conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
(layer1): Sequential(
(0): BasicBlock(
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(1): BasicBlock(
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(2): BasicBlock(
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)

GPU Utilities and Training Loop

In [25]:

def get_default_device():
"""Pick GPU if available, else CPU"""
if torch.cuda.is_available():
return torch.device('cuda')
else:
return torch.device('cpu')

def to_device(data, device):

"""Move tensor(s) to chosen device"""
if isinstance(data, (list, tuple)):
return [to_device(x, device) for x in data]
return data.to(device, non_blocking=True)

class DeviceDataLoader():
"""Wrap a dataloader to move data to a device"""

( )
 def __init__(self, dl, device):
self.dl = dl

self.device = device

def __iter__(self):
"""Yield a batch of data after moving it to device"""
for b in self.dl:
yield to_device(b, self.device)

def __len__(self):
"""Number of batches"""
return len(self.dl)

In [26]:

import torch
from tqdm.notebook import tqdm

@torch.no_grad()
def evaluate(model, val_loader):
model.eval()
outputs = [model.validation_step(batch) for batch in val_loader]
return model.validation_epoch_end(outputs)

def fit(epochs, lr, model, train_loader, val_loader, opt_func=torch.optim.SGD):

history = []
optimizer = opt_func(model.parameters(), lr)
for epoch in range(epochs):
# Training Phase
model.train()
train_losses = []
for batch in tqdm(train_loader):
loss = model.training_step(batch)
train_losses.append(loss)
loss.backward()
optimizer.step()
optimizer.zero_grad()
# Validation phase
result = evaluate(model, val_loader)
result['train_loss'] = torch.stack(train_losses).mean().item()
model.epoch_end(epoch, result)
history.append(result)
return history

def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']

def fit_one_cycle(epochs, max_lr, model, train_loader, val_loader,

weight decay=0 grad clip=None opt func=torch optim SGD):
 weight_decay=0, grad_clip=None, opt_func=torch.optim.SGD):
torch.cuda.empty_cache()
history = []

# Set up custom optimizer with weight decay

optimizer = opt_func(model.parameters(), max_lr, weight_decay=weight_decay)
# Set up one-cycle learning rate scheduler
sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, epochs=epochs,
steps_per_epoch=len(train_loader))

for epoch in range(epochs):

# Training Phase
model.train()
train_losses = []
lrs = []
for batch in tqdm(train_loader):
loss = model.training_step(batch)
train_losses.append(loss)
loss.backward()

# Gradient clipping
if grad_clip:
nn.utils.clip_grad_value_(model.parameters(), grad_clip)

optimizer.step()
optimizer.zero_grad()

# Record & update learning rate

lrs.append(get_lr(optimizer))
sched.step()

# Validation phase
result = evaluate(model, val_loader)
result['train_loss'] = torch.stack(train_losses).mean().item()
result['lrs'] = lrs
model.epoch_end(epoch, result)
history.append(result)
return history

In [27]:

device = get_default_device()
device

Out[27]:
device(type='cuda')

In [28]:

train_dl = DeviceDataLoader(train_dl, device)

valid_dl = DeviceDataLoader(valid_dl, device)
Finetuning the Pretrained Model
In [29]:

model = PetsModel(len(dataset.classes))
to_device(model, device);

In [30]:

history = [evaluate(model, valid_dl)]

history

Out[30]:
[{'val_acc': 0.02273196540772915, 'val_loss': 3.9689669609069824}]

In [31]:

epochs = 6
max_lr = 0.01
grad_clip = 0.1
weight_decay = 1e-4
opt_func = torch.optim.Adam

In [32]:

%%time
history += fit_one_cycle(epochs, max_lr, model, train_dl, valid_dl,
grad_clip=grad_clip,
weight_decay=weight_decay,
opt_func=opt_func)
HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [0],last_lr: 0.00589, train_loss: 1.3532, val_loss: 105.4349, val_acc: 0.0149

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [1],last_lr: 0.00994, train_loss: 2.0033, val_loss: 3.2823, val_acc: 0.1467

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [2],last_lr: 0.00812, train_loss: 1.3266, val_loss: 2.3671, val_acc: 0.3779

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [3],last_lr: 0.00463, train_loss: 0.8274, val_loss: 2.0568, val_acc: 0.4321

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [4],last lr: 0.00133, train loss: 0.4891, val loss: 0.8032, val acc: 0.7526
Epoch [4],last_lr: 0.00133, train_loss: 0.4891, val_loss: 0.8032, val_acc: 0.7526
HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [5],last_lr: 0.00000, train_loss: 0.3090, val_loss: 0.6730, val_acc: 0.7900

CPU times: user 49.2 s, sys: 36.4 s, total: 1min 25s
Wall time: 2min 50s

Training a model from scratch

Let's repeat the training without using weights from the pretrained ResNet34 model.

In [33]:

model2 = PetsModel(len(dataset.classes), pretrained=False)

to_device(model2, device);

In [34]:

history2 = [evaluate(model2, valid_dl)]

history2

Out[34]:
[{'val_acc': 0.02125636674463749, 'val_loss': 89.57513427734375}]

In [35]:

%%time
history2 += fit_one_cycle(epochs, max_lr, model2, train_dl, valid_dl,
grad_clip=grad_clip,
weight_decay=weight_decay,
opt_func=opt_func)
HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [0],last_lr: 0.00589, train_loss: 3.5611, val_loss: 191.8064, val_acc: 0.0225

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [1],last_lr: 0.00994, train_loss: 3.4294, val_loss: 6.0370, val_acc: 0.0733

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [2],last_lr: 0.00812, train_loss: 3.1648, val_loss: 3.4771, val_acc: 0.0873

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [3],last_lr: 0.00463, train_loss: 2.9549, val_loss: 3.2375, val_acc: 0.1161

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))
Epoch [4],last_lr: 0.00133, train_loss: 2.6875, val_loss: 2.9029, val_acc: 0.1638

HBox(children=(FloatProgress(value=0.0, max=26.0), HTML(value='')))

Epoch [5],last_lr: 0.00000, train_loss: 2.4688, val_loss: 2.7200, val_acc: 0.2486

CPU times: user 50 s, sys: 36.7 s, total: 1min 26s
Wall time: 2min 49s

While the pretrained model reached an accuracy of 80% in less than 3 minutes, the model without pretrained weights
could only reach an accuracy of 24%.

In [36]:

!pip install jovian --upgrade --quiet

In [37]:

import jovian

In [38]:

jovian.commit(project='transfer-learning-pytorch')

[jovian] Detected Colab notebook...

[jovian] Please enter your API key ( from https://jovian.ai/ ):
API KEY: ··········
[jovian] Uploading colab notebook to Jovian...
[jovian] Capturing environment..
[jovian] Committed successfully! https://jovian.ai/dinushasan/transfer-learning-pytorch
Out[38]:
'https://jovian.ai/dinushasan/transfer-learning-pytorch'

Docs Forum Help [email protected]