PyTorch Basics: Tensors & Gradients

PyTorch is an open-source machine learning framework that is primarily used for

developing and training deep learning models. It was developed by Facebook's AI
Research Lab and released in 2016. PyTorch provides a flexible and dynamic
approach to building neural networks, making it a popular choice among
researchers and developers.

The framework is built on a dynamic computational graph concept, which means

that the graph is built and modified on-the-fly as the program runs. This allows for
more intuitive and flexible model development, as you can use standard Python
control flow statements and debug the model easily.

PyTorch supports automatic differentiation, which enables efficient computation of

gradients for training neural networks using backpropagation. It provides a rich set
of tools and libraries for tasks such as data loading, model building, optimization,
and evaluation.

One of the key advantages of PyTorch is its support for GPU acceleration, allowing
you to train models on GPUs to significantly speed up computations. It also has a
large and active community, which means there are plenty of resources, tutorials,
and pre-trained models available.

PyTorch is often compared to TensorFlow, another popular deep learning

framework. While TensorFlow focuses more on static computation graphs, PyTorch
emphasizes dynamic computation graphs. This fundamental difference in design
philosophy gives PyTorch an edge when it comes to flexibility and ease of use.
 Overall, PyTorch is widely used in the research community and is gaining popularity
in industry applications as well. It provides a powerful and user-friendly platform for
building and training deep learning models.

In [ ]: import torch

Tensors
In [ ]: # Number/Scalar

t1 = torch.tensor(4.)
t1

# 4. is a shorthand notation for 4.0

In [ ]: t1.dtype

In [ ]: #Vector
t2 = torch.tensor([1., 2, 3, 4])
t2

In [ ]: # Matrix
t3 = torch.tensor([
[5., 6],
[7,8],
[9,10]
])
t3

In [ ]: # 3 dimensional array
t4 = torch.tensor([
[
[11, 12, 13],
[14, 15, 16]
],
[
[17,18,19],
[20,21,22]
]
])

t4

In [ ]: print(t1)
t1.shape

In [ ]: print(t2)
t2.shape
In [ ]: print(t3)
t3.shape

In [ ]: print(t4)
t4.shape

Tensor operation and gradients

In [ ]: # Create tensors

x = torch.tensor(3.)
w = torch.tensor(4. , requires_grad = True)
b = torch.tensor(5., requires_grad = True)

x, w, b

In [ ]: # Arithemitc operations

y = w*x + b
y

In [ ]: # Compute derivates
y.backward()

In [ ]: # Display gradients
print("dy/dx: " ,x.grad)
print("dy/dw: ", w.grad)
print("dy/db: ", b.grad)

In [ ]:

Tensor functions
In [ ]: # Create a tensor with a fixed value for every elemnt

t6 = torch.full((3,2), 42)
t6

In [ ]: # Concatinate two tensors with compatible shapes

t7 = torch.cat((t3, t6))
t7

In [ ]: # change the sin of each element

t8 = torch.sin(t7)
t8

In [ ]: # change the shape of a tensor

 t9 = t8.reshape(3,2,2)
t9

Interoperability with Numpy

In [ ]: import numpy as np

x = np.array([[1,2], [3, 4.]])

x

In [ ]: y = torch.from_numpy(x)
y

In [ ]: x.dtype, y.dtype

In [ ]: z = y.numpy()
z

In [ ]:

Linear Regression from Scratch using

pytorch
In [ ]: import numpy as np
import torch

In [ ]: # Making training data

# Input ---> (temp, rainfall, humidity) ---> yield of apple and oranges crops

inputs = np.array([
[73, 67,43],
[91, 88, 64],
[87, 134, 58],
[102, 43, 37],
[69, 96, 70],
], dtype = 'float32')

In [ ]: # Target (apples, oranges)

target = np.array([
[56, 70],
[81, 101],
[119, 113],
[22, 37],
[103, 119]
 ], dtype = 'float32')

In [ ]: # Convert inputs and target to tensors

inputs = torch.from_numpy(inputs)
target = torch.from_numpy(target)

print(inputs)
print(target)

In [ ]: # Weights and biases

w = torch.randn(2, 3, requires_grad = True)
b = torch.randn(2, requires_grad = True)

print(w)
print(b)

In [ ]: # define the model

# Z = X * W + B
def model(x):
return x @ w.t() + b

In [ ]: # prediction
preds = model(inputs)
print(preds)

In [ ]: # loss funtion we will use is MSE -> Mean squared error

def MSE(y, y_hat):
diff = y - y_hat

return torch.sum(diff*diff)/diff.numel()

In [ ]: # error
loss = MSE(target, preds)
print(loss)

In [ ]: #Compute gradients
loss.backward()

In [ ]: print(w)
print(w.grad)

In [ ]: print(b)
print(b.grad)

In [ ]: #reset grad
w.grad.zero_()
b.grad.zero_()

print(w.grad)
 print(b.grad)

In [ ]: # Adjust params

preds = model(inputs)

print(preds)

loss = MSE(target, preds)

print(loss)

In [ ]: loss.backward()

print(w.grad)
print(b.grad)

In [ ]: # adjust weight & reset grad

learning_rate = 1e-5

with torch.no_grad():
w -= w.grad * 1e-5
b -= b.grad * 1e-5

w.grad.zero_()
b.grad.zero_()

In [ ]: print(w)
print(b)

In [ ]: # Calculate again

preds = model(inputs)
loss = MSE(target, preds)
print(loss)

In [ ]: # Training for muliple epochs

for i in range(400):
preds = model(inputs)
loss = MSE(target, preds)
loss.backward()
with torch.no_grad():
w -= w.grad * 1e-5
b -= b.grad * 1e-5

w.grad.zero_()
b.grad.zero_()
print(f"Epochs({i}/{100}) & Loss {loss}")

In [ ]: preds = model(inputs)
 loss = MSE(target, preds)
print(loss)

In [ ]: from math import sqrt

sqrt(loss)

In [ ]: preds

In [ ]: target

In [ ]:

Fashion MNIST Neural Net example

using Pytorch
In [ ]: import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor, Lambda, Compose
import matplotlib.pyplot as plt

In [ ]: # Download training data from open datasets

training_data = datasets.FashionMNIST(
root = 'data',
train= True,
download = True,
transform = ToTensor(),
)

# Download test data from open datasets

test_data = datasets.FashionMNIST(
root="data",
train=False,
download = True,
transform = ToTensor(),
)

In [ ]: type(training_data)

In [ ]: batch_size = 64

# Create data loaders

train_dataloader = DataLoader(training_data, batch_size = batch_size)

 test_dataloader = DataLoader(test_data, batch_size = batch_size)

for X, y in test_dataloader:
print("Shape of X [N, C, H, W] ", X.shape)
print("Shape of y: ", y.shape, y.dtype)
break

In [ ]: # Get cpu or gpu device for training

device = "cuda" if torch.cuda.is_available() else "cpu"

print(f"Using {device} device")

In [ ]: # Define the NN Model

class NeuralNetwork(nn.Module):

def __init__(self):
super(NeuralNetwork, self).__init__()

self.flatten = nn.Flatten()

# Hidden Layers with ReLU activation function

self.linear_relu_stack = nn.Sequential(
nn.Linear(28*28, 512),
nn.ReLU(),
nn.Linear(512, 512),
nn.ReLU(),
nn.Linear(512, 10) # Output layer
)

def forward(self, x):

x = self.flatten(x)
logits = self.linear_relu_stack(x)
return logits

model= NeuralNetwork().to(device)

print(model)

In [ ]: # Cross Entropy Loss ----> Because it is a multiclass classification problem

loss_fn = nn.CrossEntropyLoss()

# Optimizer ---> SGD ---> Stochastic Gradient Descent

# lr = Learning Rate
optimizer = torch.optim.SGD(model.parameters(), lr = 1e-3)

In [ ]: # Model Training

def train(dataloader, model, loss_fn, optimizer):

size = len(dataloader.dataset)
model.train()
 for batch, (X, y) in enumerate(dataloader):
X, y= X.to(device), y.to(device) # related to gpu computation

# Compute prediction error

pred = model(X)
loss = loss_fn(pred, y)

# BackPropagation
optimizer.zero_grad()
loss.backward()
optimizer.step()

if batch %100 ==0 :

loss, current = loss.item(), batch * len(X)
print(f"Loss: {loss} [{current}/{size}]")

In [ ]: def test(dataloader, model, loss_fn):

size = len(dataloader.dataset)

num_batches = len(dataloader)

model.eval()

test_loss, correct = 0, 0

with torch.no_grad():
for X, y in dataloader:
X, y = X.to(device), y.to(device)
pred = model(X)

test_loss += loss_fn(pred, y).item()

correct += (pred.argmax(1) == y).type(torch.float).sum().item()

test_loss /= num_batches # average loss per batch

correct /= size # %age of correct predictions or accuracy

print(f"Test Error: \n Accuracy: {100*correct} %, Avg loss {test_loss}\n")

In [ ]: epochs = 1
for t in range(epochs):
print(f"Epoch {t+1} \n --------------------------")
train(train_dataloader, model, loss_fn, optimizer)
test(test_dataloader, model, loss_fn)

print("Done\n")

In [ ]:

In [ ]: #save model
torch.save(model.state_dict(), "model.pth")
print("Saved model state to model.pth")
In [ ]: ## Prediction

classes = [
"T-shirt/top",

"Trouser",

"Pullover",

"Dress",

"Coat",

"Sandal",

"Shirt",

"Sneaker",

"Bag",

"Ankle boot"

model.eval()

x, y = test_data[10][0], test_data[10][1]
x = x.to(device)
# y = y.to(device)
with torch.no_grad():
pred = model(x)
predicted, actual = classes[pred[0].argmax(0)], classes[y]

print(f"Predicted: {predicted} Actual: {actual}")

In [ ]: import matplotlib.pyplot as plt

# Assuming 'x' contains the image data

plt.imshow(x.cpu().squeeze()) # Move tensor to CPU if using GPU
plt.show()

In [ ]: