Scribd
Upload
0 views

Vertopal.com Lesson 3

The document describes the implementation of various layers for a neural network, including multiplication, addition, activation functions (ReLU and Sigmoid), affine layers, and softmax with loss. It details the forward and backward propagation processes for each layer, demonstrating how to compute outputs and gradients. Additionally, it introduces a TwoLayerNet class for building a neural network that supports backpropagation.

Uploaded by

Thùy Minh
Copyright
© © All Rights Reserved
Available Formats
Download as PDF, TXT or read online on Scribd
0 views

Vertopal.com Lesson 3

The document describes the implementation of various layers for a neural network, including multiplication, addition, activation functions (ReLU and Sigmoid), affine layers, and softmax with loss. It details the forward and backward propagation processes for each layer, demonstrating how to compute outputs and gradients. Additionally, it introduces a TwoLayerNet class for building a neural network that supports backpropagation.

Uploaded by

Thùy Minh
Copyright
© © All Rights Reserved
Available Formats
Download as PDF, TXT or read online on Scribd
You are on page 1/ 6

Lession 3

Implementing a Multiplication Layer


class MulLayer:
def __init__(self):
self.a = None
self.b = None

def forward(self, a, b):


self.a = a
self.b = b
output = a * b

return output

def backward(self, douput):


da = douput * self.b # Reverse a and b
db = douput * self.a

return da, db

an_apple_price = 100
apple_num = 2
consumption_tax = 1.1

# Layer
multiplication_apple_layer = MulLayer()
multiplication_tax_layer = MulLayer()

# Forward propagation
total_apple_price = multiplication_apple_layer.forward(an_apple_price,
apple_num)
total_cost = multiplication_tax_layer.forward(total_apple_price,
consumption_tax)

print(total_cost) #220

220.00000000000003

# Backward propagation
div_price_output = 1
div_total_apple_price, div_tax =
multiplication_tax_layer.backward(div_price_output)
div_an_apple_price, div_apple_num =
multiplication_apple_layer.backward(div_total_apple_price)
print(div_an_apple_price, div_apple_num, div_tax) # 2.2 110 200
2.2 110.00000000000001 200

Implementing an Addition Layer


class AddLayer:
def __init__(self):
pass

def forward(self, a, b):


output = a + b

return output

def backward(self, doutput):


da = doutput * 1
db = doutput * 1

return da, db

an_apple_price = 100
apple_num = 2
an_orange_price = 150
orange_num = 3
consumption_tax = 1.1

# Layer
multiplication_apple_layer = MulLayer()
multiplication_orange_layer = MulLayer()
addition_apple_orange_layer = AddLayer()
multiplication_tax_layer = MulLayer()

# Forward propagation
total_apple_price = multiplication_apple_layer.forward(an_apple_price,
apple_num) # (1)
total_orange_price =
multiplication_orange_layer.forward(an_orange_price, orange_num) # (2)
total_cost_bef_tax =
addition_apple_orange_layer.forward(total_apple_price,
total_orange_price) # (3)
total_cost_aft_tax =
multiplication_tax_layer.forward(total_cost_bef_tax, consumption_tax)
# (4)

# Backward propation
div_price = 1
div_total_cost_bef_tax, div_tax =
multiplication_tax_layer.backward(div_price) # (4)
div_total_apple_price, div_total_orange_price =
addition_apple_orange_layer.backward(div_total_cost_bef_tax) # (3)
div_an_orange_price, div_orange_num =
multiplication_orange_layer.backward(div_total_orange_price) #(2)
div_an_apple_price, div_apple_num =
multiplication_apple_layer.backward(div_total_apple_price) # (1)

print(total_cost_aft_tax) # 715
print(div_apple_num, div_an_apple_price, div_an_orange_price,
div_orange_num, div_total_cost_bef_tax, div_tax) # 110 2.2 3.3 165 650

715.0000000000001
110.00000000000001 2.2 3.3000000000000003 165.0 1.1 650

Implementing the Activation Function Layer


ReLU Layer
class ReLU:
def __init__(self):
self.mask = None

def forward(self, a):


self.mask = (a <= 0)
output = a.copy()
output[self.mask] = 0 # return output the values of self.mask

return output

def backward(self, div_output):


div_output[self.mask] = 0
div_a = div_output

return div_a

import numpy as np
a = np.array([[2.0, -0.6], [-3.0, 4.0]])
print(a)
mask = (a <= 0)
print(mask)

[[ 2. -0.6]
[-3. 4. ]]
[[False True]
[ True False]]
[0. 0.]

Sigmoid Layer
class SigMoid:
def __init__(self):
self.output = None
def forward(self, a):
output = 1 / (1 + np.exp(-a))
self.output = output

return output

def backward(self, div_output):


div_a = div_output * (1.0 - self.output) * self.output

return div_a

Implementing the Affine and Softmax Layers


Affine Layer
X_input = np.random.rand(2) # Input values
W_weight = np.random.rand(2,3) # Weights
B_bias = np.random.rand(3) # Biases

X_input.shape # (2,)
W_weight.shape # (2,3)
B_bias.shape # (3,)

Y_output = np.dot(X_input, W_weight) + B_bias


print(Y_output)

[0.48021792 1.08183057 1.21569381]

Batch-Based Affine Layer


X_dot_W = np.array([[1, 1, 1], [11, 11, 11]])
B = np.array([2, 4, 6])

X_dot_W

array([[ 1, 1, 1],
[11, 11, 11]])

X_dot_W + B

array([[ 3, 5, 7],
[13, 15, 17]])

class Affine:
def __init__(self, W, b):
self.W = W
self.b = b
self.x = None
self.div_W = None
self.div_b = None
def forward(self, x):
self.x = x
output = np.dot(x, self.W) + self.b

return output

def backward(self, div_out):


div_x = np.dot(div_out, self.W.T)
self.W = np.dot(self.x.t, div_out)
self.b = np.sum(div_out, axis = 0)

return div_x

Softmax-with-Loss Layer
class SoftMaxWithLoss:
def __init__(self):
self.loss = None # Loss
self.y = None # Output of softmax
self.t = None # Label data (one-hot vector)

def forward(self, x, t):


self.t = t
self.y = softmax(x)
self.loss = cross_entropy_error(self.y, self.t)

return self.loss

def backward(self, div_out=1):


batch_size = self.t.shape[0]
div_x = (self.y - self.t) / batch_size

return div_x

Implementing Backpropagation
Implementing a Neural Network That Supports Backpropagation
import sys, os
sys.path.append(os.pardir)
import numpy as np
from common.layers import *
from common.gradient import numerical_gradient
from collections import OrderedDict

class TwoLayerNet:
def __init__(self, input_size, hidden_size, output_size,
weight_init_std=0.01):
# Initialize weights
self.params = {}
self.params['W1'] = weight_init_std * \
np.random.randn(input_size, hidden_size)
self.params['b1'] = np.zeros(hidden_size)
self.params['W2'] = weight_init_std * \
np.random.randn(hidden_size, output_size)
self.params['b2'] = np.zeros(output_size)

# Create layers

You might also like