Experiments with Normalization/Optimization

CGML Assignment 4
Camille Chow
For classifying the CIFAR100 dataset, I was able to adapt my convolutional neural net
from Assignment 3 with very few alterations, the main difference being the use of batch
normalization and the number of filters at each layer. Other hyperparameters were adjusted and
tested, however, none of these experiments produced a significantly improved top-5 accuracy.
The number of epochs was also increased to 10 and the activation function for the convolutional
layers was changed to elu rather than relu.
For the CIFAR10 dataset, I initially attempted to use a similar model as the CIFAR100,
but couldn't produce an accuracy better than 70%. Initial attempts at increasing the depth of the
model resulted in decreased accuracy. The current architecture is modeled after the following
tutorial: https://appliedmachinelearning.blog/2018/03/24/achieving-90-accuracy-in-object-
recognition-task-on-cifar-10-dataset-with-keras-convolutional-neural-networks/, which uses six
convolutional layers with increasing filter size. I attempted to implement data augmentation,
however, this too resulted in lower accuracy rates (likely due to overfitting) and extra
computation time, so I chose to omit it. I added an additional dense layer and dropout layer to
improve accuracy. After tuning and testing with various optimizers, pooling and kernel sizes,
filter numbers, dropout rates, activation functions, batch norm parameters, and learning rate
schedules, I arrived to similar hyperparameters from the tutorial, as these were already close to
optimal. From that point on, the only gains made in accuracy were achieved by increasing the
number of epochs. I was able to achieve up to 80% accuracy with 10 epochs, 85% accuracy with
30 epochs, and 87% with 100 epochs. With more time and computational capability, an accuracy
closer to state of the art can be achieved.
cifar10.py Wed Oct 03 14:41:25 2018 1
#Camille Chow
#ECE 471 Assignment 4
#Classifying CIFAR10 data
#Citation: Learning Multiple Layers of Features from Tiny Images, Alex Krizhevsky, 2009.
import numpy as np
import tensorflow as tf

#dimensional constants
num_classes = 10
image_h = 32
image_w = 32
channels = 3

#get data
(x_train, y_train),(x_test, y_test) = tf.keras.datasets.cifar10.load_data()

#shape test data

x_test = x_test.reshape(x_test.shape[0], image_h, image_w, channels)
x_test = x_test.astype(’float32’)
x_test /= 255.0
# shape_data(x_test)
y_test = tf.keras.utils.to_categorical(y_test, num_classes)

#shape training data

x_train = x_train.reshape(x_train.shape[0], image_h, image_w, channels)
x_train = x_train.astype(’float32’)
x_train /= 255.0

# shape_data(x_train)
y_train = tf.keras.utils.to_categorical(y_train, num_classes)

#tunable hyperparams
batch_size = 100
epochs = 100
lam = .001

def lr_schedule(epoch):
lrate = 0.001
if epoch > 65:
lrate = 0.0005
elif epoch > 85:
lrate = 0.0001
return lrate

def add_conv_layer(model, num_filters):

model.add(tf.keras.layers.Conv2D(num_filters, kernel_size=3,
strides=(1, 1), activation=’elu’, padding=’same’))

def add_pooling_layer(model):
model.add(tf.keras.layers.MaxPooling2D(pool_size=3, strides=2))

def add_bn_layer(model):
model.add(tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=0.001))

#build cnn
model = tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(32, kernel_size=3,
strides=(1, 1), activation=’elu’, padding=’same’, input_shape=(image_h, i
mage_w, channels)))
add_bn_layer(model)
add_conv_layer(model, 32)
add_bn_layer(model)
add_pooling_layer(model)
model.add(tf.keras.layers.Dropout(.2))
add_conv_layer(model, 64)
add_bn_layer(model)
add_conv_layer(model, 64)
add_bn_layer(model)
add_pooling_layer(model)
model.add(tf.keras.layers.Dropout(.3))
cifar10.py Wed Oct 03 14:41:25 2018 2
add_conv_layer(model, 128)
add_bn_layer(model)
add_conv_layer(model, 128)
add_bn_layer(model)
add_pooling_layer(model)
model.add(tf.keras.layers.Dropout(.4))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation=’elu’,kernel_regularizer=tf.keras.regular
izers.l2(lam)))
model.add(tf.keras.layers.Dropout(.5))
model.add(tf.keras.layers.Dense(num_classes, activation=’softmax’))

#train model
model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer=tf.keras.optimizer
s.Adam(), metrics=[’accuracy’])
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_s
plit=.1, callbacks=[tf.keras.callbacks.LearningRateScheduler(lr_schedule)])

#test model
score = model.evaluate(x_test, y_test, verbose=0)
print(’Test loss:’, score[0])
print(’Test accuracy:’, score[1])
cifar100.py Tue Oct 02 15:42:10 2018 1
#Camille Chow
#ECE 471 Assignment 4
#Classifying CIFAR100 data
#Citation: Learning Multiple Layers of Features from Tiny Images, Alex Krizhevsky, 2009.
import numpy as np
import tensorflow as tf

#dimensional constants
num_classes = 100
image_h = 32
image_w = 32
channels = 3

#get data
(x_train, y_train),(x_test, y_test) = tf.keras.datasets.cifar100.load_data()

#shape test data

x_test = x_test.reshape(x_test.shape[0], image_h, image_w, channels)
x_test = x_test.astype(’float32’)
x_test /= 255.0
y_test = tf.keras.utils.to_categorical(y_test, num_classes)

#shape training data

x_train = x_train.reshape(x_train.shape[0], image_h, image_w, channels)
x_train = x_train.astype(’float32’)
x_train /= 255.0
y_train = tf.keras.utils.to_categorical(y_train, num_classes)

#tunable hyperparams
batch_size = 100
epochs = 10
dropout = .5
dense_units = 1000
lam = .001

def add_conv_layer(model, num_filters):

model.add(tf.keras.layers.Conv2D(num_filters, kernel_size=3,
strides=(1, 1), activation=’elu’, padding=’same’))

def add_pooling_layer(model):
model.add(tf.keras.layers.MaxPooling2D(pool_size=3, strides=3))

def add_bn_layer(model):
model.add(tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=0.001))

#build cnn
model = tf.keras.Sequential()
add_conv_layer(model, 64)
add_bn_layer(model)
add_pooling_layer(model)
add_conv_layer(model, 32)
add_bn_layer(model)
add_pooling_layer(model)
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(dense_units, activation=’relu’, kernel_regularizer=tf.ker
as.regularizers.l2(lam)))
model.add(tf.keras.layers.Dropout(dropout))
model.add(tf.keras.layers.Dense(num_classes, activation=’softmax’))

#train model
model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer=tf.keras.optimizer
s.Adam(), metrics=[’accuracy’, ’top_k_categorical_accuracy’])
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_s
plit=.1)

#test model
score = model.evaluate(x_test, y_test, verbose=0)
print(’Test loss:’, score[0])
print(’Test accuracy:’, score[1])
print(’Test top-5 accuracy:’, score[2])