In [1]: from numpy import arange

from numpy import meshgrid

from matplotlib import pyplot

# objective function
def objective(x, y):
return x**2.0 + y**2.0

# define range for input

r_min, r_max = -1.0, 1.0
# sample input range uniformly at 0.1 increments
xaxis = arange(r_min, r_max, 0.1)
yaxis = arange(r_min, r_max, 0.1)
# create a mesh from the axis
x, y = meshgrid(xaxis, yaxis)
# compute targets
results = objective(x, y)
# create a surface plot with the jet color scheme
fig, ax = pyplot.subplots(subplot_kw={"projection": "3d"})
ax.plot_surface(x, y, results, cmap='jet')
# show the plot
pyplot.show()

In [ ]: 3D plot of the test function

In [2]: from numpy import asarray

from numpy import arange
from numpy import meshgrid
from matplotlib import pyplot

# objective function
def objective(x, y):
return x**2.0 + y**2.0

# define range for input

bounds = asarray([[-1.0, 1.0], [-1.0, 1.0]])
# sample input range uniformly at 0.1 increments
xaxis = arange(bounds[0,0], bounds[0,1], 0.1)
yaxis = arange(bounds[1,0], bounds[1,1], 0.1)
# create a mesh from the axis
x, y = meshgrid(xaxis, yaxis)
# compute targets
results = objective(x, y)
# create a filled contour plot with 50 levels and jet color scheme
pyplot.contourf(x, y, results, levels=50, cmap='jet')
# show the plot
pyplot.show()

In [ ]: Two-Dimensional Contour Plot of the Test Objective Function

In [3]: from math import sqrt

from numpy import asarray
from numpy.random import rand
from numpy.random import seed

# objective function
def objective(x, y):
return x**2.0 + y**2.0

# derivative of objective function

def derivative(x, y):
return asarray([x * 2.0, y * 2.0])

# gradient descent algorithm with adadelta

def adadelta(objective, derivative, bounds, n_iter, rho, ep=1e-3):
# generate an initial point
solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
# list of the average square gradients for each variable
sq_grad_avg = [0.0 for _ in range(bounds.shape[0])]
# list of the average parameter updates
sq_para_avg = [0.0 for _ in range(bounds.shape[0])]
# run the gradient descent
for it in range(n_iter):
# calculate gradient
gradient = derivative(solution[0], solution[1])
# update the average of the squared partial derivatives
for i in range(gradient.shape[0]):
# calculate the squared gradient
sg = gradient[i]**2.0
# update the moving average of the squared gradient
sq_grad_avg[i] = (sq_grad_avg[i] * rho) + (sg * (1.0-rho))
# build a solution one variable at a time
new_solution = list()
for i in range(solution.shape[0]):
# calculate the step size for this variable
alpha = (ep + sqrt(sq_para_avg[i])) / (ep + sqrt(sq_grad_avg[i]))
# calculate the change
change = alpha * gradient[i]
# update the moving average of squared parameter changes
sq_para_avg[i] = (sq_para_avg[i] * rho) + (change**2.0 * (1.0-rho))
# calculate the new position in this variable
value = solution[i] - change
# store this variable
new_solution.append(value)
# evaluate candidate point
solution = asarray(new_solution)
solution_eval = objective(solution[0], solution[1])
# report progress
print('>%d f(%s) = %.5f' % (it, solution, solution_eval))
return [solution, solution_eval]

# seed the pseudo random number generator

seed(1)
# define range for input
bounds = asarray([[-1.0, 1.0], [-1.0, 1.0]])
# define the total iterations
n_iter = 120
# momentum for adadelta
rho = 0.99
# perform the gradient descent search with adadelta
best, score = adadelta(objective, derivative, bounds, n_iter, rho)
print('Done!')
print('f(%s) = %f' % (best, score))

>0 f([-0.15624846 0.43076118]) = 0.20997

>1 f([-0.14299468 0.41693563]) = 0.19428
>2 f([-0.12914643 0.40184605]) = 0.17816
>3 f([-0.11566994 0.38630269]) = 0.16261
>4 f([-0.10299941 0.37069192]) = 0.14802
>5 f([-0.09132837 0.35523116]) = 0.13453
>6 f([-0.08072226 0.34005317]) = 0.12215
>7 f([-0.07117441 0.32524231]) = 0.11085
>8 f([-0.06263776 0.31085275]) = 0.10055
>9 f([-0.05504377 0.29691871]) = 0.09119
>10 f([-0.04831412 0.28346065]) = 0.08268
>11 f([-0.04236781 0.27048927]) = 0.07496
>12 f([-0.03712545 0.25800822]) = 0.06795
>13 f([-0.03251183 0.24601593]) = 0.06158
>14 f([-0.02845711 0.234507 ]) = 0.05580
>15 f([-0.02489748 0.22347322]) = 0.05056
>16 f([-0.02177521 0.21290424]) = 0.04580
>17 f([-0.01903851 0.20278822]) = 0.04149
>18 f([-0.01664114 0.19311222]) = 0.03757
>19 f([-0.01454205 0.18386255]) = 0.03402
>20 f([-0.01270485 0.17502504]) = 0.03080
>21 f([-0.01109744 0.16658525]) = 0.02787
>22 f([-0.00969149 0.15852865]) = 0.02523
>23 f([-0.00846208 0.15084071]) = 0.02282
>24 f([-0.0073873 0.14350705]) = 0.02065
>25 f([-0.00644791 0.13651347]) = 0.01868
>26 f([-0.00562703 0.12984606]) = 0.01689
>27 f([-0.00490983 0.12349122]) = 0.01527
>28 f([-0.00428333 0.11743568]) = 0.01381
>29 f([-0.00373617 0.11166659]) = 0.01248
>30 f([-0.00325838 0.10617146]) = 0.01128
>31 f([-0.00284123 0.10093823]) = 0.01020
>32 f([-0.00247708 0.09595523]) = 0.00921
>33 f([-0.00215926 0.09121123]) = 0.00832
>34 f([-0.00188191 0.08669541]) = 0.00752
>35 f([-0.00163992 0.08239738]) = 0.00679
>36 f([-0.00142882 0.07830713]) = 0.00613
>37 f([-0.00124469 0.07441507]) = 0.00554
>38 f([-0.00108411 0.070712 ]) = 0.00500
>39 f([-0.00094409 0.0671891 ]) = 0.00452
>40 f([-0.00082202 0.06383794]) = 0.00408
>41 f([-0.00071561 0.06065043]) = 0.00368
>42 f([-0.00062288 0.05761885]) = 0.00332
>43 f([-0.00054207 0.05473579]) = 0.00300
>44 f([-0.00047167 0.05199421]) = 0.00270
>45 f([-0.00041034 0.04938735]) = 0.00244
>46 f([-0.00035693 0.04690876]) = 0.00220
>47 f([-0.00031041 0.04455231]) = 0.00199
>48 f([-0.00026991 0.04231211]) = 0.00179
>49 f([-0.00023466 0.04018258]) = 0.00161
>50 f([-0.00020397 0.03815835]) = 0.00146
>51 f([-0.00017727 0.03623436]) = 0.00131
>52 f([-0.00015403 0.03440572]) = 0.00118
>53 f([-0.00013382 0.03266783]) = 0.00107
>54 f([-0.00011624 0.03101626]) = 0.00096
>55 f([-0.00010095 0.02944681]) = 0.00087
>56 f([-8.76564211e-05 2.79554771e-02]) = 0.00078
>57 f([-7.60997823e-05 2.65384440e-02]) = 0.00070
>58 f([-6.60548950e-05 2.51920772e-02]) = 0.00063
>59 f([-5.73255352e-05 2.39129162e-02]) = 0.00057
>60 f([-4.97407527e-05 2.26976655e-02]) = 0.00052
>61 f([-4.31516407e-05 2.15431875e-02]) = 0.00046
>62 f([-3.74285151e-05 2.04464954e-02]) = 0.00042
>63 f([-3.24584521e-05 1.94047458e-02]) = 0.00038
>64 f([-2.81431386e-05 1.84152321e-02]) = 0.00034
>65 f([-2.43969967e-05 1.74753787e-02]) = 0.00031
>66 f([-2.11455462e-05 1.65827342e-02]) = 0.00027
>67 f([-1.83239771e-05 1.57349659e-02]) = 0.00025
>68 f([-1.58759036e-05 1.49298543e-02]) = 0.00022
>69 f([-1.37522776e-05 1.41652876e-02]) = 0.00020
>70 f([-1.19104414e-05 1.34392565e-02]) = 0.00018
>71 f([-1.03133014e-05 1.27498497e-02]) = 0.00016
>72 f([-8.92860836e-06 1.20952491e-02]) = 0.00015
>73 f([-7.72832934e-06 1.14737250e-02]) = 0.00013
>74 f([-6.68810108e-06 1.08836325e-02]) = 0.00012
>75 f([-5.78675328e-06 1.03234070e-02]) = 0.00011
>76 f([-5.00589352e-06 9.79156055e-03]) = 0.00010
>77 f([-4.32954559e-06 9.28667801e-03]) = 0.00009
>78 f([-3.74383469e-06 8.80741378e-03]) = 0.00008
>79 f([-3.23671313e-06 8.35248827e-03]) = 0.00007
>80 f([-2.79772173e-06 7.92068480e-03]) = 0.00006
>81 f([-2.41778201e-06 7.51084654e-03]) = 0.00006
>82 f([-2.08901542e-06 7.12187364e-03]) = 0.00005
>83 f([-1.80458599e-06 6.75272043e-03]) = 0.00005
>84 f([-1.55856350e-06 6.40239286e-03]) = 0.00004
>85 f([-1.34580444e-06 6.06994592e-03]) = 0.00004
>86 f([-1.16184848e-06 5.75448133e-03]) = 0.00003
>87 f([-1.00282848e-06 5.45514522e-03]) = 0.00003
>88 f([-8.65392235e-07 5.17112600e-03]) = 0.00003
>89 f([-7.46634481e-07 4.90165228e-03]) = 0.00002
>90 f([-6.44037778e-07 4.64599092e-03]) = 0.00002
>91 f([-5.55421172e-07 4.40344515e-03]) = 0.00002
>92 f([-4.78895592e-07 4.17335284e-03]) = 0.00002
>93 f([-4.12825106e-07 3.95508473e-03]) = 0.00002
>94 f([-3.55793284e-07 3.74804289e-03]) = 0.00001
>95 f([-3.06573985e-07 3.55165915e-03]) = 0.00001
>96 f([-2.64106011e-07 3.36539365e-03]) = 0.00001
>97 f([-2.27471101e-07 3.18873346e-03]) = 0.00001
>98 f([-1.95874837e-07 3.02119123e-03]) = 0.00001
>99 f([-1.68630085e-07 2.86230396e-03]) = 0.00001
>100 f([-1.45142626e-07 2.71163181e-03]) = 0.00001
>101 f([-1.24898699e-07 2.56875692e-03]) = 0.00001
>102 f([-1.07454197e-07 2.43328237e-03]) = 0.00001
>103 f([-9.24253035e-08 2.30483111e-03]) = 0.00001
>104 f([-7.94803792e-08 2.18304501e-03]) = 0.00000
>105 f([-6.83329263e-08 2.06758392e-03]) = 0.00000
>106 f([-5.87354975e-08 1.95812477e-03]) = 0.00000
>107 f([-5.04744185e-08 1.85436071e-03]) = 0.00000
>108 f([-4.33652179e-08 1.75600036e-03]) = 0.00000
>109 f([-3.72486699e-08 1.66276699e-03]) = 0.00000
>110 f([-3.19873691e-08 1.57439783e-03]) = 0.00000
>111 f([-2.74627662e-08 1.49064334e-03]) = 0.00000
>112 f([-2.3572602e-08 1.4112666e-03]) = 0.00000
>113 f([-2.02286891e-08 1.33604264e-03]) = 0.00000
>114 f([-1.73549914e-08 1.26475787e-03]) = 0.00000
>115 f([-1.48859650e-08 1.19720951e-03]) = 0.00000
>116 f([-1.27651224e-08 1.13320504e-03]) = 0.00000
>117 f([-1.09437923e-08 1.07256172e-03]) = 0.00000
>118 f([-9.38004754e-09 1.01510604e-03]) = 0.00000
>119 f([-8.03777865e-09 9.60673346e-04]) = 0.00000
Done!
f([-8.03777865e-09 9.60673346e-04]) = 0.000001

In [ ]: Gradient descent optimization with Adadelta for a two-dimensional

test function

In [4]: from math import sqrt

from numpy import asarray
from numpy import arange
from numpy.random import rand
from numpy.random import seed
from numpy import meshgrid
from matplotlib import pyplot
from mpl_toolkits.mplot3d import Axes3D

# objective function
def objective(x, y):
return x**2.0 + y**2.0

# derivative of objective function

def derivative(x, y):
return asarray([x * 2.0, y * 2.0])

# gradient descent algorithm with adadelta

def adadelta(objective, derivative, bounds, n_iter, rho, ep=1e-3):
# track all solutions
solutions = list()
# generate an initial point
solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
# list of the average square gradients for each variable
sq_grad_avg = [0.0 for _ in range(bounds.shape[0])]
# list of the average parameter updates
sq_para_avg = [0.0 for _ in range(bounds.shape[0])]
# run the gradient descent
for it in range(n_iter):
# calculate gradient
gradient = derivative(solution[0], solution[1])
# update the average of the squared partial derivatives
for i in range(gradient.shape[0]):
# calculate the squared gradient
sg = gradient[i]**2.0
# update the moving average of the squared gradient
sq_grad_avg[i] = (sq_grad_avg[i] * rho) + (sg * (1.0-rho))
# build solution
new_solution = list()
for i in range(solution.shape[0]):
# calculate the step size for this variable
alpha = (ep + sqrt(sq_para_avg[i])) / (ep + sqrt(sq_grad_avg[i]))
# calculate the change
change = alpha * gradient[i]
# update the moving average of squared parameter changes
sq_para_avg[i] = (sq_para_avg[i] * rho) + (change**2.0 * (1.0-rho))
# calculate the new position in this variable
value = solution[i] - change
# store this variable
new_solution.append(value)
# store the new solution
solution = asarray(new_solution)
solutions.append(solution)
# evaluate candidate point
solution_eval = objective(solution[0], solution[1])
# report progress
print('>%d f(%s) = %.5f' % (it, solution, solution_eval))
return solutions

# seed the pseudo random number generator

seed(1)
# define range for input
bounds = asarray([[-1.0, 1.0], [-1.0, 1.0]])
# define the total iterations
n_iter = 120
# rho for adadelta
rho = 0.99
# perform the gradient descent search with adadelta
solutions = adadelta(objective, derivative, bounds, n_iter, rho)
# sample input range uniformly at 0.1 increments
xaxis = arange(bounds[0,0], bounds[0,1], 0.1)
yaxis = arange(bounds[1,0], bounds[1,1], 0.1)
# create a mesh from the axis
x, y = meshgrid(xaxis, yaxis)
# compute targets
results = objective(x, y)
# create a filled contour plot with 50 levels and jet color scheme
pyplot.contourf(x, y, results, levels=50, cmap='jet')
# plot the sample as black circles
solutions = asarray(solutions)
pyplot.plot(solutions[:, 0], solutions[:, 1], '.-', color='w')
# show the plot
pyplot.show()

>0 f([-0.15624846 0.43076118]) = 0.20997

>1 f([-0.14299468 0.41693563]) = 0.19428
>2 f([-0.12914643 0.40184605]) = 0.17816
>3 f([-0.11566994 0.38630269]) = 0.16261
>4 f([-0.10299941 0.37069192]) = 0.14802
>5 f([-0.09132837 0.35523116]) = 0.13453
>6 f([-0.08072226 0.34005317]) = 0.12215
>7 f([-0.07117441 0.32524231]) = 0.11085
>8 f([-0.06263776 0.31085275]) = 0.10055
>9 f([-0.05504377 0.29691871]) = 0.09119
>10 f([-0.04831412 0.28346065]) = 0.08268
>11 f([-0.04236781 0.27048927]) = 0.07496
>12 f([-0.03712545 0.25800822]) = 0.06795
>13 f([-0.03251183 0.24601593]) = 0.06158
>14 f([-0.02845711 0.234507 ]) = 0.05580
>15 f([-0.02489748 0.22347322]) = 0.05056
>16 f([-0.02177521 0.21290424]) = 0.04580
>17 f([-0.01903851 0.20278822]) = 0.04149
>18 f([-0.01664114 0.19311222]) = 0.03757
>19 f([-0.01454205 0.18386255]) = 0.03402
>20 f([-0.01270485 0.17502504]) = 0.03080
>21 f([-0.01109744 0.16658525]) = 0.02787
>22 f([-0.00969149 0.15852865]) = 0.02523
>23 f([-0.00846208 0.15084071]) = 0.02282
>24 f([-0.0073873 0.14350705]) = 0.02065
>25 f([-0.00644791 0.13651347]) = 0.01868
>26 f([-0.00562703 0.12984606]) = 0.01689
>27 f([-0.00490983 0.12349122]) = 0.01527
>28 f([-0.00428333 0.11743568]) = 0.01381
>29 f([-0.00373617 0.11166659]) = 0.01248
>30 f([-0.00325838 0.10617146]) = 0.01128
>31 f([-0.00284123 0.10093823]) = 0.01020
>32 f([-0.00247708 0.09595523]) = 0.00921
>33 f([-0.00215926 0.09121123]) = 0.00832
>34 f([-0.00188191 0.08669541]) = 0.00752
>35 f([-0.00163992 0.08239738]) = 0.00679
>36 f([-0.00142882 0.07830713]) = 0.00613
>37 f([-0.00124469 0.07441507]) = 0.00554
>38 f([-0.00108411 0.070712 ]) = 0.00500
>39 f([-0.00094409 0.0671891 ]) = 0.00452
>40 f([-0.00082202 0.06383794]) = 0.00408
>41 f([-0.00071561 0.06065043]) = 0.00368
>42 f([-0.00062288 0.05761885]) = 0.00332
>43 f([-0.00054207 0.05473579]) = 0.00300
>44 f([-0.00047167 0.05199421]) = 0.00270
>45 f([-0.00041034 0.04938735]) = 0.00244
>46 f([-0.00035693 0.04690876]) = 0.00220
>47 f([-0.00031041 0.04455231]) = 0.00199
>48 f([-0.00026991 0.04231211]) = 0.00179
>49 f([-0.00023466 0.04018258]) = 0.00161
>50 f([-0.00020397 0.03815835]) = 0.00146
>51 f([-0.00017727 0.03623436]) = 0.00131
>52 f([-0.00015403 0.03440572]) = 0.00118
>53 f([-0.00013382 0.03266783]) = 0.00107
>54 f([-0.00011624 0.03101626]) = 0.00096
>55 f([-0.00010095 0.02944681]) = 0.00087
>56 f([-8.76564211e-05 2.79554771e-02]) = 0.00078
>57 f([-7.60997823e-05 2.65384440e-02]) = 0.00070
>58 f([-6.60548950e-05 2.51920772e-02]) = 0.00063
>59 f([-5.73255352e-05 2.39129162e-02]) = 0.00057
>60 f([-4.97407527e-05 2.26976655e-02]) = 0.00052
>61 f([-4.31516407e-05 2.15431875e-02]) = 0.00046
>62 f([-3.74285151e-05 2.04464954e-02]) = 0.00042
>63 f([-3.24584521e-05 1.94047458e-02]) = 0.00038
>64 f([-2.81431386e-05 1.84152321e-02]) = 0.00034
>65 f([-2.43969967e-05 1.74753787e-02]) = 0.00031
>66 f([-2.11455462e-05 1.65827342e-02]) = 0.00027
>67 f([-1.83239771e-05 1.57349659e-02]) = 0.00025
>68 f([-1.58759036e-05 1.49298543e-02]) = 0.00022
>69 f([-1.37522776e-05 1.41652876e-02]) = 0.00020
>70 f([-1.19104414e-05 1.34392565e-02]) = 0.00018
>71 f([-1.03133014e-05 1.27498497e-02]) = 0.00016
>72 f([-8.92860836e-06 1.20952491e-02]) = 0.00015
>73 f([-7.72832934e-06 1.14737250e-02]) = 0.00013
>74 f([-6.68810108e-06 1.08836325e-02]) = 0.00012
>75 f([-5.78675328e-06 1.03234070e-02]) = 0.00011
>76 f([-5.00589352e-06 9.79156055e-03]) = 0.00010
>77 f([-4.32954559e-06 9.28667801e-03]) = 0.00009
>78 f([-3.74383469e-06 8.80741378e-03]) = 0.00008
>79 f([-3.23671313e-06 8.35248827e-03]) = 0.00007
>80 f([-2.79772173e-06 7.92068480e-03]) = 0.00006
>81 f([-2.41778201e-06 7.51084654e-03]) = 0.00006
>82 f([-2.08901542e-06 7.12187364e-03]) = 0.00005
>83 f([-1.80458599e-06 6.75272043e-03]) = 0.00005
>84 f([-1.55856350e-06 6.40239286e-03]) = 0.00004
>85 f([-1.34580444e-06 6.06994592e-03]) = 0.00004
>86 f([-1.16184848e-06 5.75448133e-03]) = 0.00003
>87 f([-1.00282848e-06 5.45514522e-03]) = 0.00003
>88 f([-8.65392235e-07 5.17112600e-03]) = 0.00003
>89 f([-7.46634481e-07 4.90165228e-03]) = 0.00002
>90 f([-6.44037778e-07 4.64599092e-03]) = 0.00002
>91 f([-5.55421172e-07 4.40344515e-03]) = 0.00002
>92 f([-4.78895592e-07 4.17335284e-03]) = 0.00002
>93 f([-4.12825106e-07 3.95508473e-03]) = 0.00002
>94 f([-3.55793284e-07 3.74804289e-03]) = 0.00001
>95 f([-3.06573985e-07 3.55165915e-03]) = 0.00001
>96 f([-2.64106011e-07 3.36539365e-03]) = 0.00001
>97 f([-2.27471101e-07 3.18873346e-03]) = 0.00001
>98 f([-1.95874837e-07 3.02119123e-03]) = 0.00001
>99 f([-1.68630085e-07 2.86230396e-03]) = 0.00001
>100 f([-1.45142626e-07 2.71163181e-03]) = 0.00001
>101 f([-1.24898699e-07 2.56875692e-03]) = 0.00001
>102 f([-1.07454197e-07 2.43328237e-03]) = 0.00001
>103 f([-9.24253035e-08 2.30483111e-03]) = 0.00001
>104 f([-7.94803792e-08 2.18304501e-03]) = 0.00000
>105 f([-6.83329263e-08 2.06758392e-03]) = 0.00000
>106 f([-5.87354975e-08 1.95812477e-03]) = 0.00000
>107 f([-5.04744185e-08 1.85436071e-03]) = 0.00000
>108 f([-4.33652179e-08 1.75600036e-03]) = 0.00000
>109 f([-3.72486699e-08 1.66276699e-03]) = 0.00000
>110 f([-3.19873691e-08 1.57439783e-03]) = 0.00000
>111 f([-2.74627662e-08 1.49064334e-03]) = 0.00000
>112 f([-2.3572602e-08 1.4112666e-03]) = 0.00000
>113 f([-2.02286891e-08 1.33604264e-03]) = 0.00000
>114 f([-1.73549914e-08 1.26475787e-03]) = 0.00000
>115 f([-1.48859650e-08 1.19720951e-03]) = 0.00000
>116 f([-1.27651224e-08 1.13320504e-03]) = 0.00000
>117 f([-1.09437923e-08 1.07256172e-03]) = 0.00000
>118 f([-9.38004754e-09 1.01510604e-03]) = 0.00000
>119 f([-8.03777865e-09 9.60673346e-04]) = 0.00000
In [ ]: Example of plotting the Adadelta search on a contour plot of the
test function

In [ ]: