Question 1

import numpy as np

# Define the function

def f(x):
x, y = x
return 5 * x**2 + y**2 + 4 * x * y - 6 * x - 4 * y + 15

# Define the gradient of the function

def gradient(x):
x, y = x
df_dx = 10 * x + 4 * y - 6
df_dy = 2 * y + 4 * x - 4
return np.array([df_dx, df_dy])

# Define the Hessian of the function

def hessian(x):
return np.array([[10, 4], [4, 2]])

# Closed-form exact line search for alpha

def exact_line_search(x, direction):
grad = gradient(x)
H = hessian(x)
# Calculate alpha using the closed-form solution
alpha = -np.dot(direction, grad) / np.dot(direction, H @
direction)
return alpha

# Define a function to print the results table

def print_table(history):
print("Iteration Table:")
print(f"{'k':<5}{'xk':<40}{'f(xk)':<15}{'∇f(xk)':<40}{'||
∇f(xk)||':<15}{'αk':<10}")
for row in history[:20]: # Show only the first 20 iterations
k, xk, fxk, gradk, grad_normk, alpha_k = row
print(f"{k:<5}{xk!s:<40}{fxk:<15.4f}{gradk!s:40}
{grad_normk:<15.4f}{alpha_k:<10.4f}")

# Define the Steepest Descent method

def steepest_descent(x0, tol=1e-6, max_iter=100):
x = np.array(x0)
history = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the descent direction

direction = -grad
 # Calculate step size using the closed-form exact line search
alpha = exact_line_search(x, direction)

# Store iteration history

history.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Steepest Descent Method:")
print("Local Minima:", x)
print("Function Value at Minima:", f(x))
print("Norm of Gradient at Minima:", grad_norm)
print_table(history)

# Initial point and calling the methods

x0 = [0, 10]

# Run Steepest Descent

steepest_descent(x0)

Steepest Descent Method:

Local Minima: [-0.99999999 4.00000003]
Function Value at Minima: 10.0
Norm of Gradient at Minima: 1.4500289199187473e-07
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [ 0 10] 75.0000 [34 16]
37.5766 0.0860
1 [-2.92303945 8.62445202] 14.3039 [-
0.73258646 1.55674623] 1.7205 2.7154
2 [-0.93378545 4.39728727] 10.2850
[2.25129454 1.05943273] 2.4881 0.0860
3 [-1.12733318 4.30620599] 10.0189 [-
0.04850788 0.10307924] 0.1139 2.7154
4 [-0.99561563 4.0263062 ] 10.0012
[0.14906844 0.07014986] 0.1647 0.0860
5 [-1.00843131 4.02027529] 10.0001 [-
0.00321193 0.00682535] 0.0075 2.7154
6 [-0.99970969 4.00174185] 10.0000 [0.0098705
0.00464494] 0.0109 0.0860
7 [-1.00055828 4.00134252] 10.0000 [-
0.00021268 0.00045194] 0.0005 2.7154
8 [-0.99998078 4.00011534] 10.0000
[0.00065357 0.00030756] 0.0007 0.0860
9 [-1.00003697 4.00008889] 10.0000 [-
1.40822643e-05 2.99248117e-05] 0.0000 2.7154
10 [-0.99999873 4.00000764] 10.0000
[4.32758815e-05 2.03651207e-05] 0.0000 0.0860
11 [-1.00000245 4.00000589] 10.0000 [-
9.3245073e-07 1.9814578e-06] 0.0000 2.7154
12 [-0.99999992 4.00000051] 10.0000
[2.86549283e-06 1.34846721e-06] 0.0000 0.0860
13 [-1.00000016 4.00000039] 10.0000 [-
6.17418028e-08 1.31201328e-07] 0.0000 2.7154

# Define the Newton's method

def newton_method(x_init, tol=1e-6, max_iter=100):
x = np.array(x_init)
history1 = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the Newton direction (solve H * direction = -grad)

H = hessian(x)
try:
direction = np.linalg.solve(H, -grad)
except np.linalg.LinAlgError:
print("Singular Hessian matrix encountered.")
break # Exit if Hessian is singular

# Calculate step size using exact line search

alpha = 1

# Store iteration history

history1.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Newton's Method Results:")
print("Local Minima:", x)
 print("Function Value at Minima:", f(x))
print("Norm of Gradient at Minima:", grad_norm)
print_table(history1)

# Initial point and calling the methods

x0 = [0, 10]

# Run Newton's Method

newton_method(x0)

Newton's Method Results:

Local Minima: [-1. 4.]
Function Value at Minima: 10.0
Norm of Gradient at Minima: 0.0
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [ 0 10] 75.0000 [34 16]
37.5766 1.0000
1 [-1. 4.] 10.0000 [0. 0.]
0.0000 1.0000

Question 2
# Define the function
def f(x):
x, y = x
return (1 - x)**2 + (100*(y - (x*x))**2)

# Define the gradient of the function

def gradient(x):
x, y = x
df_dx = 2*x - 2 - 400*x*(y - (x*x))
df_dy = 200*(y - (x*x))
return np.array([df_dx, df_dy])

# Define the Hessian of the function

def hessian(x):
x,y = x
return np.array([[(2-400*(y - 3*x*x)), (-400*x)], [(-400*x),
(200)]])

# Closed-form exact line search for alpha

def exact_line_search(x, direction):
grad = gradient(x)
H = hessian(x)
# Calculate alpha using the closed-form solution
alpha = -np.dot(direction, grad) / np.dot(direction, H @
direction)
return alpha
# Define a function to print the results table
def print_table(history):
print("Iteration Table:")
print(f"{'k':<5}{'xk':<40}{'f(xk)':<15}{'∇f(xk)':<40}{'||
∇f(xk)||':<15}{'αk':<10}")
for row in history[:20]: # Show only the first 20 iterations
k, xk, fxk, gradk, grad_normk, alpha_k = row
print(f"{k:<5}{xk!s:<40}{fxk:<15.4f}{gradk!s:40}
{grad_normk:<15.4f}{alpha_k:<10.4f}")

# Define the Steepest Descent method

def steepest_descent(x_init, tol=1e-6, max_iter=10000):
x = np.array(x_init)
history = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the descent direction

direction = -grad

# Calculate step size using exact line search

alpha = exact_line_search(x, direction)

# Store iteration history

history.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Steepest Descent Results:")
print("Local Minima:", x)
print("Function Value at Minima:", np.round(f(x)))
print("Norm of Gradient at Minima:", np.round(grad_norm))
print_table(history)

# Initial point and calling the methods

x0 = [-1.2 ,1]
# Run Steepest Descent
steepest_descent(x0)

Steepest Descent Results:

Local Minima: [0.99981956 0.99963818]
Function Value at Minima: 0.0
Norm of Gradient at Minima: 0.0
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [-1.2 1. ] 24.2000 [-215.6 -
88. ] 232.8677 0.0007
1 [-1.05669744 1.05849084] 4.5678 [-
28.6789261 -11.62372983] 30.9450 0.0009
2 [-1.03089402 1.06894911] 4.1284 [-
1.50243928 1.24132486] 1.9489 0.0055
3 [-1.02260132 1.06209762] 4.1178
[2.65658762 3.27683435] 4.2184 0.0012
4 [-1.02572358 1.05824639] 4.1073 [-
1.53328598 1.22750477] 1.9641 0.0053
5 [-1.01767197 1.0518005 ] 4.0971
[2.53647925 3.22885142] 4.1060 0.0012
6 [-1.0207146 1.04792734] 4.0870 [-
1.56352453 1.21380877] 1.9794 0.0050
7 [-1.01287573 1.0418418 ] 4.0770
[2.42608788 3.18491161] 4.0037 0.0012
8 [-1.01584553 1.03794312] 4.0672 [-
1.59326549 1.20019506] 1.9947 0.0048
9 [-1.0081957 1.03218056] 4.0576
[2.32394252 3.14439643] 3.9100 0.0012
10 [-1.01109839 1.02825309] 4.0480 [-
1.62260259 1.18662744] 2.0102 0.0046
11 [-1.00361756 1.02278228] 4.0386 [2.22887
3.10681347] 3.8236 0.0013
12 [-1.00645798 1.01882303] 4.0293 [-
1.65161666 1.17307396] 2.0258 0.0044
13 [-0.99912899 1.01361756] 4.0201
[2.13992027 3.07176466] 3.7437 0.0013
14 [-1.00191127 1.00962372] 4.0110 [-
1.68037848 1.1595059 ] 2.0416 0.0043
15 [-0.99471927 1.00466105] 4.0020
[2.05631368 3.03892384] 3.6693 0.0013
16 [-0.99744696 1.00062993] 3.9931 [-
1.70895097 1.145897 ] 2.0576 0.0041
17 [-0.990379 0.99589068] 3.9842
[1.97740279 3.00802055] 3.5998 0.0014
18 [-0.99305517 0.99181969] 3.9755 [-
1.73739082 1.13222286] 2.0738 0.0040
19 [-0.98609984 0.98728704] 3.9668
[1.90264418 2.97882811] 3.5346 0.0014

After almost 10,000 iterations, steepest descent converges to minima

# Define the Newton's method

def newton_method(x_init, tol=1e-6, max_iter=400):
x = np.array(x_init)
history1 = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the Newton direction (solve H * direction = -grad)

H = hessian(x)
try:
direction = np.linalg.solve(H, -grad)
except np.linalg.LinAlgError:
print("Singular Hessian matrix encountered.")
break # Exit if Hessian is singular

# Calculate step size using exact line search

alpha = 1

# Store iteration history

history1.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Newton's Method Results:")
print("Local Minima:", x)
print("Function Value at Minima:", np.round(f(x)))
print("Norm of Gradient at Minima:", np.round(grad_norm))
print_table(history1)

# Initial point and calling the methods

x_init = [-1.2 ,1]
# Run Newton's Method
newton_method(x_init)

Newton's Method Results:

Local Minima: [1. 1.]
Function Value at Minima: 0.0
Norm of Gradient at Minima: 0.0
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [-1.2 1. ] 24.2000 [-215.6 -
88. ] 232.8677 1.0000
1 [-1.1752809 1.38067416] 4.7319 [-
4.63781641 -0.12220679] 4.6394 1.0000
2 [ 0.76311487 -3.17503385] 1411.8452
[1146.45069037 -751.47563227] 1370.7898 1.0000
3 [0.76342968 0.58282478] 0.0560 [-
4.73110379e-01 -1.98207786e-05] 0.4731 1.0000
4 [0.99999531 0.94402732] 0.3132
[ 22.38520499 -11.19265967] 25.0274 1.0000
5 [0.9999957 0.99999139] 0.0000 [-
8.60863355e-06 -2.95763414e-11] 0.0000 1.0000
6 [1. 1.] 0.0000
[ 7.41096051e-09 -3.70548037e-09] 0.0000 1.0000

Newton method is far more efficient

Question 3
# Define the function
import math
def f(x):
x1, x2 = x
return (1 - math.exp(-(x1**2 + x2**2)/2))

# Define the gradient of the function

def gradient(x):
x, y = x
df_dx = x*math.exp(-(x**2 + y**2)/2)
df_dy = y*math.exp(-(x**2 + y**2)/2)
return np.array([df_dx, df_dy])

# Define the Hessian of the function

def hessian(x):
x,y = x
return np.array([[((1 - x**2)*math.exp(-(x**2 + y**2)/2)), (-
x*y*math.exp(-(x**2 + y**2)/2))],
[(-x*y*math.exp(-(x**2 + y**2)/2)), ((1 -
y**2)*math.exp(-(x**2 + y**2)/2))]])
# Closed-form exact line search for alpha
def exact_line_search(x, direction):
grad = gradient(x)
H = hessian(x)
# Calculate alpha using the closed-form solution
alpha = -np.dot(direction, grad) / np.dot(direction, H @
direction)
return alpha

# Define a function to print the results table

def print_table(history):
print("Iteration Table:")
print(f"{'k':<5}{'xk':<40}{'f(xk)':<15}{'∇f(xk)':<40}{'||
∇f(xk)||':<15}{'αk':<10}")
for row in history[:20]: # Show only the first 20 iterations
k, xk, fxk, gradk, grad_normk, alpha_k = row
print(f"{k:<5}{xk!s:<40}{fxk:<15.4f}{gradk!s:40}
{grad_normk:<15.4f}{alpha_k:<10.4f}")

# Define the Steepest Descent method

def steepest_descent(x_init, tol=1e-6, max_iter=5000):
x = np.array(x_init)
history = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the descent direction

direction = -grad

# Calculate step size using exact line search

alpha = exact_line_search(x, direction)

# Store iteration history

history.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion
 # Print results
print("Steepest Descent Results:")
print("Local Minima:", x)
print("Function Value at Minima:", f(x))
print("Norm of Gradient at Minima:", grad_norm)
print_table(history)

# Initial point and calling the methods

x0 = [0.3,0.3]

# Run Steepest Descent

steepest_descent(x0)

Steepest Descent Results:

Local Minima: [0. 0.]
Function Value at Minima: 0.0
Norm of Gradient at Minima: 5.410085993607444e-10
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [0.3 0.3] 0.0861
[0.27417936 0.27417936] 0.3877 1.3344
1 [-0.06585366 -0.06585366] 0.0043 [-
0.06556869 -0.06556869] 0.0927 1.0131
2 [0.00057617 0.00057617] 0.0000
[0.00057617 0.00057617] 0.0008 1.0000
3 [-3.82550849e-10 -3.82550849e-10] 0.0000 [-
3.82550849e-10 -3.82550849e-10] 0.0000 1.0000

# Define the Newton's method

def newton_method(x_init, tol=1e-6, max_iter=400):
x = np.array(x_init)
history1 = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the Newton direction (solve H * direction = -grad)

H = hessian(x)
try:
direction = np.linalg.solve(H, -grad)
except np.linalg.LinAlgError:
print("Singular Hessian matrix encountered.")
break # Exit if Hessian is singular
 # Calculate step size using exact line search
alpha = 1

# Store iteration history

history1.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Newton's Method Results:")
print("Local Minima:", x)
print("Function Value at Minima:", f(x))
print("Norm of Gradient at Minima:", grad_norm)
print_table(history1)

# Initial point and calling the methods

x0 = [0.3,0.3]

# Run Newton's Method

newton_method(x0)

Newton's Method Results:

Local Minima: [0. 0.]
Function Value at Minima: 0.0
Norm of Gradient at Minima: 5.410085992840798e-10
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [0.3 0.3] 0.0861
[0.27417936 0.27417936] 0.3877 1.0000
1 [-0.06585366 -0.06585366] 0.0043 [-
0.06556869 -0.06556869] 0.0927 1.0000
2 [0.00057617 0.00057617] 0.0000
[0.00057617 0.00057617] 0.0008 1.0000
3 [-3.82550849e-10 -3.82550849e-10] 0.0000 [-
3.82550849e-10 -3.82550849e-10] 0.0000 1.0000

x0 = [0.5,0.5]

# Run Steepest Descent

steepest_descent(x0)
Steepest Descent Results:
Local Minima: [-4.66774454 -4.66774454]
Function Value at Minima: 0.999999999655125
Norm of Gradient at Minima: 6.269059725527811e-09
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 2.5681
1 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 2.5681
2 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 2.5681
3 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 2.5681
4 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 2.5681
5 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 2.5681
6 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 2.5681
7 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 2.5681
8 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 2.5681
9 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 2.5681
10 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 2.5681
11 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 2.5681
12 [0.50000001 0.50000001] 0.2212 [0.3894004
0.3894004] 0.5507 2.5681
13 [-0.50000007 -0.50000007] 0.2212 [-
0.38940042 -0.38940042] 0.5507 2.5681
14 [0.50000034 0.50000034] 0.2212
[0.38940052 0.38940052] 0.5507 2.5681
15 [-0.50000169 -0.50000169] 0.2212 [-
0.38940105 -0.38940105] 0.5507 2.5681
16 [0.50000843 0.50000843] 0.2212
[0.38940367 0.38940367] 0.5507 2.5682
17 [-0.50004213 -0.50004213] 0.2212 [-
0.3894168 -0.3894168] 0.5507 2.5686
18 [0.50021071 0.50021071] 0.2214 [0.3894824
0.3894824] 0.5508 2.5708
19 [-0.5010548 -0.5010548] 0.2220 [-
0.38981005 -0.38981005] 0.5513 2.5817

newton_method(x0)
Newton's Method Results:
Local Minima: [0.5 0.5]
Function Value at Minima: 0.22119921692859512
Norm of Gradient at Minima: 0.5506953149031838
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
1 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
2 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
3 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
4 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
5 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
6 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
7 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
8 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
9 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
10 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
11 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
12 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
13 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
14 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
15 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
16 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
17 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000
18 [0.5 0.5] 0.2212
[0.38940039 0.38940039] 0.5507 1.0000
19 [-0.5 -0.5] 0.2212 [-
0.38940039 -0.38940039] 0.5507 1.0000

We use grid search on [0,1] with step size 100 instead of closed form to calculate alpha
def exact_line_search1(x, direction):
alpha = np.linspace(0, 1, 100)
best_alpha = alpha[0]
best_fval = f(x + best_alpha * direction)

for a in alpha:
new_fval = f(x + a * direction)
if new_fval < best_fval:
best_alpha = a
best_fval = new_fval

return best_alpha

# Define the Steepest Descent method

def steepest_descent(x_init, tol=1e-6, max_iter=5000):
x = np.array(x_init)
history = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the descent direction

direction = -grad

# Calculate step size using exact line search

alpha = exact_line_search1(x, direction)

# Store iteration history

history.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Steepest Descent Results:")
print("Local Minima:", x)
print("Function Value at Minima:", f(x))
print("Norm of Gradient at Minima:", grad_norm)
print_table(history)
# Initial point and calling the methods
x0 = [3,3]

# Run Steepest Descent

steepest_descent(x0)

Steepest Descent Results:

Local Minima: [2.97182878e-13 2.97182878e-13]
Function Value at Minima: 0.0
Norm of Gradient at Minima: 4.20280057066815e-13
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [3 3] 0.9999
[0.00037023 0.00037023] 0.0005 1.0000
1 [2.99962977 2.99962977] 0.9999
[0.00037101 0.00037101] 0.0005 1.0000
2 [2.99925876 2.99925876] 0.9999
[0.00037179 0.00037179] 0.0005 1.0000
3 [2.99888698 2.99888698] 0.9999
[0.00037257 0.00037257] 0.0005 1.0000
4 [2.9985144 2.9985144] 0.9999
[0.00037336 0.00037336] 0.0005 1.0000
5 [2.99814105 2.99814105] 0.9999
[0.00037415 0.00037415] 0.0005 1.0000
6 [2.9977669 2.9977669] 0.9999
[0.00037494 0.00037494] 0.0005 1.0000
7 [2.99739196 2.99739196] 0.9999
[0.00037574 0.00037574] 0.0005 1.0000
8 [2.99701622 2.99701622] 0.9999
[0.00037654 0.00037654] 0.0005 1.0000
9 [2.99663968 2.99663968] 0.9999
[0.00037734 0.00037734] 0.0005 1.0000
10 [2.99626234 2.99626234] 0.9999
[0.00037815 0.00037815] 0.0005 1.0000
11 [2.99588419 2.99588419] 0.9999
[0.00037896 0.00037896] 0.0005 1.0000
12 [2.99550523 2.99550523] 0.9999
[0.00037977 0.00037977] 0.0005 1.0000
13 [2.99512545 2.99512545] 0.9999
[0.00038059 0.00038059] 0.0005 1.0000
14 [2.99474487 2.99474487] 0.9999
[0.00038141 0.00038141] 0.0005 1.0000
15 [2.99436346 2.99436346] 0.9999
[0.00038223 0.00038223] 0.0005 1.0000
16 [2.99398122 2.99398122] 0.9999
[0.00038306 0.00038306] 0.0005 1.0000
17 [2.99359816 2.99359816] 0.9999
[0.00038389 0.00038389] 0.0005 1.0000
18 [2.99321427 2.99321427] 0.9999
[0.00038472 0.00038472] 0.0005 1.0000
19 [2.99282955 2.99282955] 0.9999
[0.00038556 0.00038556] 0.0005 1.0000

def exact_line_search1(x, direction):

alpha = np.linspace(0, 1, 100)
best_alpha = alpha[0]
best_fval = f(x + best_alpha * direction)

for a in alpha:
new_fval = f(x + a * direction)
if new_fval < best_fval:
best_alpha = a
best_fval = new_fval

return best_alpha

# Define the Steepest Descent method

def steepest_descent(x_init, tol=1e-6, max_iter=5000):
x = np.array(x_init)
history = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the descent direction

direction = -grad

# Calculate step size using exact line search

alpha = exact_line_search1(x, direction)

# Store iteration history

history.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])

# Update the position

x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Steepest Descent Results:")
print("Local Minima:", x)
print("Function Value at Minima:", f(x))
 print("Norm of Gradient at Minima:", grad_norm)
print_table(history)

# Initial point and calling the methods

x0 = [3,3]

# Run Steepest Descent

steepest_descent(x0)

Steepest Descent Results:

Local Minima: [2.97182878e-13 2.97182878e-13]
Function Value at Minima: 0.0
Norm of Gradient at Minima: 4.20280057066815e-13
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [3 3] 0.9999
[0.00037023 0.00037023] 0.0005 1.0000
1 [2.99962977 2.99962977] 0.9999
[0.00037101 0.00037101] 0.0005 1.0000
2 [2.99925876 2.99925876] 0.9999
[0.00037179 0.00037179] 0.0005 1.0000
3 [2.99888698 2.99888698] 0.9999
[0.00037257 0.00037257] 0.0005 1.0000
4 [2.9985144 2.9985144] 0.9999
[0.00037336 0.00037336] 0.0005 1.0000
5 [2.99814105 2.99814105] 0.9999
[0.00037415 0.00037415] 0.0005 1.0000
6 [2.9977669 2.9977669] 0.9999
[0.00037494 0.00037494] 0.0005 1.0000
7 [2.99739196 2.99739196] 0.9999
[0.00037574 0.00037574] 0.0005 1.0000
8 [2.99701622 2.99701622] 0.9999
[0.00037654 0.00037654] 0.0005 1.0000
9 [2.99663968 2.99663968] 0.9999
[0.00037734 0.00037734] 0.0005 1.0000
10 [2.99626234 2.99626234] 0.9999
[0.00037815 0.00037815] 0.0005 1.0000
11 [2.99588419 2.99588419] 0.9999
[0.00037896 0.00037896] 0.0005 1.0000
12 [2.99550523 2.99550523] 0.9999
[0.00037977 0.00037977] 0.0005 1.0000
13 [2.99512545 2.99512545] 0.9999
[0.00038059 0.00038059] 0.0005 1.0000
14 [2.99474487 2.99474487] 0.9999
[0.00038141 0.00038141] 0.0005 1.0000
15 [2.99436346 2.99436346] 0.9999
[0.00038223 0.00038223] 0.0005 1.0000
16 [2.99398122 2.99398122] 0.9999
[0.00038306 0.00038306] 0.0005 1.0000
17 [2.99359816 2.99359816] 0.9999
[0.00038389 0.00038389] 0.0005 1.0000
18 [2.99321427 2.99321427] 0.9999
[0.00038472 0.00038472] 0.0005 1.0000
19 [2.99282955 2.99282955] 0.9999
[0.00038556 0.00038556] 0.0005 1.0000

A bigger point [3,3] works

x0 = [4,4]
steepest_descent(x0)

Steepest Descent Results:

Local Minima: [3.99999955 3.99999955]
Function Value at Minima: 0.99999988746442
Norm of Gradient at Minima: 6.365950813280083e-07
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [4 4] 1.0000
[4.50140699e-07 4.50140699e-07] 0.0000 1.0000

def newton_method(x_init, tol=1e-6, max_iter=400):

x = np.array(x_init)
history1 = []

for k in range(max_iter):
grad = gradient(x)
grad_norm = np.linalg.norm(grad)

# Determine the Newton direction (solve H * direction = -grad)

H = hessian(x)
try:
direction = np.linalg.solve(H, -grad)
except np.linalg.LinAlgError:
print("Singular Hessian matrix encountered.")
break # Exit if Hessian is singular

# Calculate step size using exact line search

alpha = exact_line_search1(x,direction)

# Store iteration history

history1.append([k, x.copy(), f(x), grad.copy(), grad_norm,
alpha])
 # Update the position
x = x + alpha * direction

if grad_norm < tol:

break # Convergence criterion

# Print results
print("Newton's Method Results:")
print("Local Minima:", x)
print("Function Value at Minima:", f(x))
print("Norm of Gradient at Minima:", grad_norm)
print_table(history1)

# Initial point and calling the methods

x0 = [0.7,0.7]

# Run Newton's Method

newton_method(x0)

Newton's Method Results:

Local Minima: [7.14211378e-09 7.14211378e-09]
Function Value at Minima: 0.0
Norm of Gradient at Minima: 9.999469427852664e-07
Iteration Table:
k xk f(xk) ∇f(xk)
||∇f(xk)|| αk
0 [0.7 0.7] 0.3874
[0.42883848 0.42883848] 0.6065 0.0202
1 [-0.00707071 -0.00707071] 0.0000 [-
0.00707035 -0.00707035] 0.0100 1.0000
2 [7.07069264e-07 7.07069264e-07] 0.0000
[7.07069264e-07 7.07069264e-07] 0.0000 0.9899

It works till [0.7,0.7]

So, we observe that as the initial points move farther and farther from the origin, the minima is
not attained. This is because as the points move away from the origin, the term exp-(x^2 + y^2)
decreases exponentially and the function becomes flatter and flatter. As that happens, it
becomes difficult for the methods to find the direction of the descent. Further, away from the
origin, the Hessian becomes nearly singular and the function loses its convex structure.