WIP

2024-11-27 02:48:17 +02:00
parent 65fd57a70f
commit 3b40c20ec0
25 changed files with 413 additions and 162 deletions
@@ -16,7 +16,8 @@ grad_fun    = matlabFunction(grad_fexpr, 'Vars', [x, y]); % Gradient
 hessian_fun = matlabFunction(hessian_fexpr, 'Vars', [x, y]);  % Hessian

 % Amijo globals
-global amijo_beta amijo_sigma
+global amijo_beta; % Step reduction factor in [0.1, 0.5] (typical range: [0.1, 0.8])
+global amijo_sigma; % Sufficient decrease constant in [1e-5, 0.1] (typical range: [0.01, 0.3])

 %fixed step size globals
 global  gamma_fixed_step
@@ -5,22 +5,19 @@ GivenEnv
 max_iter = 300; % Maximum iterations
 tol = 1e-4; % Tolerance

-% Methods tuning
-amijo_beta = 0.5; % Armijo reduction factor
-amijo_sigma = 0.1; % Armijo condition constant
-
 % Point x0 = (0, 0)
+% =========================================================================
 point = 1;
 x0 = [0, 0];
 f = fun(x0(1), x0(2));
 gf = grad_fun(x0(1), x0(2));
 hf = hessian_fun(x0(1), x0(2));
-fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Can not use method\n\n', x0, f, gf, hf);
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Can NOT use method\n', x0, f, gf, hf);
+disp(' ');

-% Points x0 = (-1, 1), (1, -1)
-%x0s = [-1, 1 ; 1, -1]; % Initial points

 % Point x0 = (-1, 1)
+% =========================================================================
 point = 2;
 x0 = [-1, 1];
 point_str = "[" + x0(1) + ", " + x0(2) + "]";
@@ -49,14 +46,69 @@ fprintf('Fixed step:     Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f),
 plotPointsOverContour(x_fixed, fun, [-2, 0], [-2, 2], 100, point_str + ": Steepest descent $\gamma$ = " + gamma_fixed_step, "figures/StDes_fixed_" + point + ".png");


-    % Minimized f
-    [x_minimized, f_minimized, kk] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'minimized');
-    fprintf('Minimized f(g): Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_minimized(end, :), f_minimized(end));
-    plotPointsOverContour(x_minimized, fun, [-2, 2], [-2, 2], 100, point_str + ": Steepest descent minimized $f(x_k + \gamma_kd_k)$", "StDes_minimized_" + i + ".png");  
+% Minimized f
+[x_minimized, f_minimized, kk] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'minimized');
+fprintf('Minimized f(g): Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_minimized(end, :), f_minimized(end));
+plotPointsOverContour(x_minimized, fun, [-2, 0], [-2, 2], 100, point_str + ": Steepest descent minimized $f(x_k + \gamma_kd_k)$", "figures/StDes_minimized_" + point + ".png");  
+
+
+% Armijo Rule
+
+% Methods tuning
+amijo_beta = 0.4;  % typical range: [0.1, 0.8]
+amijo_sigma = 0.1; % typical range: [0.01, 0.3]
+
+[x_armijo, f_armijo, kk] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'armijo');
+fprintf('Armijo step:    Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_armijo(end, :), f_armijo(end));
+plotPointsOverContour(x_armijo, fun, [-2, 0], [-2, 2], 100, point_str + ": Steepest descent Armijo method", "figures/StDes_armijo_" + point + ".png");  
+disp(' ');
+
+% Point x0 = (1, -1)
+% =========================================================================
+point = 3;
+x0 = [1, -1];
+point_str = "[" + x0(1) + ", " + x0(2) + "]";
+
+f = fun(-1, 1);
+gf = grad_fun(x0(1), x0(2));
+hf = hessian_fun(x0(1), x0(2));
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Can use method\n', x0, f, gf, hf);
+
+% Find the best fixed gamma
+k = zeros(100, 1);
+j = 1;
+n = linspace(0.1, 1, 100);
+for g = n
+    gamma_fixed_step = g;
+    [~, ~, k(j)] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'fixed');
+    j = j + 1;
+end
+%if min(k) == max_iter
+%    fprintf('Fixed step:     Initial point (%d, %d). Can NOT use method\n', x0);
+%end
+%plotItersOverGamma(n, k, "Iteration for different $\gamma$ values", "figures/StDes_Iter_o_gamma_" + point + ".png");
+
+%gamma_fixed_step = 1;
+[x_fixed, f_fixed, kk] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'fixed');
+fprintf('Fixed step:     Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_fixed(end, :), f_fixed(end));
+plotPointsOverContour(x_fixed, fun, [-1, 2], [-2, 2], 100, point_str + ": Steepest descent $\gamma$ = " + gamma_fixed_step, "figures/StDes_fixed_" + point + ".png");
+
+
+% Minimized f
+[x_minimized, f_minimized, kk] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'minimized');
+fprintf('Minimized f(g): Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_minimized(end, :), f_minimized(end));
+plotPointsOverContour(x_minimized, fun, [-2, 2], [-3, 2], 100, point_str + ": Steepest descent minimized $f(x_k + \gamma_kd_k)$", "figures/StDes_minimized_" + point + ".png");  
+
+
+% Armijo Rule
+
+% Methods tuning
+amijo_beta = 0.4;  % typical range: [0.1, 0.8]
+amijo_sigma = 0.1; % typical range: [0.01, 0.3]
+
+[x_armijo, f_armijo, kk] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'armijo');
+fprintf('Armijo step:    Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_armijo(end, :), f_armijo(end));
+plotPointsOverContour(x_armijo, fun, [-1, 2], [-2, 2], 100, point_str + ": Steepest descent Armijo method", "figures/StDes_armijo_" + point + ".png");


-    % Armijo Rule
-    [x_armijo, f_armijo, kk] = method_steepest_descent(fun, grad_fun, x0, tol, max_iter, 'armijo');
-    fprintf('Armijo step:    Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_armijo(end, :), f_armijo(end));
-    plotPointsOverContour(x_armijo, fun, [-2, 2], [-2, 2], 100, point_str + ": Steepest descent Armijo method", "StDes_armijo_" + i + ".png");  

@@ -1,51 +1,43 @@
 % Define environment (functions, gradients etc...)
 GivenEnv

-% Define parameters
-max_iter = 300; % Maximum iterations
-tol = 1e-4; % Tolerance
-
-% Methods tuning
-amijo_beta = 0.5; % Armijo reduction factor
-amijo_sigma = 0.1; % Armijo condition constant

 % Point x0 = (0, 0)
-% x0 = [0, 0];
+% =========================================================================
+point = 1;
+x0 = [0, 0];
+f = fun(x0(1), x0(2));
+gf = grad_fun(x0(1), x0(2));
+hf = hessian_fun(x0(1), x0(2));
+ev = eig(hf);
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Eigenvalues= [%f, %f], Can NOT use method\n', x0, f, gf, hf, ev);
+disp(' ');


-% Point x0 = (0, 0)
-x0s = [0, 0; -1, 1 ; 1, -1]; % Initial points
-for i = 1:size(x0s, 1)
-    x0 = x0s(i, :);
-    point_str = "[" + x0(1) + ", " + x0(2) + "]";
+% Point x0 = (-1, 1)
+% =========================================================================
+point = 2;
+x0 = [-1, 1];
+point_str = "[" + x0(1) + ", " + x0(2) + "]";

-    % Find the best fixed gamma
-    k = zeros(100, 1);
-    j = 1;
-    n = linspace(0.1, 1.5, 100);
-    for g = n
-        gamma_fixed_step = g;
-        [~, ~, k(j)] = newton(fun, grad_fun, hessian_fun, x0, tol, max_iter, 'fixed');
-        j = j + 1;
-    end
-    plotIterationsOverGamma(n, k, "Iteration for different $\gamma$ values", "Newton_Iter_o_gamma_" + i + ".png");
+f = fun(-1, 1);
+gf = grad_fun(x0(1), x0(2));
+hf = hessian_fun(x0(1), x0(2));
+ev = eig(hf);
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Eigenvalues= [%f, %f], Can NOT use method\n', x0, f, gf, hf, ev);
+disp(' ');

-    [~, j] = min(k);
-    gamma_fixed_step = n(j);
+% Point x0 = (1, -1)
+% =========================================================================
+point = 3;
+x0 = [1, -1];
+point_str = "[" + x0(1) + ", " + x0(2) + "]";

-    [x_fixed, f_fixed, kk] = newton(fun, grad_fun, hessian_fun, x0, tol, max_iter, 'fixed');
-    fprintf('Fixed step:     Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_fixed(end, :), f_fixed(end));
-    plotPointsOverContour(x_fixed, fun, [-2, 2], [-3, 3], 100, point_str + ": Newton $\gamma$ = " + gamma_fixed_step, "Newton_fixed_" + i + ".png");
+f = fun(-1, 1);
+gf = grad_fun(x0(1), x0(2));
+hf = hessian_fun(x0(1), x0(2));
+ev = eig(hf);
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Eigenvalues= [%f, %f], Can NOT use method\n', x0, f, gf, hf, ev);


-    % Minimized f
-    [x_minimized, f_minimized, kk] = newton(fun, grad_fun, hessian_fun, x0, tol, max_iter, 'minimized');
-    fprintf('Minimized f(g): Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_minimized(end, :), f_minimized(end));
-    plotPointsOverContour(x_minimized, fun, [-2, 2], [-3, 3], 100, point_str + ": Newton minimized $f(x_k + \gamma_kd_k)$", "Newton_minimized_" + i + ".png");  

-
-    % Armijo Rule
-    [x_armijo, f_armijo, kk] = newton(fun, grad_fun, hessian_fun, x0, tol, max_iter, 'armijo');
-    fprintf('Armijo step:    Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_armijo(end, :), f_armijo(end));
-    plotPointsOverContour(x_armijo, fun, [-2, 2], [-3, 3], 100, point_str + ": Newton Armijo method", "Newton_armijo_" + i + ".png");  
-end
@@ -5,49 +5,112 @@ GivenEnv
 max_iter = 300; % Maximum iterations
 tol = 1e-4; % Tolerance

-% Methods tuning
-amijo_beta = 0.5; % Armijo reduction factor
-amijo_sigma = 0.1; % Armijo condition constant
-
-m = 0.01;

 % Point x0 = (0, 0)
-% x0 = [0, 0];
+% =========================================================================
+point = 1;
+x0 = [0, 0];
+f = fun(x0(1), x0(2));
+gf = grad_fun(x0(1), x0(2));
+hf = hessian_fun(x0(1), x0(2));
+ev = eig(hf);
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Eigenvalues= [%f, %f], Can NOT use method\n', x0, f, gf, hf, ev);
+disp(' ');


-% Point x0 = (0, 0)
-x0s = [0, 0; -1, 1 ; 1, -1]; % Initial points
-for i = 1:size(x0s, 1)
-    x0 = x0s(i, :);
-    point_str = "[" + x0(1) + ", " + x0(2) + "]";
+% Point x0 = (-1, 1)
+% =========================================================================
+point = 2;
+x0 = [-1, 1];
+point_str = "[" + x0(1) + ", " + x0(2) + "]";

-    % Find the best fixed gamma
-    k = zeros(100, 1);
-    j = 1;
-    n = linspace(0.1, 1.5, 100);
-    for g = n
-        gamma_fixed_step = g;
-        [~, ~, k(j)] = lev_mar(fun, grad_fun, hessian_fun, m, x0, tol, max_iter, 'fixed');
-        j = j + 1;
+f = fun(-1, 1);
+gf = grad_fun(x0(1), x0(2));
+hf = hessian_fun(x0(1), x0(2));
+ev = eig(hf);
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Eigenvalues= [%f, %f], Can use method\n', x0, f, gf, hf, ev);
+
+
+% Find the best fixed gamma
+k = zeros(100, 1);
+j = 1;
+n = linspace(0.1, 1.5, 100);
+for g = n
+    gamma_fixed_step = g;
+    [x, f, k(j)] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'fixed');
+    if ~(x(end, 1) < -1.57 && x(end, 1) > -1.59 && x(end, 2) < 0.01 && x(end,2) > -0.01 && f(end) < -0.8 && f(end) > -0.82)
+        k(j) = 300;
    end
-    plotIterationsOverGamma(n, k, "Iteration for different $\gamma$ values", "LevMar_Iter_o_gamma_" + i + ".png");
-
-    [~, j] = min(k);
-    gamma_fixed_step = n(j);
-
-    [x_fixed, f_fixed, kk] = lev_mar(fun, grad_fun, hessian_fun, m, x0, tol, max_iter, 'fixed');
-    fprintf('Fixed step:     Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_fixed(end, :), f_fixed(end));
-    plotPointsOverContour(x_fixed, fun, [-2, 2], [-3, 3], 100, point_str + ": LevMar $\gamma$ = " + gamma_fixed_step, "LevMar_fixed_" + i + ".png");
-
-
-    % Minimized f
-    [x_minimized, f_minimized, kk] = lev_mar(fun, grad_fun, hessian_fun, m, x0, tol, max_iter, 'minimized');
-    fprintf('Minimized f(g): Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_minimized(end, :), f_minimized(end));
-    plotPointsOverContour(x_minimized, fun, [-2, 2], [-3, 3], 100, point_str + ": LevMar minimized $f(x_k + \gamma_kd_k)$", "LevMar_minimized_" + i + ".png");  
-
-
-    % Armijo Rule
-    [x_armijo, f_armijo, kk] = lev_mar(fun, grad_fun, hessian_fun, m, x0, tol, max_iter, 'armijo');
-    fprintf('Armijo step:    Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_armijo(end, :), f_armijo(end));
-    plotPointsOverContour(x_armijo, fun, [-2, 2], [-3, 3], 100, point_str + ": LevMar Armijo method", "LevMar_armijo_" + i + ".png");  
+    j = j + 1;
 end
+
+[~, j] = min(k);
+gamma_fixed_step = n(j);
+
+[x_fixed, f_fixed, kk] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'fixed');
+fprintf('Fixed step:     Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_fixed(end, :), f_fixed(end));
+plotPointsOverContour(x_fixed, fun, [-3, 0], [-2, 2], 100, point_str + ": Levenberg-Marquardt $\gamma$ = " + gamma_fixed_step, "figures/LevMar_fixed_" + point + ".png");
+
+[x_fixed, f_fixed, kk] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'minimized');
+fprintf('Minimized f(g): Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_fixed(end, :), f_fixed(end));
+plotPointsOverContour(x_fixed, fun, [-3, 0], [-2, 2], 100, point_str + ": Levenberg-Marquardt minimized $f(x_k + \gamma_kd_k)$", "figures/LevMar_minimized_" + point + ".png");
+
+% Armijo Rule
+
+% Methods tuning
+amijo_beta = 0.4;  % typical range: [0.1, 0.8]
+amijo_sigma = 0.1; % typical range: [0.01, 0.3]
+
+[x_armijo, f_armijo, kk] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'armijo');
+fprintf('Armijo step:    Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_armijo(end, :), f_armijo(end));
+plotPointsOverContour(x_armijo, fun, [-3, 0], [-2, 2], 100, point_str + ": Levenberg-Marquardt Armijo method", "figures/StDes_armijo_" + point + ".png");  
+disp(' ');
+
+
+% Point x0 = (1, -1)
+% =========================================================================
+point = 3;
+x0 = [1, -1];
+point_str = "[" + x0(1) + ", " + x0(2) + "]";
+
+f = fun(-1, 1);
+gf = grad_fun(x0(1), x0(2));
+hf = hessian_fun(x0(1), x0(2));
+ev = eig(hf);
+fprintf('Initial point (%d, %d), f = %f, grad = [%f;%f], hessian = [%f %f ; %f %f]. Eigenvalues= [%f, %f], Can use method\n', x0, f, gf, hf, ev);
+
+
+% Find the best fixed gamma
+k = zeros(100, 1);
+j = 1;
+n = linspace(0.1, 1.5, 100);
+for g = n
+    gamma_fixed_step = g;
+    [x, f, k(j)] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'fixed');
+    if ~(x(end, 1) < -1.57 && x(end, 1) > -1.59 && x(end, 2) < 0.01 && x(end,2) > -0.01 && f(end) < -0.8 && f(end) > -0.82)
+        k(j) = 300;
+    end
+    j = j + 1;
+end
+
+[~, j] = min(k);
+gamma_fixed_step = n(j);
+
+[x_fixed, f_fixed, kk] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'fixed');
+fprintf('Fixed step:     Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_fixed(end, :), f_fixed(end));
+plotPointsOverContour(x_fixed, fun, [-3, 2], [-2, 2], 100, point_str + ": Levenberg-Marquardt $\gamma$ = " + gamma_fixed_step, "figures/LevMar_fixed_" + point + ".png");
+
+[x_fixed, f_fixed, kk] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'minimized');
+fprintf('Minimized f(g): Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_fixed(end, :), f_fixed(end));
+plotPointsOverContour(x_fixed, fun, [-3, 2], [-2, 2], 100, point_str + ": Levenberg-Marquardt minimized $f(x_k + \gamma_kd_k)$", "figures/LevMar_minimized_" + point + ".png");
+
+% Armijo Rule
+
+% Methods tuning
+amijo_beta = 0.4;  % typical range: [0.1, 0.8]
+amijo_sigma = 0.1; % typical range: [0.01, 0.3]
+
+[x_armijo, f_armijo, kk] = method_lev_mar(fun, grad_fun, hessian_fun, 0.3, x0, tol, max_iter, 'armijo');
+fprintf('Armijo step:    Initial point (%f, %f), steps:%d, Final (x,y)=(%f, %f), f(x,y)=%f\n', x0, kk, x_armijo(end, :), f_armijo(end));
+plotPointsOverContour(x_armijo, fun, [-3, 2], [-2, 2], 100, point_str + ": Levenberg-Marquardt Armijo method", "figures/StDes_armijo_" + point + ".png");  
+disp(' ');
@@ -0,0 +1,49 @@
+function [a, b, k, n] = fmin_bisection(fun, alpha, beta, epsilon, lambda)
+% Bisection method for finding the local minimum of a function.
+%
+% fun:      The objective function
+% alpha:    (number) The starting point of the interval in which we seek
+%           for minimum
+% beta:     (number) The ending point of the interval in which we seek
+%           for minimum
+% epsilon:  (number) The epsilon value (distance from midpoint)
+% lambda:   (number) The lambda value (accuracy)
+%
+% return:
+%  a:       (vector) Starting points of the interval for each iteration
+%  b:       (vector) Ending points of the interval for each iteration
+%  k:       (number) The number of iterations
+%  n:       (number) The calls of objective function fun_expr
+%
+
+
+
+% Error checking
+if alpha > beta || 2*epsilon >= lambda || lambda <= 0
+    error ('Input criteria not met')
+end
+
+% Init
+a = alpha;
+b = beta;
+n = 0;
+
+k=1;
+while b(k) - a(k) > lambda
+    % bisect [a,b]
+    mid = (a(k) + b(k)) / 2;
+    x_1 = mid - epsilon;
+    x_2 = mid + epsilon;
+    
+    % set new search interval
+    k = k + 1;
+    if fun(x_1) < fun(x_2)
+        a(k) = a(k-1);
+        b(k) = x_2;
+    else
+        a(k) = x_1;
+        b(k) = b(k-1);
+    end
+end
+
+end
@@ -1,25 +1,33 @@
-function [gamma] = gamma_armijo(f, grad_f, x0)
+function [gamma] = gamma_armijo(f, grad_f, dk, xk)
    % Calculates the best step based on amijo method
    %
-    % f(xk− γk*∇f(xk)) ≤ f(xk) − σ*γk*∥∇f(xk)∥^2
+    % f(xk+ γk*dk) ≤ f(xk) + σ * γk * dk^T * ∇f(xk)
+    % γk = β*γk_0
    %
    % f:        Objective function
-    % x0:       Initial (x,y) point
+    % grad_fun: Gradient function of f
+    % dk:       Current value of selected direction -∇f or -inv{H}*∇f or -inv{H + lI}*∇f
+    % xk:       Current point (x,y)
    
    
-    % beta:     beta factor in (0, 1)
-    % signam:   sigma factor in (0,1)
+    % beta:     beta  factor in [0.1, 0.5]
+    % signam:   sigma factor in (0, 0.1]
    global amijo_beta
    global amijo_sigma

+    gf = grad_f(xk(1), xk(2));
    gamma = 1; % Start with a step size of 1

-    grad = grad_f(x0(1), x0(2));
-
    % Perform Armijo line search
-    while f(x0(1) - gamma * grad(1), x0(2) - gamma * grad(2)) > ...
-          f(x0(1), x0(2)) - amijo_sigma * gamma * norm(grad)^2
+    while f(xk(1) + gamma * dk(1), xk(2) + gamma * dk(2)) > ...
+          f(xk(1), xk(2)) + amijo_sigma * gamma * dk' * gf
+    %while f(xk(1) + gamma * dk(1), xk(2) + gamma * dk(2)) > ...
+    %      f(xk(1), xk(2)) + amijo_sigma * gamma * norm(dk)^2
        gamma = amijo_beta * gamma; % Reduce step size
+        if gamma < 1e-12 % Safeguard to prevent infinite reduction
+            warning('Armijo step size became too small.');
+            break;
+        end
    end

 end
@@ -1,4 +1,4 @@
-function [gamma] = gamma_fixed(~, ~, ~)
+function [gamma] = gamma_fixed(~, ~, ~, ~)
    % Return a fixed step
    %
    % This is for completion and code symmetry. 
@@ -1,18 +1,24 @@
-function [gamma] = gamma_minimized(f, grad_f, x0)
-    % Calculates the step based on minimizing f(xk− γ*∇f(xk))
+function [gamma] = gamma_minimized(f, ~, dk, xk)
+    % Calculates the step based on minimizing f(xk− γk*dk)
    %
    %
-    % f:        Objective function
-    % grad_f:   Gradient of objective function
-    % x0:       Initial (x,y) point
+    % f:    Objective function
+    % ~:    Gradient function of f - Not used
+    % dk:       Current value of selected direction -∇f or -inv{H}*∇f or -inv{H + lI}*∇f
+    % xk:   Current point (x,y)

-    
-    % Define the line search function g(gamma) = f(x0 - gamma * grad)
-    grad = grad_f(x0(1), x0(2));
-    g = @(gamma) f(x0(1) - gamma * grad(1), x0(2) - gamma * grad(2));
+    % Define the line search function fmin(g) = f(xk - g * dk)
+    fmin = @(g) f(xk(1) + g*dk(1), xk(2) + g*dk(2));

-    % Perform line search
-    gamma = fminbnd(g, 0, 1);
-        % ToDo: Check if we can use fmin_bisection_der
-        %       from the previous assigment here!
+    % find g that minimizes fmin
+    e = 0.0001;
+    l = 0.001;
+    [a,b,k,~] = fmin_bisection(fmin, 0, 5, e, l);
+    gamma = 0.5*(a(k) + b(k));
+
+    % Define the line search function fmin(g) = f(xk - g * dk)
+    %fmin = @(g) f(xk(1) - gamma * dk(1), xk(2) - gamma * dk(2));
+
+    % find g that minimizes fmin
+    %gamma = fminbnd(g, 0, 1);
 end
@@ -1,8 +1,9 @@
-function [x_vals, f_vals, k] = method_lev_mar(f, grad_f, hessian_f, m, x0, tol, max_iter, mode)
+function [x_vals, f_vals, k] = method_lev_mar(f, grad_f, hessian_f, e, xk, tol, max_iter, mode)
    % f:         Objective function
    % grad_f:    Gradient of the function
    % hessian_f: Hessian of the function
-    % x0:        Initial point [x0, y0]
+    % e:         mu offset for hessian damping H' = H_k + mI
+    % xk:        Initial point [xk, yk]
    % tol:       Tolerance for stopping criterion
    % max_iter:  Maximum number of iterations
    
@@ -12,36 +13,46 @@ function [x_vals, f_vals, k] = method_lev_mar(f, grad_f, hessian_f, m, x0, tol,


    if strcmp(mode, 'armijo') == 1
-        gamma_f = @(f, grad_f, x0) gamma_armijo(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_armijo(f, grad_f, dk, xk);
    elseif strcmp(mode, 'minimized') == 1
-        gamma_f = @(f, grad_f, x0) gamma_minimized(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_minimized(f, grad_f, dk, xk);
    else % mode == 'fixed'
-        gamma_f = @(f, grad_f, x0) gamma_fixed(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_fixed(f, grad_f, dk, xk);
    end

-    x_vals = x0;  % Store iterations
-    f_vals = f(x0(1), x0(2));
+    x_vals = xk;  % Store iterations
+    f_vals = f(xk(1), xk(2));
    
    for k = 1:max_iter
-        grad = grad_f(x0(1), x0(2));
+        grad = grad_f(xk(1), xk(2));
        
        % Check for convergence
        if norm(grad) < tol
            break;
        end
-        hess = hessian_f(x0(1), x0(2));
-        mI = m * eye(size(hess));
+        hess = hessian_f(xk(1), xk(2));
+        
+        % Check if hessian is not positive defined
+        lmin = min(eig(hess));
+        if lmin <= 0
+            m = abs(lmin) + e;
+            mI = m * eye(size(hess));
+            nev = eig(hess + mI);
+            if min(nev) <= 0
+                warning('Can not normalize hessian matrix.');
+            end
+        end
        
         % Solve for search direction using Newton's step
        dk = - inv(hess + mI) * grad;

        % Calculate gamma
-        gamma = gamma_f(f, grad_f, x0);
+        gk = gamma_f(f, grad_f, dk, xk);
        
-        x_next = x0 + gamma * dk'; % Update step
+        x_next = xk + gk * dk'; % Update step
        f_next = f(x_next(1), x_next(2));
        
-        x0 = x_next; % Update point
+        xk = x_next; % Update point
        x_vals = [x_vals; x_next]; % Store values
        f_vals = [f_vals; f_next]; % Store function values
    end
@@ -1,4 +1,4 @@
-function [x_vals, f_vals, k] = method_newton(f, grad_f, hessian_f, x0, tol, max_iter, mode)
+function [x_vals, f_vals, k] = method_newton(f, grad_f, hessian_f, xk, tol, max_iter, mode)
    % f:         Objective function
    % grad_f:    Gradient of the function
    % hessian_f: Hessian of the function
@@ -12,35 +12,35 @@ function [x_vals, f_vals, k] = method_newton(f, grad_f, hessian_f, x0, tol, max_


    if strcmp(mode, 'armijo') == 1
-        gamma_f = @(f, grad_f, x0) gamma_armijo(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_armijo(f, grad_f, dk, xk);
    elseif strcmp(mode, 'minimized') == 1
-        gamma_f = @(f, grad_f, x0) gamma_minimized(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_minimized(f, grad_f, dk, xk);
    else % mode == 'fixed'
-        gamma_f = @(f, grad_f, x0) gamma_fixed(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_fixed(f, grad_f, dk, xk);
    end

-    x_vals = x0;  % Store iterations
-    f_vals = f(x0(1), x0(2));
+    x_vals = xk;  % Store iterations
+    f_vals = f(xk(1), xk(2));
    
    for k = 1:max_iter
-        grad = grad_f(x0(1), x0(2));
+        grad = grad_f(xk(1), xk(2));
        
        % Check for convergence
        if norm(grad) < tol
            break;
        end
-        hess = hessian_f(x0(1), x0(2));
+        hess = hessian_f(xk(1), xk(2));
        
         % Solve for search direction using Newton's step
        dk = - inv(hess) * grad;

        % Calculate gamma
-        gamma = gamma_f(f, grad_f, x0);
+        gk = gamma_f(f, grad_f, dk, xk);
        
-        x_next = x0 + gamma * dk'; % Update step
+        x_next = xk + gk * dk'; % Update step
        f_next = f(x_next(1), x_next(2));
        
-        x0 = x_next; % Update point
+        xk = x_next; % Update point
        x_vals = [x_vals; x_next]; % Store values
        f_vals = [f_vals; f_next]; % Store function values
    end
@@ -1,7 +1,7 @@
-function [x_vals, f_vals, k] = method_steepest_descent(f, grad_f, x0, tol, max_iter, mode)
+function [x_vals, f_vals, k] = method_steepest_descent(f, grad_f, xk, tol, max_iter, mode)
    % f:        Objective function
    % grad_f:   Gradient of the function
-    % x0:       Initial point [x0, y0]
+    % xk:       Initial point [x0, y0]
    % tol:      Tolerance for stopping criterion
    % max_iter: Maximum number of iterations
    
@@ -10,19 +10,19 @@ function [x_vals, f_vals, k] = method_steepest_descent(f, grad_f, x0, tol, max_i
    % k:      Number of iterations
    
    if strcmp(mode, 'armijo') == 1
-        gamma_f = @(f, grad_f, x0) gamma_armijo(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_armijo(f, grad_f, dk, xk);
    elseif strcmp(mode, 'minimized') == 1
-        gamma_f = @(f, grad_f, x0) gamma_minimized(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_minimized(f, grad_f, dk, xk);
    else % mode == 'fixed'
-        gamma_f = @(f, grad_f, x0) gamma_fixed(f, grad_f, x0);
+        gamma_f = @(f, grad_f, dk, xk) gamma_fixed(f, grad_f, dk, xk);
    end
    
    % Storage for iterations, begin with the first point
-    x_vals = x0;
-    f_vals = f(x0(1), x0(2));
+    x_vals = xk;
+    f_vals = f(xk(1), xk(2));
    
    for k = 1:max_iter
-        grad = grad_f(x0(1), x0(2));
+        grad = grad_f(xk(1), xk(2));
        
        % Check for convergence
        if norm(grad) < tol
@@ -31,12 +31,12 @@ function [x_vals, f_vals, k] = method_steepest_descent(f, grad_f, x0, tol, max_i
        dk = - grad;

        % Calculate gamma
-        gk = gamma_f(f, grad_f, x0);
+        gk = gamma_f(f, grad_f, dk, xk);
        
-        x_next = x0 + gk * dk'; % Update step
+        x_next = xk + gk * dk'; % Update step
        f_next = f(x_next(1), x_next(2));
        
-        x0 = x_next; % Update point
+        xk = x_next; % Update point
        x_vals = [x_vals; x_next]; % Store values
        f_vals = [f_vals; f_next]; % Store function values
    end
@@ -21,7 +21,7 @@ function plotPointsOverContour(points, contour_fun, x_lim, y_lim, size, plot_tit
    Z = contour_fun(X, Y);
    
    % 2D plot
-    figure('Name', '(x,y)', 'NumberTitle', 'off');
+    figure('Name', '(x,y) convergence', 'NumberTitle', 'off');
    set(gcf, 'Position', [100, 100, image_width, image_height]); % Set the figure size
    plot(points(:, 1), points(:, 2), '-or');
    hold on