opt

02610/week1/ex2.m

+%% Gradient and Hessian of functions
+clear all; close all; clc;
+%% 2.1
+% f(x) = g1*x1+g2*x2+g3*x3
+% df(x) = [g1;g2;g3]
+% in more genral case
+% df(x) = [g1;g2;...;gn]
+
+%% 2.2
+% Expand the expression of f(x)
+% f(x) = 1/2*(h11*x1^2+h22*x2^2+h33*x3^2+(h12+h21)*x1*x2+(h13+h31)*x1*x3+(h23+h32)*x2*x3)
+% df(x) = [ h11*x1+0.5*(h12+h21)*x2+0.5*(h13+h31)*x3
+%           h22*x2+0.5*(h12+h21)*x1+0.5*(h23+h32)*x3
+%           h33*x3+0.5*(h13+h31)*x1+0.5*(h23+h32)*h2]
+% Hession = [h11,0.5*(h12+h21),0.5*(h13+h31)
+%           0.5*(h12+h21),h22,0.5*(h23+h32)
+%           0.5*(h13+h31),0.5*(h23+h32),h33];
+
+% now assume the matrix is symmteric
+% f(x) = 0.5*h11*x1*2+0.5*h22*x2^2+0.5*h33*x3^2+h12*x1*x2+h13*x1*x3+h23*x2*x3
+% df(x) = [h11*x1+h12*x2+h13*x3;h22*x2+h12*x1+h23*x3;h33*x3+h13*x1+h23*x2]
+% Hession  = [h11,h12,h13;h12,h22,h23;h13,h23,h33]
+
+% df(x) = [h11*x1+h12*x2+...+h1n*xn;h21*x1+h22*x2+...+h2n*xn,...,hn1*x1+hn2*x2+...+hnn*xn]
+% Hession = [h11,h12,...,h1n;h21,h22,...,h2n;...;hn1,hn2,...,hnn]

02610/week1/ex3.m

+%% Minimizers of univariate and multivariate problems
+clear all; close all; clc;
+%%
+% f(x) = 3/2*(x1^2+x2^2)+(1+a)*x1*x2-(x1+x2)+b
+% df(x) = [3*x1+(1+a)*x2-1;
+%           3*x2+(1+a)*x1-1]
+% Hession = [3,1+a;
+%            1+a,3]
+% a和b取何值令f(x)有且只有一个最优解？
+% x1 = x2 = (2-a)/(9+(1+a)^2)
+% eigen value is (2-a) or (4+a)
+% -4<=a<=2
+
+%%
+% f(x) = 1/2*x^T*Q*x-b^T*x
+% dx = Q*x-b^T = 0
+% x* = solve(Qx=b)
+
+%%
+% min(b^T*x)
+% df(x) = b inequal 0
+% doesn't meet the optimization necessary condition
+
+%%

02610/week1/ex4.m

+%% Convexity
+clear all; close all; clc;
+%%
+% assume 2 points in the set, Ax1<=b, Ax2<=b
+% a*x1+(1-a)*x2
+%a*A*x1+(1-a)*A*x2 <= a*b+(1-a)*b=b
+% thus the point a*x1+(1-a)*x2 is in the set
+% proof end
+
+%%
+f1 = @(x) x.^2+x+1;
+df1 = @(x) 2*x+1;
+ddf1 = @(x) ones(1,size(x,2))*2;
+f2 = @(x) -x.^2+x+1;
+df2 = @(x) -2*x+1;
+ddf2 = @(x) -2*ones(1,size(x,2));
+f3 = @(x) x.^3-5*x.^2+x+1;
+df3 = @(x) 3*x.^2 - 10*x +1;
+ddf3 = @(x) 6*x - 10;
+f4 = @(x) x.^4+x.^3-10*x.^2-x+1;
+df4 = @(x) 4*x.^3+3*x.^2-20*x-1;
+ddf4 = @(x) 12*x.^2+6*x-20;
+x1 = [-2:0.01:2];
+x2 = [-4:0.1:4];
+
+figure(1);
+subplot(3,1,1);
+plot(x1, f1(x1));
+title('function');
+subplot(3,1,2);
+plot(x1, df1(x1));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x1, ddf1(x1));
+title('2-nd order derivation');
+
+figure(2);
+subplot(3,1,1);
+plot(x1, f2(x1));
+title('function');
+subplot(3,1,2);
+plot(x1, df2(x1));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x1, ddf2(x1));
+title('2-nd order derivation');
+
+figure(3);
+subplot(3,1,1);
+plot(x2, f3(x2));
+title('function');
+subplot(3,1,2);
+plot(x2, df3(x2));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x2, ddf3(x2));
+title('2-nd order derivation');
+
+figure(4);
+subplot(3,1,1);
+plot(x2, f4(x2));
+title('function');
+subplot(3,1,2);
+plot(x2, df4(x2));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x2, ddf4(x2));
+title('2-nd order derivation');
+
+%%
+% function 1
+
+%%
+figure(1);
+subplot(3,1,1);
+plot(x1, f1(x1));
+title('function');
+subplot(3,1,2);
+plot(x1, df1(x1));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x1, ddf1(x1));
+title('2-nd order derivation');
+% local
+% minimizer = -0.5
+% maximizer = None
+% global
+% minimizer = -0.5
+% maximizer = 2
+% conditions
+% 1-st order derivation: 0
+% 2-nd order derivation: >0
+%%
+figure(2);
+subplot(3,1,1);
+plot(x1, f2(x1));
+title('function');
+subplot(3,1,2);
+plot(x1, df2(x1));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x1, ddf2(x1));
+title('2-nd order derivation');
+% local
+% minimizer = -2
+% maximizer = None
+% global
+% minimizer = -2
+% maximizer = 0.5
+% conditions
+% 1-st order derivation: 0
+% 2-nd order derivation: <0
+%%
+figure(3);
+subplot(3,1,1);
+plot(x2, f3(x2));
+title('function');
+subplot(3,1,2);
+plot(x2, df3(x2));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x2, ddf3(x2));
+title('2-nd order derivation');
+% local
+% minimizer = 3
+% maximizer = 0
+% global
+% minimizer = -4
+% maximizer = 0
+% conditions
+% 1-st order derivation: 0
+% 2-nd order derivation: >0 || <0
+%%
+figure(4);
+subplot(3,1,1);
+plot(x2, f4(x2));
+title('function');
+subplot(3,1,2);
+plot(x2, df4(x2));
+title('1-st order derivation');
+subplot(3,1,3);
+plot(x2, ddf4(x2));
+title('2-nd order derivation');
+% local
+% minimizer = -2.5,2
+% maximizer = 0
+% global
+% minimizer = -2.5
+% maximizer = 4
+% conditions
+% 1-st order derivation: 0
+% 2-nd order derivation: >0 / <0
+%%
+% f(x) = 2*x1^2-2*x1*x2+1/2*x2^2+3*x1-x2
+% f(x) = x1*(2*x1-x2+3)+x2*(1/2*x2-x1-1)
+% f(x) = [x1 x2]*[2x1-x2+3;x2/2-x1+1]
+% f(x) = [x1 x2]*[2 -1 3;-1 1/2 -1]*[x1;x2;1]
+% df(x) = [4x1-2x2+3;
+%           x2-2x1-1]
+% Hession = [4,-2;-2,1]
+% det(Hession) = 0, it's singular
+% eigen value = 0 or 5
+% Hession matrix is semi-definite
+% then f(x) is a convex function
+
+%%
+% f(global minimizers) must be equal, and they must be on a horizontal line
+% assume f({x*}) = b
+% thus a*f(x1)+(1-a)*f(x2) = b = f(somewhere between x1 and x2) = f(a*x1+(1-a)*x2)
+% thus {x*} is a convex set
+
+%%
+% determine a convex problem
+% three conditions
+% the objective function is convex
+% the equal constrainit function is linear
+% the inequal constraint function is concave
+% f(x) = x*log(x) is convex (negative entrophy)
+% x is convex, thus this is not a convex problem
+