stprm3.m

书籍“Regularization tools for training large feed-forward neural networks using Automatic Differentiat

function [xny,X1,Y1,A1,X2,Y2,A2,X3,Y3,A3,stplen,eval, Fsq,fnew]=...
stprm3(P,T,W1,B1,f1,W2,B2,f2,W3,B3,f3,x,p,mu,f0,Fsq0,Pf,alfin, xc)
%Call:
% [xny,X1,Y1,A1,X2,Y2,A2,X3,Y3,A3,stplen,eval,Fsq,fnew]=...
%stprm3(P,T,W1,B1,f1,W2,B2,f2,W3,B3,f3,x,p,mu,f0,Fsq0,Pf,alfin,xc)
%This function computes the steplength (stplen) along the direction
%p such that the objective is sufficiently reduced.

%ON ENTRY:
% P(r*q) holds the input data. (r input nodes and q data points)
% T(s2*q) holds the target values.
% W1(s1*r) holds the weights corresponding to the first hidden layer
% B1(s1*1) holds the offsets for the first hidden layer
% f1 is the name of the transfer function used in the first hidden layer
% W2(s2*s1) holds the weights corresponding to the second hidden layer
% B2(s2*1) holds the offsets for the second hidden layer
% f2 is the name of the transfer function used in the second hidden layer
% W3(s3*s2) holds the weights corresponding to the output layer
% B3(s3*1) holds the offsets for the output layer
% f3 is the name of the transfer function used in the output layer
% x holds the current point packed as [W1(:); B1(:); W2(:); B2(:); W3(:); B3(:)]
% p holds the search direction packed as x is packed
% mu holds the current regularization parameter
% f0 holds the error vector corresponding to the current point
% Fsq0 holds the current value of the augmented objective
% Pf holds the projection of faug onto the range of the Jacobian Jaug
% alfin holds the suggested steplength
% xc holds the centre of the unknowns in packed form

%ON RETURN:
%xny = x + stplen*p
%X1 = the new value of the weights in the first hidden layer
%Y1 = the new value of the biases in the first hidden layer
%A1 = the new value comming out from the first hidden layer
%X2 = the new value of the weights in the second hidden layer
%Y2 = the new value of the biases in the second hidden layer
%A2 = the new value comming out from the second hidden layer
%X3 = the new value of the weights in the output layer
%Y3 = the new value of the biases in the output layer
%A3 = the new value comming out from the output layer
%stplen = the computed steplength
%eval = the number of evaluations of the actual NN inside this function
%Fsq = the new value of the augmented objective
%fnew = the new error value (A3-T)

faug=[f0;mu*(x-xc)];
v0 = faug;
v1 = Pf;
F0 = faug;

alfa=alfin;
tau=0.1;
xny=x+alfa*p;
nrmaug=(xny-xc)'*(xny-xc)*mu^2/2;

%Don't use the suggested alfa if p is unreasonable large
indc=0;
while nrmaug > Fsq0
  alfa=alfa/2;
  indc=indc+1;
  xny=x+alfa*p;
  nrmaug=(xny-xc)'*(xny-xc)*mu^2/2;
end % of while
if indc >0
  indc
  pause
end

[X1,Y1,X2,Y2,X3,Y3]=unpack3(xny,W1,B1,W2,B2,W3,B3);
[A1,A2,A3]=simuff(P,X1,Y1,f1,X2,Y2,f2,X3,Y3,f3);
ny=A3-T;
fnew=ny;
fny=ny(:);
Fsq=0.5*sumsqr(fny)+nrmaug;
i=1;
F2 = [fny;mu*(xny-xc)];
alf2= alfa;
sec = 0;

while (Fsq0-Fsq)<tau*(Fsq0-0.5*norm(faug+alfa*Pf)^2)
  if sec == 0
	FDIFF = F2-F0;
	v2 = FDIFF-v1;
	[alfa] = parrm(v0, v1, v2,alfa);
  	xny=x+alfa*p;
	[X1,Y1,X2,Y2,X3,Y3]=unpack3(xny,W1,B1,W2,B2,W3,B3);
	[A1,A2,A3]=simuff(P,X1,Y1,f1,X2,Y2,f2,X3,Y3,f3);
	ny=A3-T;
	fnew=ny;
	fny=ny(:);
  	Fsq=0.5*(sumsqr(fny)+mu^2*(xny-xc)'*(xny-xc));
  	i=i+1;
	alf1 = alfa;
	F1 = [fny;mu*(xny-xc)];
  end
  if sec == 1
	U1 = (F1-F0) / alf1;
	U2 = ((F2-F1)/(alf2-alf1) - (F1-F0)/alf1) / alf2;
	v1 = U1-alf1*U2;
	v2 = U2;
	[alfa] = parrm(v0, v1, v2,alfa);
  	xny=x+alfa*p;
	[X1,Y1,X2,Y2,X3,Y3]=unpack3(xny,W1,B1,W2,B2,W3,B3);
	[A1,A2,A3]=simuff(P,X1,Y1,f1,X2,Y2,f2,X3,Y3,f3);
	ny=A3-T;
	fnew=ny;
	fny=ny(:);
  	Fsq=0.5*(sumsqr(fny)+mu^2*(xny-xc)'*(xny-xc));
	alf2 = alf1;
	F2 = F1;
	alf1 = alfa;
	F1 = [fny;mu*(xny-xc)];
	i = i + 1;
  end
  sec = 1;
end
eval=i;
stplen=alfa;