clevals.m

The pattern recognition matlab toolbox

%CLEVALS Classifier evaluation (feature size/learning curve), bootstrap possible
% 
% 	E = CLEVALS(A,CLASSF,FEATSIZE,TRAINSIZES,NREPS,T,FID)
%
% INPUT
%   A          Training dataset
%   CLASSF     Classifier to evaluate
%   FEATSIZE   Vector of feature sizes
%                (default: 1:K, where K is the number of features in A)
%   TRAINSIZES Vector of class sizes, used to generate subsets of A
% 	           	 (default [2,3,5,7,10,15,20,30,50,70,100])
%   NREPS 		 Number of repetitions (default 1)
%   T      		 Tuning set, or 'bootstrap' (default [], i.e. use remaining
%             		objects in A)
%		FID				 File ID to write progress to (default 0, i.e. no report)
%
% OUTPUT
%   E       	 Error structure (see PLOTE)
%
% DESCRIPTION 
% Generates at random, for all feature sizes defined in FEATSIZES or all
% class sizes defined in TRAINSIZES, training sets out of the dataset A and
% uses these for training the untrained classifier CLASSF. CLASSF may also
% be a cell array of untrained classifiers; in this case the routine will be
% run for all of them. The resulting trained classifiers are tested on all
% objects in A. This procedure is then repeated N times.
%
% Training set generation is done "with replacement" and such that for each
% run the larger training sets include the smaller ones and that for all
% classifiers the same training sets are used.
% 
% If CLASSF is fully deterministic, this function uses the RAND random
% generator and thereby reproduces if its seed is reset (see RAND). 
% If CLASSF uses RANDN, its seed may have to be set as well.
%
% Use FID = 1 to report progress to the command window.
% 
% EXAMPLES
% See PREX_CLEVAL.
%
% SEE ALSO
% MAPPINGS, DATASETS, CLEVALB, TESTC, PLOTE

% Copyright: R.P.W. Duin, [e-mail]
% Faculty of Applied Sciences, Delft University of Technology
% P.O. Box 5046, 2600 GA Delft, The Netherlands

% $Id: clevals.m,v 1.3 2007/02/15 10:11:15 davidt Exp $

function e = clevals(a,classf,featsizes,learnsizes,nreps,t,fid) 

	prtrace(mfilename);

  if (nargin < 7)
		fid = 0; 
	end;
	if (nargin < 6)
		prwarning(2,'no tuning set supplied, bootstrapping');
		t = [];
	end;
  if (nargin < 5)
		prwarning(2,'number of repetitions not specified, assuming NREPS = 1');
		nreps = 1; 
	end;

	% If a single mapping is given, convert it to a 1 x 1 cell array.

  if (ismapping(classf)), classf = {classf}; end

	% Correct for old argument order.

  if (isdataset(classf)) & (ismapping(a)) 
  	tmp = a; a = classf; classf = {tmp};
  end
  if (isdataset(classf)) & (iscell(a)) & (ismapping(a{1})) 
  	tmp = a; a = classf; classf = tmp;
  end
  if ~iscell(classf), classf = {classf}; end

	% Assert that all is right.
  isdataset(a); ismapping(classf{1});
	
	% Remove requested class sizes that are larger than the size of the
	% smallest class.

  mc = classsizes(a); [m,k,c] = getsize(a);

	% Defaults for size arrays.

	if (nargin < 4) 
		prwarning(2,'vector of training set class sizes not specified, assuming [2,3,5,7,10,15,20,30,50,70,100]');
 		learnsizes = [2,3,5,7,10,15,20,30,50,70,100]; 
	end;
  if (nargin < 3)
		prwarning(2,'vector of feature sizes not specified, assuming K');
 		featsizes = k;
	end;

	learncurve = 0; featcurve = 0;
	if (isempty(featsizes))									% Learning curve.
		toolarge = find(learnsizes >= min(mc));
		if (~isempty(toolarge))
			prwarning(2,['training set class sizes ' num2str(learnsizes(toolarge)) ...
									 ' larger than the minimal class size in A; removed them']);
		  learnsizes(toolarge) = [];
		end
		learnsizes = learnsizes(:)';
		featsizes  = k;
		sizes = learnsizes;
		learncurve = 1;
	else																		% Feature size curve.
		toolarge = find(featsizes > k);
		if (~isempty(toolarge))
			prwarning(2,['feature sizes ' num2str(featsizes(toolarge)) ...
									 ' larger than number of features in A; removed them']);
		  featsizes(toolarge) = [];
		end
		if (max(size(learnsizes)) > 1)
			error('For a feature size curve, specify a scalar LEARNSIZE.');
		end;
	  featsizes = featsizes(:)';
		sizes = featsizes;
		featcurve = 1;
	end;

	% Fill the error structure.

  nw = length(classf(:));
  datname = getname(a);

  e.error   = zeros(nw,length(sizes));
  e.std     = zeros(nw,length(sizes));
  e.xvalues = sizes(:)';
  e.n       = nreps;
  e.names   = [];
  if (nreps > 1)
  	e.ylabel= ['Averaged error (' num2str(nreps) ' experiments)'];
  elseif (nreps == 1)
  	e.ylabel = 'Error';
  else
		error('Number of repetitions NREPS should be >= 1.');
	end;

  if (~isempty(datname))
		if (isempty(t))
			 	e.title = ['Bootstrapped learning curve on ' datname];
		else
			 	e.title = ['Learning curve on ' datname];
	 	end
	end;

 	if (learncurve)
   	e.xlabel = 'Training set size';
 	else
 		e.xlabel = 'Feature size';
 	end;

  if (sizes(end)/sizes(1) > 20)
  	e.plot = 'semilogx'; 				% If range too large, use a log-plot for X.
  end
  
	% Report progress.

	if (isempty(t))
		if (learncurve)
		  fprintf(fid,['\nBootstrapped classifier evaluation (learning curve): \n' ...
  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));
		else
		  fprintf(fid,['\nBootstrapped classifier evaluation (feature size curve): \n' ...
  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));
		end
	else
		if (featcurve)
		  fprintf(fid,['\nClassifier evaluation (learning curve): \n' ...
  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));
		else
		  fprintf(fid,['\nClassifier evaluation (feature size curve): \n' ...
  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));
		end
	end
  fprintf(fid,' %i ',sizes)
  fprintf(fid,']\n');

	% Store the seed, to reset the random generator later for different
	% classifiers.

  seed = rand('state');

	% Loop over all classifiers (with index WI).

  for wi = 1:nw

		% Assert that CLASSF{WI} is an untrained mapping.
  	isuntrained(classf{wi});

  	name = getname(classf{wi});
    fprintf(fid,'Classifier: %s\n',name); e.names = char(e.names,name);

		% E1 will contain the error estimates.

  	e1 = zeros(nreps,length(sizes));

		% Take care that classifiers use same training set.

  	rand('state',seed); seed2 = seed;

		% For N repetitions...

  	for i = 1:nreps

			if (isempty(t))

  			% Bootstrap. Store the randomly permuted indices of samples of class
  			% CI to use in this training set in JR(CI,:).

    		JR = zeros(c,max(learnsizes));
    		for ci = 1:c
    			JC = findnlab(a,ci);
  				
  				% Necessary for reproducable training sets: set the seed and store
  				% it after generation, so that next time we will use the previous one.
    			rand('state',seed2);		

    			R = ceil(rand(1,max(learnsizes))*length(JC));
    			JR(ci,:) = JC(R)';

    			seed2 = rand('state'); 
    		end
	
				t = a;

			end

  		% Either the outer loop or the inner loop will be traversed just once,
			% depending on whether we want to find a learning curve or feature
			% size curve.

  		ii = 0;
    	for li = 1:length(learnsizes)
  			for fi = 1:length(featsizes)

  				ii = ii + 1;

    			% Ja will contain the indices for this training set, Jt for
					% the tuning set.

    			Ja = []; 
      		for ci = 1:c
      			Ja = [Ja;JR(ci,1:learnsizes(li))']; 
      		end;
					
					Jt = setdiff(1:m,Ja);
    				
    			% Train classifier CLASSF{WI} on this training set and 
  				% calculate error on tuning set.

					if (learncurve)
	        	e1(i,ii) = testc(a, ...
														 a(Ja,1:featsizes(fi))*classf{wi});
					else
	        	e1(i,ii) = testc(t(Jt,1:featsizes(fi)), ...
														 a(Ja,1:featsizes(fi))*classf{wi});
					end;

     			fprintf(fid,'.');

    		end
  		end

      fprintf(fid,'\n');

  	end

		% Calculate average error and standard deviation for this classifier
		% (or set the latter to zero if there's been just 1 repetition).

  	e.error(wi,:) = mean(e1,1);
  	if (nreps == 1)
  		e.std(wi,:) = zeros(1,size(e.std,2));
  	else
  		e.std(wi,:) = std(e1)/sqrt(nreps);
  	end

  end

  % The first element is the empty string [], remove it.
  e.names(1,:) = [];

return