source: distools/crossvald.m @ 80

%CROSSVALD Cross-validation error for dissimilarity representations
% 
%   [ERR,STD_ERR] = CROSSVALD(D,CLASSF,N,K,ITER,FID)
%   [ERR,CERR,NLAB_OUT] = CROSSVALD(D,CLASSF,N,K,1,FID)
%
% INPUT
%   A          Input dataset
%   CLASSF     Untrained classifier to be tested.
%   N          Number of dataset divisions (default: N==number of
%              samples, leave-one-out)
%   K          Desired size of the representation set (default: use all)
%   ITER       Number of iterations
%   FID        File descriptor for progress report file (default: 0)
%
% OUTPUT
%   ERR        Average test error weighted by class priors.
%   CERR       Unweighted test errors per class
%   STD_ERR    Standard deviation in the error
%   NLAB_OUT   Assigned numeric labels
%
% DESCRIPTION
% Cross-validation estimation of the error and the instability of the
% untrained classifier CLASSF using the dissimilarity dataset D. The set is 
% randomly permutated and divided in N (almost) equally sized parts. Note that
% for a dissimilarity matrix, the division has to be applied both to rows and
% to columns. The classifier is trained on N-1 parts and the remaining part is 
% used for testing. This is rotated over all parts. 
%
% ERR is their weighted avarage over the class priors. CERR are the class error 
% frequencies.  D and/or CLASSF may be cell arrays of datasets and classifiers. 
% In that case ERR is an array with errors with on position ERR(i,j) the error of 
% the j-th classifier for the i-th dataset. In this mode, CERR and NLAB_OUT are 
% returned in cell arrays.
%
% If ITER > 1 the routine is run ITER times and results are averaged. The
% standard deviation of the error is returned in STD_ERR.
%
% NOTE
% D is a square dissimilarity matrix for which the representation set has to be 
% reduced to a K-element subset from the training set. This is done by random 
% selection. If K is not chosen the entire training set is used. 
% 
% Progress is reported in file FID, default FID=0: no report.  Use FID=1
% for report in the command window.
% 
% EXAMPLE
% A = GENDATB(100);
% D = SQRT(DISTM(A));
% [E,S] = CROSSVALD (D, ldc([],1e-2,1e-6)*LOGDENS, 10, [], 5);
%
% SEE ALSO
% DATASETS, MAPPINGS, TESTC

% Copyright: R.P.W. Duin, r.duin@ieee.org
% and Elzbieta Pekalska, ela.pekalska@googlemail.com
% Faculty EWI, Delft University of Technology and
% School of Computer Science, University of Manchester


function [err,cerr,nlabout] = crossvald(data,classf,n,kred,iter,fid)

        prtrace(mfilename);

        if nargin < 6, fid = []; end
        if nargin < 5 | isempty(iter), iter = 1; end
        if nargin < 4 | isempty(kred), kred = []; end
        if nargin < 3, n = []; end
    
        if iter ~= 1
                eer = cell(1,iter);
    s = sprintf('crossvald, %i iterations: ',iter);
    prwaitbar(iter,s);
                for j = 1:iter
      prwaitbar(iter,j,[s num2str(j)]);
                        eer{j} = feval(mfilename,data,classf,n,kred,1,fid);
                end
    prwaitbar(0);
                fe = zeros(size(eer{1},1),size(eer{1},2),iter);
                for j=1:iter
                        fe(:,:,j) = eer{j};
                end
                err = mean(fe,3);
                std_err = std(fe,[],3);
                cerr = std_err;
                return
        end
        
                        
        % datasets or classifiers are cell arrays
        if iscell(classf) | iscell(data)

                seed = rand('state');
                if ~iscell(classf), classf = {classf}; end
                if ~iscell(data), data = {data}; end
                if isdataset(classf{1}) & ismapping(data{1}) % correct for old order
                        dd = data; data = classf; classf = dd;
                end
                numc = length(classf);
                numd = length(data);
                cerr = cell(numd,numc);
                nlab_out = cell(numd,numc);

                prprogress(fid,['\n%i-fold crossvalidation: ' ...
                             '%i classifiers, %i datasets\n'],n,numc,numd);

                e = zeros(numd,numc);
    if numc > 1
      snumc = sprintf('crossvald: %i classifiers: ',numc);
      prwaitbar(iter,snumc);
    end
        
                for jc = 1:numc

      if numc > 1, prwaitbar(numc,jc,[snumc num2str(jc)]); end
                        prprogress(fid,'  %s\n',getname(classf{jc}));
      if numd > 1
        snumd = sprintf('crossvald: %i datasets: ',numd);
        prwaitbar(iter,snumd);
      end
            
                        for jd = 1:numd

        if numd > 1, prwaitbar(numd,jd,[snumd num2str(jd)]); end
                                prprogress(fid,'    %s',getname(data{jd}));

                                rand('state',seed);
                                [ee,cc,nn] = feval(mfilename,data{jd},classf{jc},n,kred,1,fid);
                                e(jd,jc) = ee;
                                cerr(jd,jc) = {cc};
                                nlabout(jd,jc) = {nn};

                        end
      if numd > 1, prwaitbar(0); end
                        %fprintf(fid,'\n');

                end
                if nargout == 0
                        fprintf('\n  %i-fold cross validation result for',n);
                        disperror(data,classf,e);
                else
                        err = e;
                end
    if numc > 1, prwaitbar(0); end
    
        else

                if isdataset(classf) & ismapping(data) % correct for old order
                        dd = data; data = classf; classf = dd;
                end
   % discheck(data,[],0);
                isdataset(data);
                ismapping(classf);      
                [m,k,c] = getsize(data);
                lab = getlab(data);
                if isempty(n), n = m; end

                if n > m
                        warning('Number of batches too large: reset to leave-one-out')
                        n = m;
                elseif n <= 1
                        error('Wrong size for number of batches')
                end
                if ~isuntrained(classf)
                        error('Classifier should be untrained')
                end
                J = randperm(m);
                N = classsizes(data);

                % attempt to find an more equal distribution over the classes
                if all(N > n)

                        K = zeros(1,m);

                        for i = 1:length(N)

                                L = findnlab(data(J,:),i);

                                M = mod(0:N(i)-1,n)+1;

                                K(L) = M;

                        end

                else
            
                        K = mod(1:m,n)+1;

                end
        
                nlabout = zeros(m,1);
                rstate2 = rand('state');
    prprogress(fid,'%5.0f      ',n);
    sfolds = sprintf('crossvald: %i folds: ',n);
    prwaitbar(n,sfolds);
                for i = 1:n
      prwaitbar(n,i,[sfolds num2str(i)]);
                        OUT = find(K==i);
                        JOUT=J(OUT);
                        JIN = J; JIN(OUT) = [];
                        if ~isempty(kred)
                                if length(JIN) < kred
                                        error('Training set too small for desired size representation set.')
                                end
                                rstate1 = rand('state');
                                rand('state',rstate2);
                                RED     = randperm(length(JIN));
                                rstate2 = rand('state');
                                rand('state',rstate1);
                                RED  = RED(1:kred);  % Here we reduce the repr. set but dont take care of an equal distribution over classes
                                JINF = JIN(RED);
                                JINR = JINF;
                        else
                                JINF = JIN;
                        end
                        if (iscell(classf.data) & length(classf.data) > 0 & isparallel(classf.data{1}))
                                m = size(data,1);
                                k = size(data,2)/m;
                                if (k ~= floor(k))
                                        error('Dataset should be a concatenation of square matrices.')
                                end
                                JINF = repmat(JINF(:),1,k) + repmat([0:k-1]*m,kred,1);
                                JINF = JINF(:);
                        end
                        dlearn = data(JIN,JINF); 
                        dtest  = data(JOUT,JINF);
                        w      = dlearn*classf;   % Training
                                                  % Testing
                        [mx,nlabout(JOUT)] = max(+(dtest*w),[],2);
             % nlabout contains class assignments according to w
                                                 % needs conversion to nlabs
                        L1 = matchlablist(getlablist(data),getlabels(w));
                        [LL,L2] = sort(L1);
                        nlabout(JOUT) = L2(nlabout(JOUT));                                                                                                                                                      
                        s = sprintf('\b\b\b\b\b%5.0f',i);
                        prprogress(fid,s);
                end
    prwaitbar(0)
    prprogress(fid,'\b\b\b\b\b\b\b\b\b\b\b',n);
    %L = matchlablist(getlablist(data),getlabels(w));
                for j=1:c
                        J = findnlab(data,j);
                        f(j) = sum(nlabout(J)~=j)/length(J);
                end
                e = f*getprior(data)';
                if nargout > 0
                        err  = e;
                        cerr = f;
                else
                        disp([num2str(n) '-fold cross validation error on ' num2str(size(data,1)) ' objects: ' num2str(e)])
                end
        end
        return