run cross validation skl

%% Cross validation example with multiple classifiers
% This example runs cross validation and shows confusion matrix using
% multiple classifiers
%
% #   For CoSMoMVPA's copyright information and license terms,   #
% #   see the COPYING file distributed with CoSMoMVPA.           #

%% Define data
config=cosmo_config();
data_path=fullfile(config.tutorial_data_path,'ak6','s01');

data_fn=fullfile(data_path,'glm_T_stats_perrun.nii');
mask_fn=fullfile(data_path,'vt_mask.nii');
ds=cosmo_fmri_dataset(data_fn,'mask',mask_fn,...
                        'targets',repmat(1:6,1,10),...
                        'chunks',floor(((1:60)-1)/6)+1);

% remove constant features
ds=cosmo_remove_useless_data(ds);

%% Define classifiers in a cell
%%%% >>> Your code here <<< %%%%
nclassifiers=numel(classifiers);

%% Define partitions
partitions=cosmo_nfold_partitioner(ds);

%% Run classifications
% Compute the accuracy and predictions for each classifier, and plot the
% confusion matrix
for k=1:nclassifiers
    classifier=classifiers{k};

    % get predictions for each fold, and store the result in
    % a variable 'pred'.
    % This output is in a 60 x 10 matrix, corresponding to predictions
    % for all 60 samples and 10 folds
    %%%% >>> Your code here <<< %%%%

    % use cosmo_confusion_matrix to compute the confusion matrix for each
    % fold, and store the results in a variable 'confusion_matrix_folds'.
    % The output is 6 x 6 x 10, consisting of 10 confusion matrices each
    % of size 6 x 6
    %%%% >>> Your code here <<< %%%%

    % sum the confusion matrix_folds alnog the third dimension to
    % obtain a single 6 x 6 confusion matirx. Store the result
    % in a variable named confusion_matrix
    %%%% >>> Your code here <<< %%%%
    % show the confusion matrix
    figure();
    imagesc(confusion_matrix,[0 10])
    accuracy=sum(diag(confusion_matrix))/sum(confusion_matrix(:));
    title(sprintf('%s: %.3f', strrep(func2str(classifier),'_',' '), accuracy))

end

%% Consider effect or normalization
normalizations={'zscore',...
                'demean',...
                'scale_unit',...
                'none'};

for k=1:nclassifiers
    classifier=classifiers{k};
    classifier_name=strrep(func2str(classifier),'_',' ');
    for j=1:numel(normalizations)
        opt=struct();
        opt.normalization=normalizations{j};
        [pred,accuracy]=cosmo_crossvalidate(ds,classifier,partitions,opt);

        fprintf('%s with %s-normalization: %.3f\n', classifier_name,...
                opt.normalization, accuracy);
    end
end