function is_ok=cosmo_check_partitions(partitions, ds, varargin)
% check whether partitions are balanced and not double-dippy
%
% cosmo_check_partitions(partitions, ds, varargin)
%
% Inputs:
% partitions struct with partitions, e.g. from
% cosmo_{nfold,oddeven,nchoosek}_partitioner
% ds dataset struct with fields .sa.{targets,chunks}
% opt (optional) struct with optional field:
% .unbalanced_partitions_ok if set to true, then unbalanced
% partitions (with a different number of
% targets of each class in a chunk) is ok
% Output:
% is_ok boolean, true if partitions are ok
%
% Throws:
% - an error if partitions are double dippy or (unless specified in opt)
% not balanced
%
% Examples:
% ds=struct();
% ds.samples=zeros(9,2);
% ds.sa.targets=[1 1 2 2 2 3 3 3 3]';
% ds.sa.chunks=[1 2 2 1 1 1 2 2 2]';
% % make unbalanced partitions
% partitions=cosmo_nfold_partitioner(ds);
% cosmo_check_partitions(partitions,ds);
% %|| error('Unbalance in partition 1 [...]')
% %
% % disable unbalanced check
% cosmo_check_partitions(partitions,ds,'unbalanced_partitions_ok',true)
% %|| true
% %
% % balance partitions and check without unbalanced check
% partitions=cosmo_balance_partitions(partitions,ds);
% cosmo_check_partitions(partitions,ds,'unbalanced_partitions_ok',false)
% %|| true
% %
% % make the partitions double dippy
% partitions.train_indices{1}=partitions.test_indices{1};
% cosmo_check_partitions(partitions,ds,'unbalanced_partitions_ok',true)
% %|| error('double dipping in fold 1: chunk 1 is in train and test set')
% %
% % make partitions empty
% partitions.train_indices{1}=[];
% cosmo_check_partitions(partitions,ds);
% %|| error('partition 1: .train_indices are empty')
% %
% % partitions have values outside range
% partitions.train_indices{1}=100;
% cosmo_check_partitions(partitions,ds);
% %|| error('partition 1: .train_indices are outside range 1..9');
% %
% % use non-integers
% partitions.train_indices{1}=1.5;
% cosmo_check_partitions(partitions,ds);
% %|| error('partition 1: .train_indices are not integers');
%
% Notes:
% - the reason to require balancing by default is that chance level is
% 1/nclasses, which is useful for many subsequent analyses.
% - if this function raises an exception for partitions, consider running
% partitions=cosmo_balance_partitions(partitions,...).
%
% See also: cosmo_balance_partitions, cosmo_nfold_partitioner
%
% # For CoSMoMVPA's copyright information and license terms, #
% # see the COPYING file distributed with CoSMoMVPA. #
% process input arguments
defaults=struct();
defaults.unbalanced_partitions_ok=false;
params=cosmo_structjoin(defaults,varargin{:});
% whether to check for equal number of samples of each class in
% each chunks
check_balance=~params.unbalanced_partitions_ok;
% whether to check for the same chunk in train and test set
check_double_dipping=true;
% check dataset
check_dataset(ds);
% ensure it has targets and chunks
cosmo_isfield(ds,{'sa.targets','sa.chunks'},true);
targets=ds.sa.targets;
chunks=ds.sa.chunks;
if check_balance
[classes,unused,sample2class]=unique(targets);
end
% must have .train_indices and .test_indices
cosmo_isfield(partitions,{'train_indices','test_indices'},true);
train_indices=partitions.train_indices;
test_indices=partitions.test_indices;
if ~iscell(train_indices) || ~iscell(test_indices)
error('.train_indices and .test_indices must be a cell');
end
% ensure equal number of partitions for train and test
npartitions=numel(train_indices);
if npartitions~=numel(test_indices)
error('Partition count mismatch for train and test: %d ~= %d',...
npartitions,numel(test_indices));
end
nsamples=numel(targets);
unsorted_train_test_fold=false(npartitions,2);
for k=1:npartitions
train_idxs=train_indices{k};
test_idxs=test_indices{k};
check_range(train_idxs,nsamples,k,'train');
check_range(test_idxs,nsamples,k,'test');
if ~issorted(train_idxs)
unsorted_train_test_fold(k,1)=true;
end
if ~issorted(test_idxs)
unsorted_train_test_fold(k,2)=true;
end
if check_balance
% counts of number of samples in each each class must be the
% same
train_classes=sample2class(train_idxs);
h=histc(train_classes,1:numel(classes));
h_nonzero_idxs=find(h>0);
if k==1
first_h_nonzero_idxs=h_nonzero_idxs;
elseif ~isequal(first_h_nonzero_idxs,h_nonzero_idxs)
error(['Different targets used for training '...
'in partition %d and %d. This is weird. '...
'Consider the following scenarios:\n'...
'(1) You made the partitions manually. It is '...
'possible that you made a mistake.\n'...
'(2) You try to do cross-decoding and '...
'partitions were defined using '...
'cosmo_nchoosek_partitioner. This usually '...
'requires a *re-assignment* of .sa.targets '...
'so that the unique targets values are the '...
'same for the samples used for training '...
'and for testing. Please read the '...
'documentation of cosmo_nchoosek_partitioner '...
'(especially the examples) carefully and '...
'verify that '...
'the .sa.targets are (re)assigned properly.\n'...
'(3) You used a cosmo_ function to set the '...
'partitions, but case 2 does not apply. '...
'It may indicate a bug; in that case, '...
'please get in touch with the CoSMoMVPA '...
'developers'],1,k);
end
h_nonzero=h(h_nonzero_idxs);
if ~all(h_nonzero(1)==h_nonzero)
idx=find(h_nonzero(1)~=h_nonzero,1);
pos_first=h_nonzero_idxs(1);
pos_other=h_nonzero_idxs(idx);
error(['Unbalance in partition %d, '...
'classes %d (#=%d) and %d (#=%d). '...
'Consider the following scenarios:\n'...
'(1) the input is an MEEG dataset, or an fMRI '...
'dataset with missing samples (trials) in some '...
'of the runs. You probably want to use '...
'cosmo_balance_partitions.\n'...
'(2) all chunks are unique; this is either a (a) '...
'between-participants design and you '...
'try to discriminate between participants (e.g. '...
'patients versus controls); or this is an MEEG '...
'dataset where all samples are assumed to be '...
'independent. You probably want to use '...
'cosmo_independent_samples_partitioner to define '...
'the partitions\n'...
'(3) the input is an fMRI dataset using beta '...
'estimates or t-statistics estimated using the '...
'GLM, with one sample '...
'of each condition per run (e.g. nfold partition) '...
'or per set of runs (e.g. odd-even partition). '...
'Probably a mistake was made setting .sa.chunks '...
'or .sa.targets\n'...
'(4) you *really* know what you are doing: '...
'as a litmus test, you would be comfortable '...
'implementing a bootstrapping algorithm to '...
'estimate the cdf of your measure of interest '...
'under some null hypothesis. You can set '...
'unbalanced_partitions_ok to true as an option'],...
k, classes(pos_first), h(pos_first), ...
classes(pos_other), h(pos_other));
end
end
if check_double_dipping
% no sample allowed to be both in train and test indices
if any(cosmo_match(chunks(train_idxs),chunks(test_idxs)))
ctrain=chunks(train_idxs);
m=cosmo_match(ctrain,chunks(test_idxs));
idx=find(m,1);
error(['double dipping in fold %d: chunk %d is in '...
'train and test set'],k,ctrain(idx));
end
end
end
has_unsorted_indices=any(unsorted_train_test_fold(:));
if has_unsorted_indices
fns={'train_indices','test_indices'};
msg_parts={'','',['Unsorted partitions can lead to unintuitive '...
'ordering of results (e.g. when using fold_predictions)']};
for k=1:2
fn=fns{k};
folds=find(unsorted_train_test_fold(:,k));
if ~isempty(folds)
msg_parts{k}=sprintf('Unsorted %s in fold(s):%s',...
fn,sprintf(' %d',folds));
end
end
msk=~cellfun(@isempty,msg_parts);
msg=cosmo_strjoin(msg_parts(msk),'\n');
cosmo_warning(msg);
end
is_ok=true;
function check_range(idxs,nsamples,partition,label)
msg='';
if isempty(idxs)
msg='empty';
elseif ~isequal(idxs,round(idxs))
msg='not integers';
elseif min(idxs)<1 || max(idxs)>nsamples
msg=sprintf('outside range 1:%d',nsamples);
end
if ~isempty(msg)
error('partition %d: .%s_indices are %s',partition,label,msg);
end
function check_dataset(ds)
persistent cached_sa;
if isstruct(ds) && isfield(ds,'sa')
if ~isequal(ds.sa,cached_sa)
cosmo_check_dataset(ds);
cached_sa=ds.sa;
end
return
end
error('second input must be a dataset struct with field .sa');
