function test_suite = test_balance_partitions
% tests for cosmo_balance_partitions
%
% # For CoSMoMVPA's copyright information and license terms, #
% # see the COPYING file distributed with CoSMoMVPA. #
try % assignment of 'localfunctions' is necessary in Matlab >= 2016
test_functions = localfunctions();
catch % no problem; early Matlab versions can use initTestSuite fine
end
initTestSuite;
function test_balance_partitions_repeats
nchunks = 5;
nsamples = 200;
nclasses = 4;
[p, ds] = get_sample_data(nsamples, nchunks, nclasses);
opt_cell = {{ {'balance_test', false}, ...
{'balance_test', true}, ...
{} ...
}, ...
{ {'nrepeats', 1}, ...
{'nrepeats', 5}, ...
{} ...
}, ...
{ {'opt_as_struct', false}, ...
{'opt_as_struct', true} ...
}};
opt_prod = cosmo_cartprod(opt_cell);
defaults = struct();
defaults.nrepeats = 1;
defaults.balance_test = true;
for k = 1:size(opt_prod, 1)
opt = opt_prod(k, :);
opt_struct = cosmo_structjoin(opt);
opt_as_struct = opt_struct.opt_as_struct;
opt_struct = rmfield(opt_struct, 'opt_as_struct');
if opt_as_struct
args = opt_struct;
else
args = [fieldnames(opt_struct) struct2cell(opt_struct)]';
end
b = cosmo_balance_partitions(p, ds, args);
full_args = cosmo_structjoin(defaults, args);
nrep = full_args.nrepeats;
balance_test = full_args.balance_test;
assertEqual(numel(b.train_indices), nrep * nchunks);
assertEqual(numel(b.test_indices), nrep * nchunks);
assertEqual(fieldnames(b), {'train_indices'; 'test_indices'});
nfolds = numel(p.test_indices);
for j = 1:nfolds
pi = p.train_indices{j};
pt = ds.sa.targets(pi);
for k = 1:nrep
fold_i = (j - 1) * nrep + k;
bi = b.train_indices{fold_i};
bt = ds.sa.targets(bi);
h = histc(bt, 1:nclasses)';
assertTrue(all(min(histc(pt, 1:nclasses)) == h));
end
end
assert_partitions_ok(ds, b, balance_test);
assert_balanced_partitions_subset(p, b);
end
function assert_balanced_partitions_subset(unbal_partitions, bal_partitions)
% each training and test fold in bal_partitions must correspond to
% a fold in the original partitions
nsamples = max([cellfun(@max, unbal_partitions.train_indices) ...
cellfun(@max, unbal_partitions.test_indices)]);
unbal_nfolds = numel(unbal_partitions.train_indices);
% see which indices were used in each fold
msk_train = find_member(unbal_partitions, 'train_indices', nsamples);
msk_test = find_member(unbal_partitions, 'test_indices', nsamples);
unbal_was_used = false(unbal_nfolds, 1);
bal_nfolds = numel(bal_partitions.train_indices);
for fold_i = 1:bal_nfolds
bal_train = bal_partitions.train_indices{fold_i};
bal_test = bal_partitions.test_indices{fold_i};
candidate_msk = all(msk_train(:, bal_train), 2) & ...
all(msk_test(:, bal_test), 2);
assert(any(candidate_msk));
unbal_was_used(candidate_msk) = true;
end
assertEqual(unbal_was_used, true(unbal_nfolds, 1));
function msk = find_member(partitions, label, nsamples)
folds = partitions.(label);
nfolds = numel(folds);
msk = false(nfolds, nsamples);
for k = 1:nfolds
msk(k, folds{k}) = true;
end
function assert_partitions_ok(ds, partitions, balanced_test_indices)
assertEqual(sort(fieldnames(partitions)), sort({'train_indices'; ...
'test_indices'}));
nfolds = numel(partitions.train_indices);
assertEqual(numel(partitions.test_indices), nfolds);
for fold_i = 1:nfolds
assert_fold_balanced(ds, partitions, fold_i, 'train_indices');
if balanced_test_indices
assert_fold_balanced(ds, partitions, fold_i, 'test_indices');
end
assert_fold_no_double_dipping(ds, partitions, fold_i);
assert_fold_targets_match(ds, partitions, fold_i);
assert_fold_indices_unique(partitions, fold_i);
end
function assert_fold_no_double_dipping(ds, partitions, fold)
train_indices = partitions.train_indices;
test_indices = partitions.test_indices;
train_chunks = ds.sa.chunks(train_indices{fold});
test_chunks = ds.sa.chunks(test_indices{fold});
assert(isempty(intersect(train_chunks, test_chunks)));
function assert_fold_balanced(ds, partitions, fold, label)
all_indices = partitions.(label);
indices = all_indices{fold};
unq_targets = unique(ds.sa.targets);
targets = ds.sa.targets(indices);
assertEqual(unique(targets), unq_targets);
h = histc(targets, unq_targets);
assertEqual(h(1) * ones(size(h)), h);
function assert_fold_targets_match(ds, partitions, fold)
train_indices = partitions.train_indices{fold};
test_indices = partitions.test_indices{fold};
nsamples = size(ds.samples, 1);
assert_all_int_with_max(train_indices, nsamples);
assert_all_int_with_max(test_indices, nsamples);
train_targets = ds.sa.targets(train_indices);
test_targets = ds.sa.targets(test_indices);
assertEqual(unique(train_targets), unique(test_targets));
function assert_fold_indices_unique(partitions, fold)
train_indices = partitions.train_indices{fold};
test_indices = partitions.test_indices{fold};
assert(isequal(sort(train_indices), unique(train_indices)));
assert(isequal(sort(test_indices), unique(test_indices)));
function assert_all_int_with_max(indices, max_value)
assert(min(indices) >= 1);
assert(max(indices) <= max_value);
assert(all(round(indices) == indices));
function test_balance_partitions_nmin
nchunks = 5;
nsamples = 200;
nclasses = 4;
[p, ds] = get_sample_data(nsamples, nchunks, nclasses);
nmin = 8 + round(rand() * 4);
args = struct();
args.nmin = nmin;
args.balance_test = [false, true];
arg_prod = cosmo_cartprod(args);
for arg_i = 1:numel(arg_prod)
arg = arg_prod{arg_i};
b = cosmo_balance_partitions(p, ds, arg);
counter = zeros(nsamples, nchunks);
for j = 1:numel(b.train_indices)
bi = b.train_indices{j};
bj = b.test_indices{j};
ch = unique(ds.sa.chunks(bj));
assert(numel(ch) == 1);
if arg.balance_test
% no other indices
assertEqual(setdiff(bj, p.test_indices{ch}), zeros(0, 1));
else
assertEqual(sort(bj), p.test_indices{ch});
end
bt = ds.sa.targets(bi);
h = histc(bt, 1:nclasses);
assertEqual(ones(nclasses, 1) * h(1), h);
counter(bi, ch) = counter(bi, ch) + 1;
end
for k = 1:nchunks
msk = ds.sa.chunks ~= k;
assert(min(counter(msk, k)) >= nmin);
assert(all(counter(~msk, k) == 0));
end
assert_partitions_ok(ds, b, arg.balance_test);
assert_balanced_partitions_subset(p, b);
end
function test_balance_partitions_exceptions
ds = cosmo_synthetic_dataset();
p = cosmo_nfold_partitioner(ds);
aet = @(varargin)assertExceptionThrown(@() ...
cosmo_balance_partitions(varargin{:}), '');
aet(struct, struct);
aet(ds, p); % wrong order
aet(p, ds, 'nmin', 1, 'nrepeats', 1);
% create missing class
ds.sa.targets(1) = 4;
aet(p, ds);
% missing target
p.train_indices{1} = p.train_indices{1}([1 3]);
aet(p, ds);
% double dipping
p.train_indices{1} = p.train_indices{2};
aet(p, ds);
function test_sorted_indices()
warning_state = cosmo_warning();
cleaner = onCleanup(@()cosmo_warning(warning_state));
cosmo_warning('off');
nchunks = ceil(cosmo_rand() * 10 + 10);
ntargets = ceil(cosmo_rand() * 10 + 10);
ds = cosmo_synthetic_dataset('nchunks', nchunks, 'ntargets', ntargets);
nchunks = numel(unique(ds.sa.chunks));
partitions = struct();
partitions.train_indices = cell(nchunks, 1);
partitions.test_indices = cell(nchunks, 1);
% build partitions with unsorted indices
for k = 1:nchunks
rp = cosmo_randperm(nchunks);
train_msk = (ds.sa.chunks) == rp(1) | (ds.sa.chunks) == rp(2);
train_idx = find(train_msk);
test_idx = find(~train_msk);
rp_train = cosmo_randperm(numel(train_idx));
rp_test = cosmo_randperm(numel(test_idx));
partitions.train_indices{k} = train_idx(rp_train);
partitions.test_indices{k} = test_idx(rp_test);
end
% balance partitions
bal_partitions = cosmo_balance_partitions(partitions, ds);
% all partitions must be sorted
fns = {'train_indices', 'test_indices'};
for k = 1:nchunks
for j = 1:2
fn = fns{j};
idx = partitions.(fn){k};
bal_idx = bal_partitions.(fn){k};
% indices must be the same
assertEqual(sort(idx(:)), sort(bal_idx(:)));
% balanced partitions must be sorted
assertTrue(issorted(bal_idx));
end
end
function [p, ds] = get_sample_data(nsamples, nchunks, nclasses)
ds = struct();
ds.samples = (1:nsamples)';
ds.sa.targets = ceil(cosmo_rand(nsamples, 1) * nclasses);
ds.sa.chunks = ceil(cosmo_rand(nsamples, 1) * nchunks);
p = cosmo_nfold_partitioner(ds);
