test run tests

function test_suite = test_run_tests
    % tests for cosmo_run_tests
    %
    % #   For CoSMoMVPA's copyright information and license terms,   #
    % #   see the COPYING file distributed with CoSMoMVPA.           #
    try % assignment of 'localfunctions' is necessary in Matlab >= 2016
        test_functions = localfunctions();
    catch % no problem; early Matlab versions can use initTestSuite fine
    end
    initTestSuite;

function test_run_tests_passing
    test_content = 'assert(true);';
    [result, output] = helper_test_make_and_run(test_content);
    assertTrue(result);
    assert(~isempty(strfind(output, 'OK')));

function test_run_tests_failing
    test_content = 'assert(false);';
    [result, output] = helper_test_make_and_run(test_content);
    assertFalse(result);
    assert(~isempty(strfind(output, 'FAILED')));

function test_run_tests_no_file_found_absolute_path()
    if skip_test_if_octave_package_io_2_4_2_or_later()
        return
    end

    cosmo_test_dir = fileparts(mfilename('fullpath'));
    fn = cosmo_make_temp_filename(fullfile(cosmo_test_dir, 'test_'), '.m');
    assertExceptionThrown(@()helper_run_tests({fn}), '');

function test_run_tests_no_file_found_relative_path()
    if skip_test_if_octave_package_io_2_4_2_or_later()
        return
    end

    fn = cosmo_make_temp_filename('test_', '.m');
    assertExceptionThrown(@()helper_run_tests({fn}), '');

function test_run_tests_missing_logfile_argument()
    if skip_test_if_octave_package_io_2_4_2_or_later()
        return
    end

    assertExceptionThrown(@()helper_run_tests({'-logfile'}), '');

function [result, output] = helper_test_make_and_run(test_content)
    cosmo_test_dir = fileparts(mfilename('fullpath'));

    fn = cosmo_make_temp_filename(fullfile(cosmo_test_dir, 'test_'), '.m');

    [unused, test_name] = fileparts(fn);

    fid = fopen(fn, 'w');
    cleaner = onCleanup(@()delete(fn));
    fprintf(fid, ...
            ['function test_suite=%s\n'...
             'try \n'...
             '    test_functions=localfunctions();\n'...
             'catch\n'...
             'end\n'...
             'initTestSuite;\n'...
             'function %s_generated\n'...
             '%s'], ...
            test_name, test_name, test_content);
    fclose(fid);

    args = {fn};
    [result, output] = helper_run_tests(args);

function [result, output] = helper_run_tests(args)
    warning_state = cosmo_warning();
    orig_path = path();
    orig_pwd = pwd();
    log_fn = tempname();

    path_cleaner = onCleanup(@()path(orig_path));
    warning_cleaner = onCleanup(@()cosmo_warning(warning_state));
    pwd_cleaner = onCleanup(@()cd(orig_pwd));
    log_fn_cleaner = onCleanup(@()delete_if_exists(log_fn));

    % ensure path is set; disable warnings by cosmo_set_path
    cosmo_warning('off');

    more_args = {'-verbose', '-logfile', log_fn, '-no_doc_test'};
    result = cosmo_run_tests(more_args{:}, args{:});
    fid = fopen(log_fn);
    file_closer = onCleanup(@()fclose(fid));

    output = fread(fid, Inf, 'char=>char')';

function delete_if_exists(fn)
    if exist(fn, 'file')
        delete(fn);
    end

function is_2_4_2_or_later = skip_test_if_octave_package_io_2_4_2_or_later()
    % July 2016:
    % Octave package 'io' version 2.4.2 gave the following error with
    % three tests:
    % - test_run_tests_no_file_found_absolute_path
    % - test_run_tests_no_file_found_relative_path
    % - test_run_tests_missing_logfile_argument
    %
    %         failure: '__octave_config_info__' undefined near line 1 column 1
    %   __init_io__:30 (/Users/nick/octave/io-2.4.2/__init_io__.m)
    %   /Users/nick/octave/io-2.4.2/PKG_ADD:2 (/Users/nick/octave/io-2.4.2/PKG_ADD)
    %
    % These tests are disabled for now when using Octave and io 2.4.2.
    % (Version 2.4.0 seems to work fine)
    %
    % Update Sept 2016: same issue with io 2.4.3. Updated code to take any
    % version from 2.4.2 or later

    is_2_4_2_or_later = false;

    if ~cosmo_wtf('is_octave')
        return
    end

    pkgs = pkg('list', 'io');
    if numel(pkgs) ~= 1
        return
    end

    version = pkgs{1}.version;
    elem_str = regexp(version, '(\d+)\.(\d+)\.(\d+)', 'tokens', 'once');
    elem = cellfun(@str2num, elem_str);

    is_2_4_2_or_later = elem(1) >= 2 || ...
                        (elem(1) == 2 && elem(2) >= 4) || ...
                        (elem(1) == 2 && elem(2) == 4 && elem(3) >= 2);

    if ~is_2_4_2_or_later
        return
    end

    reason = ['Octave io package 2.4.2 gives unexpected error '...
              '"''__octave_config_info__'' undefined" in '...
              '__init_io__.m:30; therefore '...
              'the tests causing this error are temporarily disabled'];
    cosmo_notify_test_skipped(reason);