cosmo normalizeΒΆ

function [ds, params]=cosmo_normalize(ds, params, dim)
% normalize dataset either by estimating or applying estimated parameters
%
% [ds, est_params]=cosmo_normalize(ds, norm_type[, dim])
%
% Inputs
%   ds            a dataset struct with field .samples of size PxQ, or a
%                 numeric array of that size
%   params        either the type of normalization:
%                   - 'demean'     (mean of zero)
%                   - 'zscore'     (demean and unit variance)
%                   - 'scale_unit' (scale to interval [-1,1])
%                 -or-
%                 previously estimated parameters using the 'params'
%                 output result from a previous call to this function.
%   dim           1 or 2, indicating along with dimension of ds to
%                 normalize (if params it not a string and ends with '1' or
%                 2').
%
% Output
%   ds_norm       a dataset struct similar to ds, but with .samples data
%                 normalized. If the input was a numeric array then ds_norm
%                 is a numeric array as well.
%   params        estimated parameters for normalization. These can be
%                 re-used for a second normalization step of an independant
%                 dataset. For example, parameters can be estimated from a
%                 training dataset and then applied to a testing dataset
%
%
% Examples:
%     ds=struct();
%     ds.samples=reshape(1:15,5,3)*2;
%     cosmo_disp(ds);
%     %|| .samples
%     %||   [  2        12        22
%     %||      4        14        24
%     %||      6        16        26
%     %||      8        18        28
%     %||     10        20        30 ]
%     %
%     % demean along first dimension
%     dsn=cosmo_normalize(ds,'demean',1);
%     cosmo_disp(dsn);
%     %|| .samples
%     %||   [ -4        -4        -4
%     %||     -2        -2        -2
%     %||      0         0         0
%     %||      2         2         2
%     %||      4         4         4 ]
%     %
%     % demean along second dimension
%     dsn=cosmo_normalize(ds,'demean',2);
%     cosmo_disp(dsn);
%     %|| .samples
%     %||   [ -10         0        10
%     %||     -10         0        10
%     %||     -10         0        10
%     %||     -10         0        10
%     %||     -10         0        10 ]
%     %
%     % scale to range [-1,1] along first dimension
%     dsn=cosmo_normalize(ds,'scale_unit',1);
%     cosmo_disp(dsn);
%     %|| .samples
%     %||   [   -1        -1        -1
%     %||     -0.5      -0.5      -0.5
%     %||        0         0         0
%     %||      0.5       0.5       0.5
%     %||        1         1         1 ]
%     %
%     % z-score along first dimension
%     dsn=cosmo_normalize(ds,'zscore',1);
%     cosmo_disp(dsn);
%     %|| .samples
%     %||   [  -1.26     -1.26     -1.26
%     %||     -0.632    -0.632    -0.632
%     %||          0         0         0
%     %||      0.632     0.632     0.632
%     %||       1.26      1.26      1.26 ]
%     %
%     % z-score along second dimension
%     dsn=cosmo_normalize(ds,'zscore',2);
%     cosmo_disp(dsn);
%     %|| .samples
%     %||   [ -1         0         1
%     %||     -1         0         1
%     %||     -1         0         1
%     %||     -1         0         1
%     %||     -1         0         1 ]
%     %
%     % use samples 1, 3, and 4 to estimate parameters ('training set'),
%     % and apply these to samples 2 and 5
%     ds_train=cosmo_slice(ds,[1 3 4]);
%     ds_test=cosmo_slice(ds,[2 5]);
%     [dsn_train,params]=cosmo_normalize(ds_train,'scale_unit', 1);
%     cosmo_disp(dsn_train);
%     %|| .samples
%     %||   [    -1        -1        -1
%     %||     0.333     0.333     0.333
%     %||         1         1         1 ]
%     %
%     % show estimated parameters (min and max for each column, in this
%     % case)
%     cosmo_disp(params);
%     %|| .method
%     %||   'scale_unit'
%     %|| .dim
%     %||   [ 1 ]
%     %|| .min
%     %||   [ 2        12        22 ]
%     %|| .max
%     %||   [ 8        18        28 ]
%     %
%     % apply parameters to test dataset
%     dsn_test=cosmo_normalize(ds_test,params);
%     cosmo_disp(dsn_test);
%     %|| .samples
%     %||   [ -0.333    -0.333    -0.333
%     %||       1.67      1.67      1.67 ]
%
% #   For CoSMoMVPA's copyright information and license terms,   #
% #   see the COPYING file distributed with CoSMoMVPA.           #

if isempty(params) || (ischar(params) && strcmp(params,'none'))
    return;
end

if nargin<3, dim=[]; end

apply_params=isstruct(params);
if apply_params;
    if ~isempty(dim) && params.dim~=dim
        error('Dim specified as %d, but estimates used %d',dim,params.dim);
    end
elseif ischar(params)
    params=get_params_from_string(params,dim);
else
    error('norm_spec must be struct or string');
end

method=params.method;
dim=params.dim;


is_ds=isstruct(ds) && isfield(ds,'samples');

if is_ds
    samples=ds.samples;
else
    samples=ds;
end

switch method
    case 'demean'
        if apply_params
            mu=params.mu;
        else
            mu=mean(samples,dim);
            params.mu=mu;
        end
        samples=bsxfun(@minus,samples,mu);

    case 'zscore'
        if apply_params
            mu=params.mu;
            sigma=params.sigma;
        else
            n=size(samples,dim);
            mu=sum(samples,dim)/n;
            params.mu=mu;
        end

        d=bsxfun(@minus,samples,mu);

        if ~apply_params
            sigma=sqrt(sum(d.^2,dim)/(n-1));
            params.sigma=sigma;
        end

        if isempty(sigma)
            % Octave 3.8.2 on Ubuntu gives it the wrong size
            sigma=zeros(size(d));
        end

        % In Octave, std can give non-NaN even with NaN input
        nan_mask=any(isnan(samples),dim);
        sigma(nan_mask)=NaN;

        samples=bsxfun(@rdivide,d,sigma);

    case 'scale_unit'
        if apply_params
            min_=params.min;
            max_=params.max;
        else
            min_=min(samples,[],dim);
            max_=max(samples,[],dim);
            params.min=min_;
            params.max=max_;

        end
        samples=bsxfun(@times,1./(max_-min_),...
                            (bsxfun(@minus,samples,min_)))*2-1;

    otherwise
        error('Unsupported normalization: %s', method);

end

nan_msk=~isfinite(samples);
if any(nan_msk(:))
    cosmo_warning('%d samples are NaN or Inf after normalization', ...
                        sum(nan_msk(:)));
end

if is_ds
    ds.samples=samples;
else
    ds=samples;
end

function params=get_params_from_string(norm_method,dim_arg)
    persistent cached_params;
    persistent cached_norm_method;
    persistent cached_dim_arg;


    if ~(isequal(norm_method, cached_norm_method) && ...
            isequal(dim_arg,cached_dim_arg))

        assert(ischar(norm_method));
        assert(numel(norm_method)>0);

        dim=[];
        method=norm_method;

        if isempty(dim_arg)
            if isempty(dim)
                dim=1;
            end
        else
            if isempty(dim)
                dim=dim_arg;
            else
                error(['Cannot have third argument when second argument '...
                            'ends with a number']);
            end
        end

        cached_params=struct();
        cached_params.method=method;
        cached_params.dim=dim;

        cached_norm_method=norm_method;
        cached_dim_arg=dim_arg;
    end

    params=cached_params;