/* $Id: kmo_priv_stats.c,v 1.7 2013-06-17 07:52:26 aagudo Exp $
 *
 * This file is part of the KMOS Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: aagudo $
 * $Date: 2013-06-17 07:52:26 $
 * $Revision: 1.7 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
 *                              Includes
 *----------------------------------------------------------------------------*/

#include <string.h>

#include <cpl.h>

#include "kmclipm_math.h"
#include "kmclipm_functions.h"
#include "kmclipm_vector.h"

#include "kmo_utils.h"
#include "kmo_dfs.h"
#include "kmo_error.h"
#include "kmo_priv_stats.h"
#include "kmo_cpl_extensions.h"
#include "kmo_debug.h"

/*----------------------------------------------------------------------------*/
/**
    @defgroup kmos_priv_stats     Helper functions for recipe kmo_stats.

    @{
 */
/*----------------------------------------------------------------------------*/

/**
    @brief
        Calculates the statistics of an imagelist.

    @param data     The input cube.
    @param mask     Optional: A mask.
                    (0: exclude pixel, 1: include pixel)
    @param cpos_rej The positive rejection threshold.
    @param cneg_rej The negative rejection threshold.
    @param citer    The number of iterations for rejection.

    @return         The statistics vector.
*/
kmclipm_vector* kmo_calc_stats_cube(const cpl_imagelist *data,
                                const cpl_image *mask,
                                double cpos_rej,
                                double cneg_rej,
                                int citer)
{
    int             nr_mask_pix         = 0;

    kmclipm_vector  *data_in_vec        = NULL,
                    *data_out           = NULL;

    const cpl_image *tmp_img            = NULL;

    KMO_TRY
    {
        KMO_TRY_ASSURE(data != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "No input data is provided!");

        KMO_TRY_ASSURE((cpos_rej >= 0.0) && (cneg_rej >= 0.0),
                       CPL_ERROR_ILLEGAL_INPUT,
                       "No negative thresholds!");

        KMO_TRY_ASSURE(citer >= 0,
                       CPL_ERROR_ILLEGAL_INPUT,
                       "No negative iterations!");

        KMO_TRY_EXIT_IF_NULL(
            tmp_img = cpl_imagelist_get_const(data, 0));

        if (mask != NULL) {
            KMO_TRY_ASSURE((cpl_image_get_size_x(tmp_img) ==
                                                  cpl_image_get_size_x(mask)) &&
                           (cpl_image_get_size_y(tmp_img) ==
                                                  cpl_image_get_size_y(mask)),
                           CPL_ERROR_ILLEGAL_INPUT,
                           "Data and mask haven't the same size!");
        }

        // initialise output-vector, pre-process input data-cube
        // (identify infinite values, convert to vector)
        KMO_TRY_EXIT_IF_NULL(
            data_out = kmclipm_vector_new(11));

        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_fill(data_out, -1.0));

        KMO_TRY_CHECK_ERROR_STATE();

        KMO_TRY_EXIT_IF_NULL(
            data_in_vec = kmo_imagelist_to_vector(data, mask, &nr_mask_pix));

        // 1.  Number of pixels
        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_set(data_out, 0, kmclipm_vector_get_size(data_in_vec)));

        // 2.  Number of finite pixels
        int nr_finite = kmclipm_vector_count_non_rejected(data_in_vec);
        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_set(data_out, 1, nr_finite));

        KMO_TRY_EXIT_IF_ERROR(
            kmo_calc_remaining(data_in_vec,
                               data_out,
                               cpos_rej,
                               cneg_rej,
                               citer,
                               nr_finite));
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        kmclipm_vector_delete(data_out); data_out = NULL;
    }

    kmclipm_vector_delete(data_in_vec); data_in_vec = NULL;

    return data_out;
}

/**
    @brief
        Calculates the statistics of an image.

    @param data     The input image.
    @param mask     Optional: A mask.
                    (0: exclude pixel, 1: include pixel)
    @param cpos_rej The positive rejection threshold.
    @param cneg_rej The negative rejection threshold.
    @param citer    The number of iterations for rejection.

    @return         The statistics vector.
*/
kmclipm_vector* kmo_calc_stats_img(const cpl_image *data,
                                   const cpl_image *mask,
                                   double cpos_rej,
                                   double cneg_rej,
                                   int citer)
{
    int             nr_mask_pix         = 0;

    kmclipm_vector  *data_in_vec        = NULL,
                    *data_out           = NULL;

    KMO_TRY
    {
        KMO_TRY_ASSURE(data != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "No input data is provided!");

        KMO_TRY_ASSURE((cpos_rej >= 0.0) && (cneg_rej >= 0.0),
                       CPL_ERROR_ILLEGAL_INPUT,
                       "No negative thresholds!");

        KMO_TRY_ASSURE(citer >= 0,
                       CPL_ERROR_ILLEGAL_INPUT,
                       "No negative iterations!");

        if (mask != NULL) {
            KMO_TRY_ASSURE((cpl_image_get_size_x(data) ==
                                                  cpl_image_get_size_x(mask)) &&
                           (cpl_image_get_size_y(data) ==
                                                    cpl_image_get_size_y(mask)),
                           CPL_ERROR_ILLEGAL_INPUT,
                           "Data and mask haven't the same size!");
        }

        // initialise output-vector, pre-process input data-cube
        //  (identify infinite values, convert to vector)
        KMO_TRY_EXIT_IF_NULL(
            data_out = kmclipm_vector_new(11));

        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_fill(data_out, -1.0));

        KMO_TRY_CHECK_ERROR_STATE();

        KMO_TRY_EXIT_IF_NULL(
            data_in_vec = kmo_image_to_vector(data, mask,
                                              &nr_mask_pix));

        // 1.  Number of pixels
        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_set(data_out, 0, kmclipm_vector_get_size(data_in_vec)));

        // 2.  Number of finite pixels
        int nr_finite = kmclipm_vector_count_non_rejected(data_in_vec);
        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_set(data_out, 1, nr_finite));

        KMO_TRY_EXIT_IF_ERROR(
            kmo_calc_remaining(data_in_vec,
                               data_out,
                               cpos_rej,
                               cneg_rej,
                               citer,
                               nr_finite));
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        kmclipm_vector_delete(data_out); data_out = NULL;
    }

    kmclipm_vector_delete(data_in_vec); data_in_vec = NULL;

    return data_out;
}

/**
    @brief
        Calculates the statistics of a vector.

    @param data     The input vector.
    @param cpos_rej The positive rejection threshold.
    @param cneg_rej The negative rejection threshold.
    @param citer    The number of iterations for rejection.

    @return         The statistics vector.
*/
kmclipm_vector* kmo_calc_stats_vec(kmclipm_vector *data,
                                   kmclipm_vector *mask,
                                   double cpos_rej,
                                   double cneg_rej,
                                   int citer)
{
    kmclipm_vector  *data_out           = NULL;
    int             nr_non_masked_pix   = 0,
                    rejected            = 0;
    double          val                 = 0.;
    int             i                   = 0;

    KMO_TRY
    {
        KMO_TRY_ASSURE(data != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "No input data is provided!");

        if (mask != NULL) {
            KMO_TRY_ASSURE(kmclipm_vector_get_size(data) ==
                           kmclipm_vector_get_size(mask),
                           CPL_ERROR_ILLEGAL_INPUT,
                           "Data and mask haven't the same size!");
        }

        KMO_TRY_ASSURE((cpos_rej >= 0.0) && (cneg_rej >= 0.0),
                       CPL_ERROR_ILLEGAL_INPUT,
                       "No negative thresholds!");

        KMO_TRY_ASSURE(citer >= 0,
                       CPL_ERROR_ILLEGAL_INPUT,
                       "No negative iterations!");

        // initialise output-vector, pre-process input data-vector
        // (identify infinite values)
        KMO_TRY_EXIT_IF_NULL(
            data_out = kmclipm_vector_new(11));

        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_fill(data_out, -1.0));

        // 1.  Number of pixels
        if (mask == NULL) {
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 0, kmclipm_vector_get_size(data)));
        } else {
            for (i = 0; i < kmclipm_vector_get_size(data); i++) {
                val = kmclipm_vector_get(mask, i, &rejected);
                if (!rejected && (val > 0.5)) {
                    nr_non_masked_pix++;
                }
            }

            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 0, nr_non_masked_pix));

            // synchronize rejected values
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_adapt_rejected(data, mask));
        }

        // 2.  Number of finite pixels
        if (mask == NULL) {
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 1,
                                   kmclipm_vector_count_non_rejected(data)));
        } else {
            nr_non_masked_pix = 0;
            for (i = 0; i < kmclipm_vector_get_size(data); i++) {
                val = kmclipm_vector_get(mask, i, &rejected);
                if (!rejected) {
                    if (val > 0.5) {
                        nr_non_masked_pix++;
                    } else {
                        kmclipm_vector_reject(data, i);
                    }
                }
            }
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 1, nr_non_masked_pix));
        }

        int nr_finite = kmclipm_vector_get(data_out, 1, &rejected);

        KMO_TRY_EXIT_IF_ERROR(
            kmo_calc_remaining(data,
                               data_out,
                               cpos_rej,
                               cneg_rej,
                               citer,
                               nr_finite));
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        kmclipm_vector_delete(data_out); data_out = NULL;
    }

    return data_out;
}

/**
    @brief
        Counts how many pixels are masked in @c mask.

    A pixel will be masked if the mask is smaller 0.5 at this position.

    @param mask        The mask. If it is NULL, 0 is returned.

    @return
        The number of masked pixels. In case of error -1 will be returned.

    Possible cpl_error_code set in this function:

    @li CPL_ERROR_NULL_INPUT    if @c mask is NULL.
*/
int kmo_count_masked_pixels(const cpl_image *mask)
{
    int             i       = 0,
                    j       = 0,
                    count   = 0,
                    nx      = 0;

    const float     *pmask  = NULL;

    KMO_TRY
    {
        if (mask != NULL) {
            KMO_TRY_EXIT_IF_NULL(
                pmask = cpl_image_get_data_float_const(mask));

            nx = cpl_image_get_size_x(mask);

            for (j = 0; j < cpl_image_get_size_y(mask); j++) {
                for (i = 0; i < nx; i++) {
                    if (pmask[i+j*nx] < 0.5) {
                        count++;
                    }
                }
            }
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        count = -1;
    }

    return count;
}

/**
    @brief
        Checks at which positions @c vec_in contains infinite values (nan, inf,
        -inf). The returned vector contains ones for infinite values and zeros
        for finite values. The output vector can be used as input for
        @c kmo_vector_to_vector which in turn creates a vector containing
        only the valid data of @c vec_in.

    @param vec_in      The input vector.

    @return
        A vector containing ones for infinite values and zeros for finite values.

    Possible cpl_error_code set in this function:

    @li CPL_ERROR_NULL_INPUT    if @c vec_in is NULL.
*/
cpl_vector* kmo_vector_identify_infinite(const cpl_vector *vec_in)
{
    cpl_vector      *vec        = NULL;
    int             i           = 0,
                    nx          = 0,
                    x           = 0;

    double          *pvec       = NULL;

    const double    *pvec_in    = NULL;

    KMO_TRY
    {
        KMO_TRY_ASSURE(vec_in != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "No input data is provided!");

        nx = cpl_vector_get_size(vec_in);

        KMO_TRY_EXIT_IF_NULL(
            vec = cpl_vector_new(nx));

        KMO_TRY_EXIT_IF_ERROR(
            cpl_vector_fill(vec, 0.0));

        KMO_TRY_EXIT_IF_NULL(
            pvec = cpl_vector_get_data(vec));

        KMO_TRY_EXIT_IF_NULL(
            pvec_in = cpl_vector_get_data_const(vec_in));

        /* Loop on the pixels of the data-image */
        for (i = 0; i < nx; i++) {
            if (kmclipm_is_nan_or_inf(pvec_in[i])) {
                pvec[x] = 1.0;
            }
            x++;
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        cpl_vector_delete(vec); vec = NULL;
    }

    return vec;
}

/**
    @brief
        Converts @c imglist to a vector in order to calculate mean, stddev,
        median etc. Infinite values (nan, inf, -inf) are removed, so the
        length of the vector can be shorter than the dimensions of the imagelist.

    @param imglist     The input imagelist.
    @param mask        Optional: The mask. Each spatial plane will be masked
                       with @c mask. If a pixel is 0.0 it won't be masked.
                       (0: exclude pixel, 1: include pixel)
    @param vec         Output: The output vector. Is allocated in this function.
    @param nr_mask_pix Output: The number of masked pixels in mask (when
                       mask == NULL, nr_mask_pix will be zero .

    @return
        The number of infinite pixels (In the whole data cube).

    Possible cpl_error_code set in this function:

    @li CPL_ERROR_NULL_INPUT    if @c imglist is NULL.
*/
kmclipm_vector* kmo_imagelist_to_vector(const cpl_imagelist *imglist,
                                        const cpl_image *mask,
                                        int *nr_mask_pix)
{
    const cpl_image *img        = NULL;
    int             nx          = 0,
                    ny          = 0,
                    nz          = 0,
                    x           = 0;
    const float     *pimg       = NULL,
                    *pmask      = NULL;
    kmclipm_vector  *vec        = NULL;

    KMO_TRY
    {
        KMO_TRY_ASSURE(imglist != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "Not all input data is provided!");

        KMO_TRY_EXIT_IF_NULL(
            img = cpl_imagelist_get_const(imglist, 0));

        nx = cpl_image_get_size_x(img);
        ny = cpl_image_get_size_y(img);
        nz = cpl_imagelist_get_size(imglist);

        if (mask != NULL) {
            KMO_TRY_ASSURE((nx == cpl_image_get_size_x(mask)) &&
                           (ny == cpl_image_get_size_y(mask)),
                           CPL_ERROR_ILLEGAL_INPUT,
                           "Data and mask haevn't the same size!");
        }

        // copy unmasked values from cube to vector
        *nr_mask_pix = kmo_count_masked_pixels(mask);

        KMO_TRY_EXIT_IF_NULL(
            vec = kmclipm_vector_new((nx * ny - *nr_mask_pix) * nz));

        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_fill(vec, 0.0));

        if (mask != NULL) {
            KMO_TRY_EXIT_IF_NULL(
                pmask = cpl_image_get_data_float_const(mask));
        }

        int i = 0, j = 0, g = 0;
        for (g = 0; g < nz; g++) {
            KMO_TRY_EXIT_IF_NULL(
                img = cpl_imagelist_get_const(imglist, g));

            KMO_TRY_EXIT_IF_NULL(
                pimg = cpl_image_get_data_float_const(img));
            for (j = 0; j < ny; j++) {
                for (i = 0; i < nx; i++) {
                    if ((mask == NULL) || (pmask[i+j*nx] >= 0.5)) {
                        kmclipm_vector_set(vec, x++, pimg[i+j*nx]);
                    }
                }
            }
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        kmclipm_vector_delete(vec); vec = NULL;
    }

    return vec;
}

/**
    @brief
        Converts @c img to a vector in order to calculate mean, stddev,
        median etc. Infinite values (nan, inf, -inf) are removed, so the
        length of the vector can be shorter than the dimensions of the image.

    @param img         The input image.
    @param mask        Optional: The mask. @c img will be masked
                       with @c mask. If a pixel is 0.0 it won't be masked.
                       (0: exclude pixel, 1: include pixel)
    @param vec         Output: The output vector. Is allocated in this function.
    @param nr_mask_pix Output: The number of masked pixels in mask (when
                       mask== NULL, nr_mask_pix will be zero .

    @return
        The number of infinite pixels (In the whole data image).

    Possible cpl_error_code set in this function:

    @li CPL_ERROR_NULL_INPUT    if @c img is NULL.
*/
kmclipm_vector* kmo_image_to_vector(const cpl_image *img,
                                    const cpl_image *mask,
                                    int *nr_mask_pix)
{
    int             nx          = 0,
                    ny          = 0,
                    x           = 0;
    const float     *pimg       = NULL,
                    *pmask      = NULL;
    kmclipm_vector  *vec        = NULL;

    KMO_TRY
    {
        KMO_TRY_ASSURE(img != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "Not all input data is provided!");

        nx = cpl_image_get_size_x(img);
        ny = cpl_image_get_size_y(img);

        if (mask != NULL) {
            KMO_TRY_ASSURE((nx == cpl_image_get_size_x(mask)) &&
                           (ny == cpl_image_get_size_y(mask)),
                        CPL_ERROR_ILLEGAL_INPUT,
                        "Data and mask haevn't the same size!");
        }

        *nr_mask_pix = kmo_count_masked_pixels(mask);

        KMO_TRY_EXIT_IF_NULL(
            vec = kmclipm_vector_new(nx * ny - *nr_mask_pix));

        KMO_TRY_EXIT_IF_ERROR(
            kmclipm_vector_fill(vec, 0.0));

        if (mask != NULL) {
            KMO_TRY_EXIT_IF_NULL(
                pmask = cpl_image_get_data_float_const(mask));
        }

        // Loop on the pixels of the data-image
        KMO_TRY_EXIT_IF_NULL(
            pimg = cpl_image_get_data_float_const(img));
        int i = 0, j = 0;
        for (j = 0; j < ny; j++) {
            for (i = 0; i < nx; i++) {
                if ((mask == NULL) || (pmask[i+j*nx] >= 0.5)) {
                    kmclipm_vector_set(vec, x++, pimg[i+j*nx]);
                }
            }
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        kmclipm_vector_delete(vec); vec = NULL;
    }

    return vec;
}

/**
    @brief
        Infinite values (nan, inf, -inf) are removed, so the
        length of the output vector can be shorter than of the input one.

    @param vec_in      The input vector.
    @param vec         Output: The output vector. Is allocated in this function.

    @return
        The number of infinite pixels.

    Possible cpl_error_code set in this function:

    @li CPL_ERROR_NULL_INPUT    if @c vec_in is NULL.
*/
int kmo_vector_to_vector(const cpl_vector *vec_in, cpl_vector **vec)
{
    int             i           = 0,
                    nx          = 0,
                    x           = 0,
                    nr_inf      = 0;

    double          *pvec       = NULL;
    const double    *pvec_in    = NULL;

    const double    *pinfinite  = NULL;

    cpl_vector      *infinite   = NULL;

    KMO_TRY
    {
        KMO_TRY_ASSURE((vec_in != NULL) && (vec != NULL),
                       CPL_ERROR_NULL_INPUT,
                       "Not all input data is provided!");

        nx = cpl_vector_get_size(vec_in);

        KMO_TRY_EXIT_IF_NULL(
            infinite = kmo_vector_identify_infinite(vec_in));

        /* count the number of infinite values and create a new vector
           accordingly*/
kmclipm_vector *ddd = kmclipm_vector_create(cpl_vector_duplicate(infinite));
        KMO_TRY_EXIT_IF_NULL(
            *vec = cpl_vector_new(nx - kmclipm_vector_get_sum(ddd)));
kmclipm_vector_delete(ddd); ddd = NULL;

        nr_inf = cpl_vector_get_size(infinite) - cpl_vector_get_size(*vec);

        KMO_TRY_EXIT_IF_NULL(
            pvec = cpl_vector_get_data(*vec));

        KMO_TRY_EXIT_IF_NULL(
            pvec_in = cpl_vector_get_data_const(vec_in));

        KMO_TRY_EXIT_IF_NULL(
            pinfinite = cpl_vector_get_data_const(infinite));

        /* Loop on the pixels of the data-image */
        x = 0;
        for (i = 0; i < nx; i++) {
            if (pinfinite[i] == 0.0) {
                pvec[x] = pvec_in[i];
                x++;
            }
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        cpl_vector_delete(*vec); *vec = NULL;
        nr_inf = -1;
    }

    cpl_vector_delete(infinite); infinite = NULL;

    return nr_inf;
}

/**
    @brief
        Calculate the standard deviation of background pixels in a cube.

    The data of the cube is provided as a vector (infinite values have already
    been removed). Values less than a minimum threshold and equal to zero
    will be removed for further calculation.
    Data above and below a defined threshold around the mean value are
    rejected and the histogram is calculated. This step is iterated.
    Then a gauss function is fitted iteratively to the histogram to
    determine the maximum position and the stddev of the gauss function.
    If the gauss-fit doesn't converge for any reason a WARNING is emmited
    and the mean and sigma calculated with rejection are returned.

    @param data     The cube-data as vector (use @c kmo_imagelist_to_vector() to
                    convert the data).
    @param mode     (Output) The mode.
    @param noise    (Output) The noise.
    @param cpos_rej The positive rejection threshold.
    @param cneg_rej The negative rejection threshold.
    @param citer    The number of iterations for rejection.

    @return CPL_ERROR_NONE in case of success.

    Possible cpl_error_code set in this function:

    @li CPL_ERROR_NULL_INPUT    if @c data is NULL.
*/
cpl_error_code kmo_calc_mode(const kmclipm_vector *data,
                             double *mode,
                             double *noise,
                             double cpos_rej,
                             double cneg_rej,
                             int citer)
{
    cpl_vector      *d          = NULL,
                    *d3         = NULL,
                    *index_vec  = NULL,
                    *h2         = NULL,
                    *x2         = NULL,
                    *x          = NULL,
                    *h          = NULL,
                    *tmp_vec    = NULL;

    const double    *pdata      = NULL;

    double          minthresh   = -1000.0,
                    sig         = 3.0,
                    sdv_sc      = 1.01,
                    *pd         = NULL,
                    *pindex_vec = NULL,
                    *px2        = NULL,
                    *ph         = NULL,
                    *ph2        = NULL,
                    area        = 0.0,
                    offset      = 0.0,
                    uthresh     = 0.8,
                    lthresh     = 2.5,
                    tmp_min     = 0.0,
                    tmp_max     = 0.0,
                    stderr      = 0.0;


    int             i           = 0,
                    count       = 0,
                    nr_bins     = 100,
                    iter        = 0,
                    ii          = 0,
                    tmp_int     = 0;

    cpl_error_code  ret_error   = CPL_ERROR_NONE,
                    fit_error   = CPL_ERROR_NONE;
    enum combine_status status  = combine_ok;

    KMO_TRY
    {
        KMO_TRY_ASSURE(data != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "No input data is provided!");

        KMO_TRY_EXIT_IF_NULL(
            pdata = cpl_vector_get_data_const(data->data    ));

        /*
         * tmp = where(finite(data) and data ne omit and data gt minthresh,tmpi)
         */

        /* check if there are any values equal 0.0 or less than minthresh */
        count = kmclipm_vector_get_size(data);
        KMO_TRY_EXIT_IF_NULL(
            index_vec = cpl_vector_new(count));
        KMO_TRY_EXIT_IF_ERROR(
            cpl_vector_fill(index_vec, 1.0));
        KMO_TRY_EXIT_IF_NULL(
            pindex_vec = cpl_vector_get_data(index_vec));

        for (i = 0; i < kmclipm_vector_get_size(data); i++) {
            if (kmclipm_vector_is_rejected(data, i) ||
                (pdata[i] == 0.0) ||
                (pdata[i] < minthresh))
            {
                pindex_vec[i] = 0.0;
                count--;
            }
        }


        if (count == 1) {
            *mode = -1;
            *noise = -1;
        } else {
            /* create new vector */
            if (count == kmclipm_vector_get_size(data)) {
                /* no values have been omitted */
                KMO_TRY_EXIT_IF_NULL(
                    d = cpl_vector_duplicate(data->data));
            } else {
                /* values have been omitted */
                KMO_TRY_EXIT_IF_NULL(
                    d = cpl_vector_new(count));

                KMO_TRY_EXIT_IF_NULL(
                    pd = cpl_vector_get_data(d));

                count = 0;
                for (i = 0; i < kmclipm_vector_get_size(data); i++) {
                    if (pindex_vec[i] == 1.0) {
                        pd[count] = pdata[i];
                        count++;
                    }
                }
            }
            cpl_vector_delete(index_vec); index_vec = NULL, pindex_vec = NULL;

            /* iterate histogram calculation using rejection */
            KMO_TRY_EXIT_IF_NULL(
                d3 = kmo_reject_sigma(d, kmo_vector_get_mean_old(d),
                                      sig, sig, cpl_vector_get_stdev(d),
                                      NULL));

            KMO_TRY_EXIT_IF_NULL(
                x2 = cpl_vector_new(nr_bins));

            KMO_TRY_EXIT_IF_NULL(
                px2 = cpl_vector_get_data(x2));

            while((cpl_vector_get_stdev(d) > cpl_vector_get_stdev(d3) * sdv_sc) ||
                  iter < 3)
            {
                KMO_TRY_CHECK_ERROR_STATE();

                cpl_vector_delete(h2); h2 = NULL;

                KMO_TRY_EXIT_IF_NULL(
                    h2 = kmo_vector_histogram_old(d3, nr_bins));

                tmp_min = cpl_vector_get_min(d3);
                tmp_max = cpl_vector_get_max(d3);
                for (i = 0; i < cpl_vector_get_size(x2); i++) {
                    px2[i] = i * (tmp_max - tmp_min) / (nr_bins - 1) + tmp_min;
                }

                cpl_vector_delete(d); d = NULL;

                d = d3;

                KMO_TRY_EXIT_IF_NULL(
                    d3 = kmo_reject_sigma(d, kmo_vector_get_mean_old(d),
                                          sig, sig, cpl_vector_get_stdev(d),
                                          NULL));
                iter++;
            }

            /* 1st fit */
            fit_error = cpl_vector_fit_gaussian(x2, NULL, h2, NULL,
                                    CPL_FIT_CENTROID | CPL_FIT_STDEV | CPL_FIT_AREA,
                                    mode,
                                    noise,
                                    &area,
                                    &offset,
                                    NULL, NULL, NULL);

            if (fit_error == CPL_ERROR_CONTINUE) {
                        cpl_msg_warning(cpl_func, "Calculation of mode & noise "
                                                  "failed (Did not converge in "
                                                  "1st iteration)!");
                KMO_TRY_RECOVER();

                KMO_TRY_EXIT_IF_NULL(
                    tmp_vec = cpl_vector_new(cpl_vector_get_size(x2)));

                KMO_TRY_EXIT_IF_ERROR(
                    cpl_vector_fill(tmp_vec, 1.0));

cpl_vector *dupd = cpl_vector_duplicate(x2);
cpl_vector *dupm = cpl_vector_duplicate(tmp_vec);
kmclipm_vector *kv = NULL;
KMO_TRY_EXIT_IF_NULL(
    kv = kmclipm_vector_create2(dupd, dupm));
                *mode = kmclipm_combine_vector(kv,
                                               NULL,
                                               "ksigma",
                                               cpos_rej,
                                               cneg_rej,
                                               citer,
                                               0,
                                               0,
                                               &tmp_int,
                                               noise,
                                               &stderr,
                                               0./0.,
                                               &status);
                KMO_TRY_CHECK_ERROR_STATE();

                KMO_TRY_ASSURE(status == combine_ok,
                               CPL_ERROR_ILLEGAL_INPUT,
                               "Vector couldn't be combined!");

kmclipm_vector_delete(kv); kv = NULL;
                cpl_vector_delete(tmp_vec); tmp_vec = NULL;
            } else {
                /* apply IDL-where to x2*/
                KMO_TRY_EXIT_IF_NULL(
                    x = kmo_reject_sigma(x2, *mode,
                                        uthresh, lthresh, *noise,
                                        &index_vec));

                /* apply same IDL-where to h2*/
                KMO_TRY_EXIT_IF_NULL(
                    h = cpl_vector_new(cpl_vector_get_size(x)));

                KMO_TRY_EXIT_IF_NULL(
                    ph = cpl_vector_get_data(h));

                KMO_TRY_EXIT_IF_NULL(
                    ph2 = cpl_vector_get_data(h2));

                KMO_TRY_EXIT_IF_NULL(
                    pindex_vec = cpl_vector_get_data(index_vec));

                ii = 0;
                for (i = 0; i < cpl_vector_get_size(h2); i++) {
                    if (pindex_vec[i] == 1.0) {
                        ph[ii] = ph2[i];
                        ii++;
                    }
                }

                cpl_vector_delete(index_vec), index_vec = NULL;

                /* 2nd fit */
                fit_error = cpl_vector_fit_gaussian(x, NULL, h, NULL,
                                    CPL_FIT_CENTROID | CPL_FIT_STDEV | CPL_FIT_AREA,
                                    mode,
                                    noise,
                                    &area,
                                    &offset,
                                    NULL, NULL, NULL);
                if (fit_error == CPL_ERROR_CONTINUE) {
                        cpl_msg_warning(cpl_func, "Calculation of mode & noise "
                                                  "failed (Did not converge in "
                                                  "2nd iteration)!");
                    KMO_TRY_RECOVER();

                    KMO_TRY_EXIT_IF_NULL(
                        tmp_vec = cpl_vector_new(cpl_vector_get_size(x)));

                    KMO_TRY_EXIT_IF_ERROR(
                        cpl_vector_fill(tmp_vec, 1.0));
cpl_vector *dupd = cpl_vector_duplicate(x);
cpl_vector *dupm = cpl_vector_duplicate(tmp_vec);
kmclipm_vector *kv = NULL;
KMO_TRY_EXIT_IF_NULL(
    kv = kmclipm_vector_create2(dupd, dupm));
                    *mode = kmclipm_combine_vector(kv,
                                                   NULL,
                                                   "ksigma",
                                                   cpos_rej,
                                                   cneg_rej,
                                                   citer,
                                                   0,
                                                   0,
                                                   &tmp_int,
                                                   noise,
                                                   &stderr,
                                                   0./0.,
                                                   &status);
                    KMO_TRY_CHECK_ERROR_STATE();

                    KMO_TRY_ASSURE(status == combine_ok,
                                   CPL_ERROR_ILLEGAL_INPUT,
                                   "Vector couldn't be combined!");

kmclipm_vector_delete(kv); kv = NULL;
                    cpl_vector_delete(tmp_vec); tmp_vec = NULL;
                } else {
                    cpl_vector_delete(x), x = NULL;
                    cpl_vector_delete(h), h = NULL;

                    /* apply IDL-where to x2*/
                    KMO_TRY_EXIT_IF_NULL(
                        x = kmo_reject_sigma(x2, *mode,
                                            uthresh, lthresh, *noise,
                                            &index_vec));

                    /* apply same IDL-where to h2*/
                    KMO_TRY_EXIT_IF_NULL(
                        h = cpl_vector_new(cpl_vector_get_size(x)));

                    KMO_TRY_EXIT_IF_NULL(
                        ph = cpl_vector_get_data(h));

                    KMO_TRY_EXIT_IF_NULL(
                        pindex_vec = cpl_vector_get_data(index_vec));

                    ii = 0;
                    for (i = 0; i < cpl_vector_get_size(h2); i++) {
                        if (pindex_vec[i] == 1.0) {
                            ph[ii] = ph2[i];
                            ii++;
                        }
                    }

                    cpl_vector_delete(index_vec), index_vec = NULL;

                    /* 3rd fit */
                    fit_error = cpl_vector_fit_gaussian(x, NULL, h, NULL,
                                    CPL_FIT_CENTROID | CPL_FIT_STDEV | CPL_FIT_AREA,
                                    mode,
                                    noise,
                                    &area,
                                    &offset,
                                    NULL, NULL, NULL);

                    if (fit_error == CPL_ERROR_CONTINUE) {
                        cpl_msg_warning(cpl_func, "Calculation of mode & noise "
                                                  "failed (Did not converge in "
                                                  "3rd iteration)!");
                        KMO_TRY_RECOVER();

                        KMO_TRY_EXIT_IF_NULL(
                            tmp_vec = cpl_vector_new(cpl_vector_get_size(x)));

                        KMO_TRY_EXIT_IF_ERROR(
                            cpl_vector_fill(tmp_vec, 1.0));
    cpl_vector *dupd = cpl_vector_duplicate(x);
    cpl_vector *dupm = cpl_vector_duplicate(tmp_vec);
    kmclipm_vector *kv = NULL;
    KMO_TRY_EXIT_IF_NULL(
        kv = kmclipm_vector_create2(dupd, dupm));
                        *mode = kmclipm_combine_vector(kv,
                                                       NULL,
                                                       "ksigma",
                                                       cpos_rej,
                                                       cneg_rej,
                                                       citer,
                                                       0,
                                                       0,
                                                       &tmp_int,
                                                       noise,
                                                       &stderr,
                                                       0./0.,
                                                       &status);
                        KMO_TRY_CHECK_ERROR_STATE();

                        KMO_TRY_ASSURE(status == combine_ok,
                                       CPL_ERROR_ILLEGAL_INPUT,
                                       "Vector couldn't be combined!");

    kmclipm_vector_delete(kv); kv = NULL;
                        cpl_vector_delete(tmp_vec); tmp_vec = NULL;
                    }
                }
            }
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        ret_error = cpl_error_get_code();
        *mode = -1;
        *noise = -1;
    }

    cpl_vector_delete(x), x = NULL;
    cpl_vector_delete(h), h = NULL;
    cpl_vector_delete(x2), x2 = NULL;
    cpl_vector_delete(h2), h2 = NULL;
    cpl_vector_delete(d), d = NULL;
    cpl_vector_delete(d3), d3 = NULL;
    cpl_vector_delete(index_vec), index_vec = NULL;

    return ret_error;
}

/**
    @brief
        Rejects values in @c d above center + high * dev and below
        center - low * dev.

    @param d         The input vector.
    @param center    The center of the values to reject.
    @param high      The upper threshold.
    @param low       The lower threshold.
    @param dev       The deviation.
    @param ret_index Optional output: If NULL is provided nothing is returned.
                     If a variable is passed, then a new vector of size of @c d
                     is returned containing zeros for rejected values and ones
                     for non-rejected values.

    @return
        A new vector with removed values lying above or below the thresholds.

    Possible cpl_error_code set in this function:

    @li CPL_ERROR_NULL_INPUT    if @c d is NULL.
*/
cpl_vector* kmo_reject_sigma(const cpl_vector *d,
                             double center,
                             double high,
                             double low,
                             double dev,
                             cpl_vector **ret_index)
{
    cpl_vector      *vec_out    = NULL,
                    *index_vec  = NULL;

    const double    *pd         = NULL;

    double          *pvec_out   = NULL,
                    *pindex_vec  = NULL,
                    hi_thresh   = 0.0,
                    lo_thresh   = 0.0;

    int             i           = 0,
                    count       = 0;

    KMO_TRY
    {
        KMO_TRY_ASSURE(d != NULL,
                       CPL_ERROR_NULL_INPUT,
                       "No input data is provided!");

        KMO_TRY_EXIT_IF_NULL(
            pd = cpl_vector_get_data_const(d));

        count = cpl_vector_get_size(d);
        KMO_TRY_EXIT_IF_NULL(
            index_vec = cpl_vector_new(count));
        KMO_TRY_EXIT_IF_ERROR(
            cpl_vector_fill(index_vec, 1.0));
        KMO_TRY_EXIT_IF_NULL(
            pindex_vec = cpl_vector_get_data(index_vec));

        hi_thresh = center + high * dev;
        lo_thresh = center - low * dev;

        /* apply rejection here, put results into index_vec */
        for (i = 0; i < cpl_vector_get_size(d); i++) {
            if ((pd[i] >= hi_thresh) || (pd[i] <= lo_thresh)) {
                pindex_vec[i] = 0.0;
                count--;
            }
        }

        /* create new vector */
        if (count == cpl_vector_get_size(d)) {
            /* no values have been omitted */
            KMO_TRY_EXIT_IF_NULL(
                vec_out = cpl_vector_duplicate(d));
        } else if (count == 0) {
            ; /* do nothing */
        } else {
            /* values have been omitted */
            KMO_TRY_EXIT_IF_NULL(
                vec_out = cpl_vector_new(count));

            KMO_TRY_EXIT_IF_NULL(
                pvec_out = cpl_vector_get_data(vec_out));

            /* copy non-rejected values into output vector */
            count = 0;
            for (i = 0; i < cpl_vector_get_size(d); i++) {
                if (pindex_vec[i] == 1.0) {
                    pvec_out[count] = pd[i];
                    count++;
                }
            }
        }

        if (ret_index == NULL) {
            cpl_vector_delete(index_vec); index_vec = NULL, pindex_vec = NULL;
        } else {
            *ret_index = index_vec;
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        cpl_vector_delete(vec_out); vec_out = NULL;
        cpl_vector_delete(index_vec); index_vec = NULL;
        if (ret_index != NULL) {
            *ret_index = NULL;
        }
    }

    return vec_out;
}

/**
    @brief Calculates the remaining statistics.

    Remaining means the statistics that are the same in calculation for any data
    type (vector, image, cube).

    @param data_in_vec  The input data.
    @param data_out     (Output) The vector with the statistics.
    @param cpos_rej     The positive rejection threshold.
    @param cneg_rej     The negative rejection threshold.
    @param citer        The number of iterations for rejection.
    @param nr_finite    The number of already identified finite pixels.

    @return             Any error set.
*/
cpl_error_code kmo_calc_remaining(kmclipm_vector *data_in_vec,
                                  kmclipm_vector *data_out,
                                  double cpos_rej,
                                  double cneg_rej,
                                  int citer,
                                  int nr_finite)
{
    cpl_error_code  ret_err             = CPL_ERROR_NONE;

    double          mean                = 0.0,
                    stdev               = 0.0,
                    mode                = 0.0,
                    noise               = 0.0,
                    stderr              = 0.0;

    int             tmp_int             = 0;
    enum combine_status status          = combine_ok;

    KMO_TRY
    {
        KMO_TRY_ASSURE((data_in_vec != NULL) &&
                       (data_out != NULL),
                       CPL_ERROR_NULL_INPUT,
                       "No input data is provided!");

        KMO_TRY_ASSURE(kmclipm_vector_get_size(data_out) == 11,
                       CPL_ERROR_ILLEGAL_INPUT,
                       "data_out vector must have size 11!");

        if (nr_finite != 0) {
            /* 3.  Mean */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 2, kmclipm_vector_get_mean(data_in_vec)));

            /* 4.  Standard deviation */
            if (kmclipm_vector_count_non_rejected(data_in_vec) < 2) {
                if (kmclipm_vector_count_non_rejected(data_in_vec) == 1) {
                    kmclipm_vector_reject(data_in_vec, 0);
                }
            } else {
                KMO_TRY_EXIT_IF_ERROR(
                    kmclipm_vector_set(data_out, 3,
                                   kmclipm_vector_get_stdev(data_in_vec)));
            }

            /* 5.  Mean with iterative rejection (i.e. mean & sigma are calculated
                   iteratively, each time rejecting pixels more than +/-N sigma from
                   the mean) */
            mean = kmclipm_combine_vector(data_in_vec,
                                          NULL,
                                          "ksigma",
                                          cpos_rej,
                                          cneg_rej,
                                          citer,
                                          0,
                                          0,
                                          &tmp_int,
                                          &stdev,
                                          &stderr,
                                          0./0.,
                                          &status);
            KMO_TRY_CHECK_ERROR_STATE();

            KMO_TRY_ASSURE(status == combine_ok,
                           CPL_ERROR_ILLEGAL_INPUT,
                           "Vector couldn't be combined!");

            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 4, mean));

            /* 6.  Standard Deviation with iterative rejection */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 5, stdev));

            /* 7.  Median */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 6,
                               kmclipm_vector_get_median(data_in_vec,
                                                         KMCLIPM_ARITHMETIC)));

            /* 8.  Mode (i.e. the peak in a histogram of pixel values) */
            KMO_TRY_EXIT_IF_ERROR(
                kmo_calc_mode(data_in_vec,
                              &mode,
                              &noise,
                              cpos_rej,
                              cneg_rej,
                              citer));

            if ((mode + 1e-6 > -1) && (mode - 1e-6 < -1)) {
                KMO_TRY_EXIT_IF_ERROR(
                    kmclipm_vector_reject(data_out, 7));
            } else {
                KMO_TRY_EXIT_IF_ERROR(
                    kmclipm_vector_set(data_out, 7, mode));
            }

            /* 9.  Noise (a robust estimate given by the standard deviation from the
                   negative side of the histogram of pixel values) */
            if ((noise + 1e-6 > -1) && (noise - 1e-6 < -1)) {
                KMO_TRY_EXIT_IF_ERROR(
                    kmclipm_vector_reject(data_out, 8));
            } else {
                KMO_TRY_EXIT_IF_ERROR(
                    kmclipm_vector_set(data_out, 8, noise));
            }

            /* 10.  Minimum */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 9,
                               kmclipm_vector_get_min(data_in_vec, NULL)));

            /* 11.  Maximum */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 10,
                               kmclipm_vector_get_max(data_in_vec, NULL)));
        } else {
            /* 3.  Mean */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 2, 0./0.));
            /* 4.  Standard deviation */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 3, 0./0.));
            /* 5.  Mean with iterative rejection */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 4, 0./0.));
            /* 6.  Standard Deviation with iterative rejection */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 5, 0./0.));
            /* 7.  Median */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 6, 0./0.));
            /* 8.  Mode (i.e. the peak in a histogram of pixel values) */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 7, 0./0.));
            /* 9.  Noise */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 8, 0./0.));
            /* 10.  Minimum */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 9, 0./0.));
            /* 11.  Maximum */
            KMO_TRY_EXIT_IF_ERROR(
                kmclipm_vector_set(data_out, 10, 0./0.));
        }
    }
    KMO_CATCH
    {
        KMO_CATCH_MSG();
        ret_err = cpl_error_get_code();
//        kmclipm_vector_delete(data_out); data_out = NULL;
    }

    return ret_err;
}

/** @} */
