/* $Id: fors_flat.cc,v 1.9 2013/09/10 19:25:32 cgarcia Exp $
 *
 * This file is part of the FORS Data Reduction Pipeline
 * Copyright (C) 2002-2010 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

/*
 * $Author: cgarcia $
 * $Date: 2013/09/10 19:25:32 $
 * $Revision: 1.9 $
 * $Name:  $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif


#include <vector>
#include <exception>
#include <math.h>
#include <cpl.h>
#include <moses.h>
#include <fors_dfs.h>
#include <fors_utils.h>
#include <fors_image.h>
#include "detected_slits.h"
#include "wavelength_calibration.h"
#include "flat_combine.h"

static int fors_flat_create(cpl_plugin *);
static int fors_flat_exec(cpl_plugin *);
static int fors_flat_destroy(cpl_plugin *);
static int fors_flat(cpl_parameterlist *, cpl_frameset *);

static char fors_flat_description[] =
"This recipe is used to subtract the master bias, produced by the recipe\n"
"fors_bias, from a set of raw flat field frames. The input raw frames are\n"
"summed, the master bias frame is rescaled accordingly, and subtracted\n"
"from the result. The overscan regions, if present, are used to compensate\n"
"for variations of the bias level between master bias and input raw frames.\n"
"The overscan regions are then trimmed from the result.\n"
"In the table below the MXU acronym can be alternatively read as MOS and\n"
"LSS.\n\n"
"Input files:\n\n"
"  DO category:               Type:       Explanation:         Required:\n"
"  SCREEN_FLAT_MXU            Raw         Raw data frame          Y\n"
"  SLIT_LOCATION_MXU          Calib       Slits positions on CCD  Y\n"
"  CURV_COEFF_MXU             Calib       Slits tracing fits      Y\n"
"  DISP_COEFF_MXU             Calib       Wavelength calibration  Y\n"
"  MASTER_BIAS                Calib       Master bias frame       Y\n\n"
"Output files:\n\n"
"  DO category:               Data type:  Explanation:\n"
"  MASTER_SCREEN_FLAT_MXU     FITS image  Bias subtracted sum frame\n\n";

#define fors_flat_exit(message)               \
{                                             \
if (message) cpl_msg_error(recipe, message);  \
cpl_image_delete(flat);                       \
cpl_image_delete(master_bias);                \
cpl_propertylist_delete(header);              \
cpl_table_delete(flat_overscans);                  \
cpl_msg_indent_less();                        \
return -1;                                    \
}

#define fors_flat_exit_memcheck(message)        \
{                                               \
if (message) cpl_msg_info(recipe, message);     \
printf("free flat (%p)\n", flat);               \
cpl_image_delete(flat);                         \
printf("free master_flat (%p)\n", master_flat); \
cpl_image_delete(master_flat);                  \
printf("free master_bias (%p)\n", master_bias); \
cpl_image_delete(master_bias);                  \
printf("free header (%p)\n", header);           \
cpl_propertylist_delete(header);                \
printf("free overscans (%p)\n", overscans);     \
cpl_table_delete(overscans);                    \
cpl_msg_indent_less();                          \
return 0;                                       \
}


/**
 * @brief    Build the list of available plugins, for this module. 
 *
 * @param    list    The plugin list
 *
 * @return   0 if everything is ok, -1 otherwise
 *
 * Create the recipe instance and make it available to the application 
 * using the interface. This function is exported.
 */

int cpl_plugin_get_info(cpl_pluginlist *list)
{
    cpl_recipe *recipe = static_cast<cpl_recipe *>(cpl_calloc(1, sizeof *recipe ));
    cpl_plugin *plugin = &recipe->interface;

    cpl_plugin_init(plugin,
                    CPL_PLUGIN_API,
                    FORS_BINARY_VERSION,
                    CPL_PLUGIN_TYPE_RECIPE,
                    "fors_spec_mflat",
                    "Computes master spectroscopic flat, removing bias first",
                    fors_flat_description,
                    "Carlo Izzo",
                    PACKAGE_BUGREPORT,
                    fors_get_license(),
                    fors_flat_create,
                    fors_flat_exec,
                    fors_flat_destroy);

    cpl_pluginlist_append(list, plugin);
    
    return 0;
}


/**
 * @brief    Setup the recipe options    
 *
 * @param    plugin  The plugin
 *
 * @return   0 if everything is ok
 *
 * Defining the command-line/configuration parameters for the recipe.
 */

static int fors_flat_create(cpl_plugin *plugin)
{
    cpl_recipe    *recipe;
    cpl_parameter *p;

    /* 
     * Check that the plugin is part of a valid recipe 
     */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    /* 
     * Create the parameters list in the cpl_recipe object 
     */

    recipe->parameters = cpl_parameterlist_new(); 


    /*
     * Number of iterations
     */
    p = cpl_parameter_new_value("fors.fors_spec_mflat.smooth_sed",
                                CPL_TYPE_DOUBLE,
                                "Smoothing size for each flat sed",
                                "fors.fors_spec_mflat",
                                0.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "smooth_sed");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /*
     * Stacking method
     */
    p = cpl_parameter_new_enum("fors.fors_spec_mflat.stack_method",
                                CPL_TYPE_STRING,
                                "Frames combination method",
                                "fors.fors_spec_mflat",
                                "sum", 4,
                                "sum", "mean", "median", "ksigma");
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "stack_method");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /* 
     * Low threshold for the sigma clipping algorithm 
     */
    p = cpl_parameter_new_value("fors.fors_spec_mflat.klow",
                                CPL_TYPE_DOUBLE,
                                "Low threshold in ksigma method",
                                "fors.fors_spec_mflat",
                                3.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "klow");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /* 
     * High threshold for the sigma clipping algorithm 
     */
    p = cpl_parameter_new_value("fors.fors_spec_mflat.khigh",
                                CPL_TYPE_DOUBLE,
                                "High threshold in ksigma method",
                                "fors.fors_spec_mflat",
                                3.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "khigh");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /* 
     * Number of iterations for the sigma clipping algorithm 
     */
    p = cpl_parameter_new_value("fors.fors_spec_mflat.kiter",
                                CPL_TYPE_INT,
                                "Max number of iterations in ksigma method",
                                "fors.fors_spec_mflat",
                                999);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "kiter");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    return 0;
}


/**
 * @brief    Execute the plugin instance given by the interface
 *
 * @param    plugin  the plugin
 *
 * @return   0 if everything is ok
 */

static int fors_flat_exec(cpl_plugin *plugin)
{
    cpl_recipe  *   recipe ;
    int             status = 1;

    /* Get the recipe out of the plugin */
    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin ;
    else return -1 ;

    /* Issue a banner */
    //fors_print_banner();

    try
    {
        status = fors_flat(recipe->parameters, recipe->frames);
    }
    catch(std::exception& ex)
    {
        cpl_msg_error(cpl_func, "Exception: %s", ex.what());
    }
    catch(...)
    {
        cpl_msg_error(cpl_func, "An uncaught error during recipe execution");
    }

    return status;

}


/**
 * @brief    Destroy what has been created by the 'create' function
 *
 * @param    plugin  The plugin
 *
 * @return   0 if everything is ok
 */

static int fors_flat_destroy(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
    
    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    cpl_parameterlist_delete(recipe->parameters); 

    return 0;
}


/**
 * @brief    Interpret the command line options and execute the data processing
 *
 * @param    parlist     The parameters list
 * @param    frameset    The set-of-frames
 *
 * @return   0 if everything is ok
 */

static int fors_flat(cpl_parameterlist *parlist, cpl_frameset *frameset)
{

    const char *recipe = "fors_spec_mflat";


    /*
     * CPL objects
     */

    cpl_image        *master_bias = NULL;
    cpl_image        *multi_bias  = NULL;
    cpl_table        *flat_overscans   = NULL;
    cpl_propertylist *header      = NULL;
    cpl_parameter    *par         = NULL;
    cpl_image        *flat = NULL;

    /*
     * Auxiliary variables
     */

    char        version[80];
    const char *master_bias_tag = "MASTER_BIAS";
    const char *slit_location_tag;
    const char *curv_coeff_tag;
    const char *disp_coeff_tag;
    const char *flat_tag;
    const char *pro_tag;
    char       *instrume = NULL;
    int         flat_mxu;
    int         flat_mos;
    int         flat_lss;
    int         nbias, nflat;
    int         i;
    double      smooth_sed;
    std::string stack_method;
    double      khigh, klow;
    int         kiter;
    
    
    cpl_errorstate      error_prevstate = cpl_errorstate_get();


    cpl_msg_set_indentation(2);


    /* 
     * Get configuration parameters
     */

    cpl_msg_info(recipe, "Recipe %s configuration parameters:", recipe);
    cpl_msg_indent_more();

    par = cpl_parameterlist_find(parlist, "fors.fors_spec_mflat.smooth_sed");
    smooth_sed = cpl_parameter_get_double(par) ;

    cpl_msg_info(cpl_func, "fors.fors_spec_mflat.smooth_sed = %f", smooth_sed);

    par = cpl_parameterlist_find(parlist, "fors.fors_spec_mflat.stack_method");
    stack_method = cpl_parameter_get_string(par) ;

    cpl_msg_info(cpl_func, "fors.fors_spec_mflat.stack_method = %s", stack_method.c_str());

    par = cpl_parameterlist_find(parlist, "fors.fors_spec_mflat.khigh");
    khigh = cpl_parameter_get_double(par);

    cpl_msg_info(cpl_func, "fors.fors_spec_mflat.khigh = %f", khigh);
    
    par = cpl_parameterlist_find(parlist, "fors.fors_spec_mflat.klow");
    klow = cpl_parameter_get_double(par) ;

    cpl_msg_info(cpl_func, "fors.fors_spec_mflat.klow = %f", klow);
    
    par = cpl_parameterlist_find(parlist, "fors.fors_spec_mflat.kiter");
    kiter = cpl_parameter_get_int(par) ;

    cpl_msg_info(cpl_func, "fors.fors_spec_mflat.kiter = %d", kiter);
    
    /* 
     * Check validity of parameters
     */
    if(stack_method != "mean" && stack_method != "median" && 
       stack_method != "ksigma" && stack_method != "sum")
        throw std::invalid_argument(stack_method+" stacking algorithm invalid");
    
    /*
     * Check input frames
     */
    cpl_msg_info(recipe, "Check input set-of-frames:");
    cpl_msg_indent_more();

    nbias = cpl_frameset_count_tags(frameset, master_bias_tag);
    if (nbias == 0) {
        cpl_msg_error(recipe, "Missing required input: %s", master_bias_tag);
        fors_flat_exit(NULL);
    }
    if (nbias > 1) {
        cpl_msg_error(recipe, "Too many in input (%d > 1): %s", 
                      nbias, master_bias_tag);
        fors_flat_exit(NULL);
    }

    nflat  = flat_mxu = cpl_frameset_count_tags(frameset, "SCREEN_FLAT_MXU");
    nflat += flat_mos = cpl_frameset_count_tags(frameset, "SCREEN_FLAT_MOS");
    nflat += flat_lss = cpl_frameset_count_tags(frameset, "SCREEN_FLAT_LSS");

    if (nflat == 0) {
        fors_flat_exit("Missing required input raw frames");
    }

    if (flat_mxu) {
        flat_tag = "SCREEN_FLAT_MXU";
        pro_tag = "MASTER_SCREEN_FLAT_MXU";
        slit_location_tag = "SLIT_LOCATION_MXU";
        curv_coeff_tag = "CURV_COEFF_MXU";
        disp_coeff_tag = "DISP_COEFF_MXU";
    }
    else if (flat_mos) {
        flat_tag = "SCREEN_FLAT_MOS";
        pro_tag = "MASTER_SCREEN_FLAT_MOS";
        slit_location_tag = "SLIT_LOCATION_MOS";
        curv_coeff_tag = "CURV_COEFF_MOS";
        disp_coeff_tag = "DISP_COEFF_MOS";
    }
    else if (flat_lss) {
        flat_tag = "SCREEN_FLAT_LSS";
        pro_tag = "MASTER_SCREEN_FLAT_LSS";
        slit_location_tag = "SLIT_LOCATION_LSS";
        curv_coeff_tag = "CURV_COEFF_LSS";
        disp_coeff_tag = "DISP_COEFF_LSS";
    }

    if (!dfs_equal_keyword(frameset, "ESO INS GRIS1 ID"))
        fors_flat_exit("Input frames are not from the same grism");

    if (!dfs_equal_keyword(frameset, "ESO INS FILT1 ID"))
        fors_flat_exit("Input frames are not from the same filter");

    if (!dfs_equal_keyword(frameset, "ESO DET CHIP1 ID")) 
        fors_flat_exit("Input frames are not from the same chip");

    header = dfs_load_header(frameset, flat_tag, 0);

    if (header == NULL) {
        cpl_msg_error(recipe, "Cannot load header of %s frame", flat_tag);
        fors_flat_exit(NULL);
    }

    instrume = (char *)cpl_propertylist_get_string(header, "INSTRUME");
    if (instrume == NULL) {
        cpl_msg_error(recipe, "Missing keyword INSTRUME in %s header", 
                      flat_tag);
        fors_flat_exit(NULL);
    }

    if (instrume[4] == '1')
        snprintf(version, 80, "%s/%s", "fors1", VERSION);
    if (instrume[4] == '2')
        snprintf(version, 80, "%s/%s", "fors2", VERSION);


    cpl_msg_indent_less();
    cpl_msg_info(recipe, "Load input frames:");
    cpl_msg_indent_more();

    /* Reading  master  bias */
    master_bias = dfs_load_image(frameset, 
                                 master_bias_tag, CPL_TYPE_FLOAT, 0, 1);
    if (master_bias == NULL)
        fors_flat_exit("Cannot load master bias");


    flat_overscans = mos_load_overscans_vimos(header, 1);

    /* Reading individual raw flats */
    std::vector<mosca::image> raw_flats;
    cpl_frameset * flatframes = hawki_extract_frameset(frameset, flat_tag);
    for (i = 0; i < nflat; i++)
    {
        cpl_frame * flatframe = cpl_frameset_get_position(flatframes, i);
        const char * filename = cpl_frame_get_filename(flatframe);
        flat = cpl_image_load(filename, CPL_TYPE_DOUBLE, 0, 0);
        cpl_propertylist * plist = cpl_propertylist_load(filename, 0) ;

        if (!flat)
            fors_flat_exit("Cannot load flat field");
        
        /* Reading gain */
        double gain = cpl_propertylist_get_double(plist, "ESO DET OUT1 GAIN");

        cpl_image * flat_corr_bias = mos_remove_bias(flat, master_bias, 
                                                     flat_overscans);
        
        
        cpl_image * flat_err = cpl_image_duplicate(flat_corr_bias);
        cpl_image_divide_scalar(flat_err, gain); //TODO: CHECK
        cpl_image_power(flat_err, 0.5); //TODO: CHECK
        mosca::image new_flat(flat_corr_bias, flat_err, true, mosca::X_AXIS);
        raw_flats.push_back(new_flat);
        cpl_image_delete(flat);
    }

    if(!cpl_errorstate_is_equal(error_prevstate))
        fors_flat_exit("Could not read the flats");
    

    /* Read the slit locations */
    cpl_msg_info(cpl_func, " Reading slit information");
    cpl_frameset * slit_frames = hawki_extract_frameset(frameset, slit_location_tag);
    cpl_frameset * curv_frames = hawki_extract_frameset(frameset, curv_coeff_tag);
    if(cpl_frameset_get_size(slit_frames) != 1)
        fors_flat_exit("One slit position frame is needed");
    if(cpl_frameset_get_size(curv_frames) != 1)
        fors_flat_exit("One curvature coefficients frame is needed");

    const char * slit_filename = cpl_frame_get_filename(cpl_frameset_get_position(slit_frames, 0));
    const char * curv_filename = cpl_frame_get_filename(cpl_frameset_get_position(curv_frames, 0));
    
    mosca::detected_slits det_slits = 
            mosca::detected_slits_load_fits(slit_filename, curv_filename,
                                            raw_flats[0].size_dispersion());
    
    if(!cpl_errorstate_is_equal(error_prevstate))
        fors_flat_exit("Could not read the slits");

    /* Read wave calib */
    cpl_msg_info(cpl_func, " Reading wavelength calibration");
    cpl_frameset * wave_frames = hawki_extract_frameset(frameset, disp_coeff_tag);
    if(cpl_frameset_get_size(wave_frames) != 1)
        fors_flat_exit("One dispersion coefficients frame is needed");
    const char * wave_filename = cpl_frame_get_filename(cpl_frameset_get_position(wave_frames, 0));
    mosca::wavelength_calibration wave_cal(wave_filename);

    if(!cpl_errorstate_is_equal(error_prevstate))
        fors_flat_exit("Could not read the wavelength calibration");

    /* Computing master flat */
    cpl_msg_info(cpl_func, " Computing master flat");
    std::auto_ptr<mosca::image> master_flat;
    if(stack_method == "mean" || stack_method == "sum")
    {
        mosca::reduce_mean reduce_method;
        master_flat = mosca::flat_combine<double, mosca::reduce_mean>
            (raw_flats, det_slits, wave_cal, smooth_sed, reduce_method);
        if(stack_method == "sum")
        {
            cpl_image_multiply_scalar(master_flat->get_cpl_image(), nflat);
            cpl_image_multiply_scalar(master_flat->get_cpl_image_err(), nflat);
        }
    }
    else if(stack_method == "median")
    {
        mosca::reduce_median reduce_method;
        master_flat = mosca::flat_combine<double, mosca::reduce_median>
            (raw_flats, det_slits, wave_cal, smooth_sed, reduce_method);        
    }
    else if(stack_method == "ksigma")
    {
        mosca::reduce_sigma_clipping reduce_method(khigh, klow, kiter);
        master_flat = mosca::flat_combine<double, mosca::reduce_sigma_clipping>
            (raw_flats, det_slits, wave_cal, smooth_sed, reduce_method);        
    }
        
    cpl_table_delete(flat_overscans); flat_overscans = NULL;
    cpl_image_delete(master_bias); master_bias = NULL;


    cpl_msg_indent_less();


    /* Save product */
    fors_image * fors_master_flat = fors_image_new(master_flat->get_cpl_image(),
            cpl_image_power_create(master_flat->get_cpl_image_err(), 2.));

    cpl_propertylist *save_header  = cpl_propertylist_new();
    cpl_propertylist_update_int(save_header, "ESO PRO DATANCOM", nflat);    

    fors_dfs_save_image_err(frameset, fors_master_flat, pro_tag,
                            save_header, parlist, recipe,
                            cpl_frameset_get_position_const(flatframes, 0));

    cpl_propertylist_delete(header); header = NULL;

    return 0;
}
