/* $Id: naco_img_lampflat.c,v 1.60 2010-03-04 15:57:20 llundin Exp $
 *
 * This file is part of the NACO Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: llundin $
 * $Date: 2010-03-04 15:57:20 $
 * $Revision: 1.60 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include "naco_recipe.h"

/*-----------------------------------------------------------------------------
                            Recipe defines
 -----------------------------------------------------------------------------*/

#define RECIPE_STRING   "naco_img_lampflat"

/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/

static cpl_image * naco_img_lampflat_reduce(cpl_propertylist *,
                                            const irplib_framelist *,
                                            int, int, int, int);

static cpl_error_code naco_img_lampflat_qc(cpl_propertylist *,
                                           cpl_propertylist *,
                                           const irplib_framelist *);

static cpl_error_code naco_img_lampflat_save(cpl_frameset *,
                                             const cpl_parameterlist *,
                                             const cpl_propertylist *,
                                             const cpl_propertylist *,
                                             const cpl_image *,
                                             int, const irplib_framelist *);

static char * naco_img_lampflat_make_tag(const cpl_frame*,
                                         const cpl_propertylist *, int);

NACO_RECIPE_DEFINE(naco_img_lampflat, NACO_PARAM_REJBORD,
                   "Flat recipe using a lamp",
                   RECIPE_STRING " -- NACO imaging flat-field creation from "
                   "lamp images.\n" 
                   "The files listed in the Set Of Frames (sof-file) "
                   "must be tagged:\n" 
                   "NACO-raw-file.fits " NACO_IMG_LAMPFLAT_RAW "\n"                    
                   "\n"                                                                
                   "Furthermore, the input set of frames must have values of "
                   "the FITS key " 
                   NACO_PFITS_INT_LAMP2 " that alternate between zero and "
                   "non-zero (with non-zero for the first frame).\n"         
                   "It is further required that the light from the lamp is "
                   "in a range without a significant thermal background.");


/*----------------------------------------------------------------------------*/
/**
 * @defgroup naco_img_lampflat   Lamp Flat
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/

/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    framelist   the frames list
  @param    parlist     the parameters list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int naco_img_lampflat(cpl_frameset            * framelist,
                             const cpl_parameterlist * parlist)
{
    cpl_errorstate cleanstate = cpl_errorstate_get();
    irplib_framelist * allframes = NULL;
    irplib_framelist * rawframes = NULL;
    irplib_framelist * f_one     = NULL;
    const char      ** taglist   = NULL;
    const char       * rej_bord;
    cpl_image        * lamp_flat = NULL;
    cpl_propertylist * qclist  = cpl_propertylist_new();
    cpl_propertylist * paflist = cpl_propertylist_new();
    int                nb_good = 0;
    int                nsets;
    int                rej_left, rej_right, rej_bottom, rej_top;
    int                i;
    

    /* Retrieve input parameters */
    rej_bord = naco_parameterlist_get_string(parlist, RECIPE_STRING, NACO_PARAM_REJBORD);
    skip_if (0);
    skip_if_ne(sscanf(rej_bord, "%d %d %d %d",
                      &rej_left, &rej_right, &rej_bottom, &rej_top), 4,
               "number(s) in string parameter (%s): \"%s\"",
               CPL_XSTRINGIFY(NACO_PARAM_REJBORD), rej_bord);
  
    /* Identify the RAW and CALIB frames in the input frameset */
    skip_if (naco_dfs_set_groups(framelist));

    allframes = irplib_framelist_cast(framelist);
    skip_if(allframes == NULL);

    rawframes = irplib_framelist_extract(allframes, NACO_IMG_LAMPFLAT_RAW);
    skip_if(rawframes == NULL);
    irplib_framelist_empty(allframes);

    skip_if(irplib_framelist_load_propertylist_all(rawframes, 0, "^("
                                                   IRPLIB_PFITS_REGEXP_RECAL "|"
                                                   NACO_PFITS_REGEXP_LAMPFLAT "|"
                                                   NACO_PFITS_REGEXP_LAMPFLAT_PAF
                                                   ")$", CPL_FALSE));

    taglist = naco_framelist_set_tag(rawframes, naco_img_lampflat_make_tag,
                                     &nsets);
    skip_if(taglist == NULL);

    cpl_msg_info(cpl_func, "Identified %d setting(s) in %d frames",
                 nsets, irplib_framelist_get_size(rawframes));

    /* Extract settings and reduce each of them */
    for (i=0 ; i < nsets ; i++) {

        /* Reduce data set nb i */
        cpl_msg_info(cpl_func, "Reducing data set %d of %d", i+1, nsets);

        /* Reduce data set nb i */
        f_one = irplib_framelist_extract(rawframes, taglist[i]);

        /* Reset the tag */
        skip_if(irplib_framelist_set_tag_all(f_one, NACO_IMG_LAMPFLAT_RAW));

        cpl_msg_info(cpl_func, "Reducing frame set %d of %d (size=%d) with "
                     "setting: %s", i+1, nsets,
                     irplib_framelist_get_size(f_one), taglist[i]);

        skip_if (f_one == NULL);
        
        lamp_flat = naco_img_lampflat_reduce(qclist, f_one, rej_left, rej_right,
                                             rej_bottom, rej_top);
        
        /* Save the products */
        if (lamp_flat == NULL) {
            if (nsets > 1)
                irplib_error_recover(cleanstate, "Could not compute the flat for "
                                     "this setting");
        } else {
            skip_if(naco_img_lampflat_qc(qclist, paflist, f_one));
            /* PRO.CATG */
            bug_if(cpl_propertylist_append_string(paflist, CPL_DFS_PRO_CATG,
                                                  NACO_IMG_LAMPFLAT_RES));
            skip_if(naco_img_lampflat_save(framelist, parlist, qclist, paflist,
                                           lamp_flat, i+1, f_one));
            cpl_image_delete(lamp_flat);
            lamp_flat = NULL;
            nb_good++;
        }
        cpl_propertylist_empty(qclist);
        cpl_propertylist_empty(paflist);
        irplib_framelist_delete(f_one);
        f_one = NULL;
    }

    irplib_ensure(nb_good > 0, CPL_ERROR_DATA_NOT_FOUND,
                  "None of the %d sets could be reduced", nsets);
    
    end_skip;

    cpl_free(taglist);
    cpl_image_delete(lamp_flat);
    irplib_framelist_delete(f_one);
    irplib_framelist_delete(allframes);
    irplib_framelist_delete(rawframes);
    cpl_propertylist_delete(qclist);
    cpl_propertylist_delete(paflist);

    return cpl_error_get_code();
}

/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    qclist     List of QC parameters
  @param    framelist  The list of frames
  @param    rej_left   The left of the rejection border
  @param    rej_right  The ditto right
  @param    rej_bottom The ditto bottom
  @param    rej_top    The ditto top
  @return   the flat field or NULL in error case
 */
/*----------------------------------------------------------------------------*/
static cpl_image * naco_img_lampflat_reduce(cpl_propertylist * qclist,
                                            const irplib_framelist * framelist,
                                            int rej_left, int rej_right,
                                            int rej_bottom, int rej_top)
{
    int                     nfiles;
    cpl_image           *   image = NULL;
    cpl_image           *   aver = NULL;
    cpl_image           *   diff = NULL;
    cpl_image           *   prev = NULL;
    cpl_image           *   image0= NULL;
    double                  gain, fp_noise;
    double                  lamp_flux = DBL_MAX; /* Avoid uninit warning */
    double                  std_diff[3];
    double                  mean = DBL_MAX; /* Avoid uninit warning */
    double                  square;
    int                     i;


    bug_if (framelist == NULL);

    nfiles = irplib_framelist_get_size(framelist);
 
    /* At least 4 frames are requested for the gain computation */
    skip_if_lt(nfiles, 4, "frames");

    /* Verify that the number of frames is even */
    irplib_ensure (nfiles % 2 == 0, CPL_ERROR_ILLEGAL_INPUT,
                   "The number of frames must be even, not %d", nfiles);

    skip_if (irplib_framelist_contains(framelist, NACO_PFITS_DOUBLE_DIT,
                                       CPL_TYPE_DOUBLE, CPL_TRUE, 1e-5));
   
    skip_if (irplib_framelist_contains(framelist, NACO_PFITS_INT_LAMP2,
                                       CPL_TYPE_INT, CPL_FALSE, 0.0));
   
    /* Print out the file names */
    /* Verify that the frame sequence is lamp on - lamp off */
    for (i=0 ; i < nfiles ; i++) {
        const cpl_frame * frame = irplib_framelist_get_const(framelist, i);
        const char * name = cpl_frame_get_filename(frame);
        const cpl_propertylist * plist
            = irplib_framelist_get_propertylist_const(framelist, i);
        int ival;

        skip_if(name == NULL);
        skip_if(plist == NULL);

        if (i == 0) {
            /* Print out the filter used in this set of frames */
            const char * sval = naco_pfits_get_filter(plist);

            skip_if (0);
            cpl_msg_info(cpl_func, "Filter: [%s]", sval);

            /* Copy the required keys to the list of QC parameters */
            skip_if(cpl_propertylist_append_string(qclist, "ESO QC FILTER OBS",
                                                   sval));
            sval = naco_pfits_get_opti3_name(plist);
            skip_if (0);
            skip_if(cpl_propertylist_append_string(qclist, "ESO QC FILTER NDENS",
                                                   sval));
            sval = naco_pfits_get_opti4_name(plist);
            skip_if (0);
            skip_if(cpl_propertylist_append_string(qclist, "ESO QC FILTER POL",
                                                   sval));
        }

        cpl_msg_info(cpl_func, "File %02d: %s", i+1, name);

        /* Get lamp info from frame 2*i */
        ival = naco_pfits_get_lamp2(plist);
        if (i % 2) {
            irplib_ensure (ival == 0, CPL_ERROR_ILLEGAL_INPUT,
                           "Frame number %d must have %s set to zero, not %d",
                            i+1, NACO_PFITS_INT_LAMP2, ival);
        } else {
            irplib_ensure (ival != 0, CPL_ERROR_ILLEGAL_INPUT,
                           "Frame number %d must have a non-zero %s",
                            i+1, NACO_PFITS_INT_LAMP2);
        }

        irplib_check(image = cpl_image_load(name, CPL_TYPE_FLOAT, 0, 0),
                     "Could not load FITS-image from %s", name);

        if (i == 0) {
            irplib_check(mean = cpl_image_get_mean(image),
                "Could not compute mean");
            image0 = image; image = NULL;
        } else if (i <= 3) {
            const int ifirst = i == 3 ? 1 : 0;
            double noise;
            /* Compute noise on : on1 - off1 */
            /* Compute noise on : on1 - on2 */
            /* Compute noise on : off1 - off2 */
            irplib_check(diff = cpl_image_subtract_create(image0, image),
                         "Could not subtract the images %d and %d",
                          ifirst+1, i+1);

            irplib_check(cpl_flux_get_noise_window(diff, NULL, -1, -1,
                                                   &noise, NULL),
                         "Could not compute noise on difference between"
                          " images %d and %d", ifirst+1, i+1);

            std_diff[i-1] = noise * noise;

            if (i == 1) {
                /* LAMPFLUX */
                /* Print out the filter used in this set of frames */
                const double median = cpl_image_get_median(diff);
                const double ditval = naco_pfits_get_dit(plist);

                skip_if (cpl_error_get_code());
                skip_if (ditval <= 0.0);

                lamp_flux = median / ditval;

                /* The 1st contribution to the average of differences */
                aver = diff;

            } else {
                cpl_image_delete(diff);
                diff = NULL;
                 /* Finished with image[i-2] */
                cpl_image_delete(image0);
                /* Update image0 - to either image[i-1] or none */
                image0 = i == 2 ? prev : NULL;
            }

        }

        if (i > 1 && i % 2 != 0) {
            /* Compute the dark subtraction */
            irplib_check(cpl_image_subtract(prev, image),
                         "Could not correct for the dark");

            /* Sum up the dark corrected frames */
            irplib_check(cpl_image_add(aver, prev),
                         "Could not sum up the dark corrected images");

            cpl_image_delete(image);
            cpl_image_delete(prev);
            prev = NULL;
        } else {
            prev = image;
        }
        image = NULL;
    }

    /* Compute the QC parameters */
    cpl_msg_info(cpl_func, "Computing the QC parameters");

    /* Deduce GAIN */
    square = std_diff[1] - std_diff[2];
    if (square != 0.0) square = 2.0*mean/square;
    gain = square > 0.0 ? sqrt(square) : 0.0;
    
    /* Deduce FPNOISE */
    square = std_diff[0] - std_diff[1] - std_diff[2];
    fp_noise = square > 0.0 ? sqrt(square) : 0.0;

    /* Print results */
    cpl_msg_info(cpl_func, "Gain:      %g", gain);
    cpl_msg_info(cpl_func, "Noise:     %g", fp_noise);
    cpl_msg_info(cpl_func, "Lamp flux: %g", lamp_flux);

    /* Average the dark corrected frames */
    cpl_msg_info(cpl_func, "Average the dark corrected frames");
    irplib_check(cpl_image_divide_scalar(aver, (double)(nfiles/2)),
                 "Could not average the %d dark corrected frames", nfiles/2);

    /* Normalize the flat */
    irplib_check(mean
                 = cpl_image_get_mean_window(aver, rej_left,
                                             cpl_image_get_size_x(aver)
                                             -rej_right,
                                             rej_bottom,
                                             cpl_image_get_size_y(aver)
                                             -rej_top);
                 cpl_image_divide_scalar(aver, mean),
                 "Could not average the %d dark corrected frames", nfiles/2);

    /* Add QC parameters */
    skip_if(cpl_propertylist_append_double(qclist, "ESO QC GAIN", gain));
    skip_if(cpl_propertylist_append_double(qclist, "ESO QC FPNOISE", fp_noise));
    skip_if(cpl_propertylist_append_double(qclist, "ESO QC LAMPFLUX",
                                           lamp_flux));

    end_skip;

    if (cpl_error_get_code()) {
        cpl_image_delete(aver);
        aver = NULL;
    }

    cpl_image_delete(image);
    cpl_image_delete(diff);
    cpl_image_delete(prev);
    cpl_image_delete(image0);

    return aver;
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Append the QC parameters to the propertylist
  @param    qclist    List of QC parameters
  @param    paflist   List of PAF parameters
  @param    rawframes List of rawframes and their propertylists
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code naco_img_lampflat_qc(cpl_propertylist       * qclist,
                                           cpl_propertylist       * paflist,
                                           const irplib_framelist * rawframes)
{

    const cpl_propertylist * reflist
        = irplib_framelist_get_propertylist_const(rawframes, 0);
    const char pafcopy[] = "^(" NACO_PFITS_REGEXP_LAMPFLAT_PAF ")$";


    bug_if (0);


    /* THE PAF FILE FOR QC PARAMETERS */
    skip_if (cpl_propertylist_copy_property_regexp(paflist, reflist, pafcopy,
                                                   0));
    skip_if (cpl_propertylist_append(paflist, qclist));

    bug_if (cpl_propertylist_copy_property_regexp(qclist, reflist, "^("
                                                  IRPLIB_PFITS_REGEXP_RECAL 
                                                  ")$", 0));

    end_skip;

    return cpl_error_get_code();
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Save the lampflat recipe products on disk
  @param    set_tot   the total input frame set
  @param    parlist   the input list of parameters
  @param    qclist    List of QC parameters
  @param    paflist   List of PAF parameters
  @param    flat      the flat image produced
  @param    set_nb    current setting number
  @param    rawframes the list of frames with the current settings
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code naco_img_lampflat_save(cpl_frameset            * set_tot,
                                             const cpl_parameterlist * parlist,
                                             const cpl_propertylist  * qclist,
                                             const cpl_propertylist  * paflist,
                                             const cpl_image         * flat,
                                             int                       set_nb,
                                             const irplib_framelist  * rawframes)
{
    cpl_frameset * proframes = irplib_frameset_cast(rawframes);
    char         * filename  = NULL;



    /* This will catch rawframes == NULL */
    bug_if (0);

    /* SAVE THE COMBINED IMAGE */
    filename = cpl_sprintf(RECIPE_STRING "_set%02d" CPL_DFS_FITS, set_nb);
    skip_if (irplib_dfs_save_image(set_tot, parlist, proframes, flat,
                               CPL_BPP_IEEE_FLOAT, RECIPE_STRING,
                               NACO_IMG_LAMPFLAT_RES, qclist, NULL,
                               naco_pipe_id, filename));

#ifdef NACO_SAVE_PAF
    cpl_free(filename);
    filename = cpl_sprintf(RECIPE_STRING "_set%02d" CPL_DFS_PAF, set_nb);
    skip_if (cpl_dfs_save_paf("NACO", RECIPE_STRING, paflist, filename));
#else
    bug_if(paflist == NULL);
#endif

    end_skip;

    cpl_free(filename);
    cpl_frameset_delete(proframes);

    return cpl_error_get_code();

}


/*----------------------------------------------------------------------------*/
/**
  @brief    Create a string suitable for frame comparison
  @param    self   Frame to create the new tag from
  @param    plist  The propertylist of the frame
  @param    dummy  A non-negative number (required in the API, but unused here)
  @return   Comparison string or NULL on error
  @note The comparison string must be deallocated with cpl_free().

 */
/*----------------------------------------------------------------------------*/
static char * naco_img_lampflat_make_tag(const cpl_frame* self,
                                         const cpl_propertylist * plist,
                                         int dummy)
{

    char       * tag = NULL;
    const char * filter;
    const char * opti3;
    const char * opti7;
    const char * rom;
    const char * mode;
    double       dit;


    skip_if (cpl_error_get_code());

    skip_if(self  == NULL);
    skip_if(plist == NULL);
    skip_if(dummy < 0); /* Avoid warning of unused variable */


    /* filters */
    filter = naco_pfits_get_filter(plist);
    skip_if(cpl_error_get_code());
 
    /* the objective */
    opti7 = naco_pfits_get_opti7_name(plist);
    skip_if(cpl_error_get_code());
 
    /* the OPTI3_NAME */
    opti3 = naco_pfits_get_opti3_name(plist);
    skip_if(cpl_error_get_code());

    /* the ROM */
    rom = naco_pfits_get_rom_name(plist);
    skip_if(cpl_error_get_code());

    /* the mode */
    mode = naco_pfits_get_mode(plist);
    skip_if(cpl_error_get_code());

    /* Compare the DIT */
    dit  = naco_pfits_get_dit(plist);
    skip_if(cpl_error_get_code());


    tag = cpl_sprintf("%s:%s:%s:%s:%s:%.5f", filter,
                                        opti7, opti3, rom, mode, dit);
    bug_if(tag == NULL);

    end_skip;

    if (cpl_error_get_code()) {
        cpl_free(tag);
        tag = NULL;
    }

    return tag;

}
