/*                                                                              *
 *   This file is part of the ESO UVES Pipeline                                 *
 *   Copyright (C) 2004,2005 European Southern Observatory                      *
 *                                                                              *
 *   This library is free software; you can redistribute it and/or modify       *
 *   it under the terms of the GNU General Public License as published by       *
 *   the Free Software Foundation; either version 2 of the License, or          *
 *   (at your option) any later version.                                        *
 *                                                                              *
 *   This program is distributed in the hope that it will be useful,            *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of             *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
 *   GNU General Public License for more details.                               *
 *                                                                              *
 *   You should have received a copy of the GNU General Public License          *
 *   along with this program; if not, write to the Free Software                *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston MA 02110-1301 USA          *
 */
 
/*
 * $Author: amodigli $
 * $Date: 2013-08-08 13:36:46 $
 * $Revision: 1.19 $
 * $Name: not supported by cvs2svn $
 * $Log: not supported by cvs2svn $
 * Revision 1.18  2013/07/01 15:36:08  amodigli
 * Rename DEBUG to debug_mode to remove compiler error on some platforms (that name is reserved to special compiler options)
 *
 * Revision 1.17  2012/03/02 16:23:45  amodigli
 * fixed compiler warnings related to CPL6 upgrade
 *
 * Revision 1.16  2011/12/08 13:59:05  amodigli
 * Fox warnings with CPL6
 *
 * Revision 1.15  2010/09/24 09:32:02  amodigli
 * put back QFITS dependency to fix problem spot by NRI on FIBER mode (with MIDAS calibs) data
 *
 * Revision 1.13  2008/02/15 12:43:49  amodigli
 * allow lower/upper chip for parameter process_chip
 *
 * Revision 1.12  2007/10/05 16:01:44  amodigli
 * using proces_chip parameter to process or not a given RED chip
 *
 * Revision 1.11  2007/06/22 09:28:24  jmlarsen
 * Changed interface of uves_save_image
 *
 * Revision 1.10  2007/06/11 13:28:26  jmlarsen
 * Changed recipe contact address to cpl at eso.org
 *
 * Revision 1.9  2007/06/08 13:06:16  jmlarsen
 * Send bug reports to Andrea
 *
 * Revision 1.8  2007/06/06 08:17:33  amodigli
 * replace tab with 4 spaces
 *
 * Revision 1.7  2007/05/22 11:29:53  jmlarsen
 * Changed text
 *
 * Revision 1.6  2007/05/14 08:09:48  amodigli
 * updated input frames and tag description in recipe man page
 *
 * Revision 1.5  2007/04/24 12:50:29  jmlarsen
 * Replaced cpl_propertylist -> uves_propertylist which is much faster
 *
 * Revision 1.4  2007/03/05 10:14:56  jmlarsen
 * Support slope parameter in 1d fitting
 *
 * Revision 1.3  2007/02/16 10:36:15  jmlarsen
 * Renamed variable y0->y_0
 *
 * Revision 1.2  2007/02/09 13:36:32  jmlarsen
 * Use defines for recipe id
 *
 * Revision 1.1  2007/02/08 11:38:37  jmlarsen
 * Added cd_align recipe
 *
 * Revision 1.31  2007/01/10 12:37:39  jmlarsen
 * Removed obsolete comments
 *
 */

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

/*----------------------------------------------------------------------------*/
/**
 * @defgroup uves_cdalign  Recipe: Cross Disperser alignment
 *
 * This recipe measures the repeatability of the Cross-Disperser position
 * See man-page for details.
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
#include <uves_cd_align_impl.h>

#include <uves.h>
#include <uves_plot.h>
#include <uves_parameters.h>
#include <uves_dfs.h>
#include <uves_pfits.h>
#include <uves_qclog.h>
#include <uves_recipe.h>
#include <uves_utils_cpl.h>
#include <uves_utils_wrappers.h>
#include <uves_error.h>
#include <uves_msg.h>

#include <cpl.h>

/*-----------------------------------------------------------------------------
                            Functions prototypes
 -----------------------------------------------------------------------------*/

static int
uves_cal_cd_align_define_parameters(cpl_parameterlist *parameters);

/*-----------------------------------------------------------------------------
                            Recipe standard code
 -----------------------------------------------------------------------------*/
#define cpl_plugin_get_info uves_cal_cd_align_get_info
UVES_RECIPE_DEFINE(
    UVES_CD_ALIGN_ID, UVES_CD_ALIGN_DOM, uves_cal_cd_align_define_parameters,
    "Jonas M. Larsen", "cpl@eso.org",
    "Measures the reproducability of the cross disperser positioning",
    "Given two input frames (CD_ALIGN_xxx where xxx = BLUE or RED) which contain only\n"
    "one echelle order, this recipe measures the shift in the cross-dispersion \n"
    "direction of that order. For RED input frames, only the lower chip is processed.\n"
    "\n"
    "The recipe produces a CD_ALIGN_TABLE_xxxx (with xxxx = BLUE or REDL) with columns\n"
    "X:         Column number\n"
    "YCENi:     Centroid from Gaussian fit (for i = 1,2)\n"
    "SIGMAi:    Stdev from Gaussian fit\n"
    "BACKi:     Constant background from Gaussian fit\n"
    "NORMi:     Normalization constant from Gaussian fit\n"
    "YDIFF:     Difference YCEN2 - YCEN1 of centroid positions\n"
    "\n"
    "and the QC-parameters ESO.QC.YDIFF(AVG|MED|RMS), which are the average,\n"
    "median and root-mean-square of the y-shift, respectively.\n");

/*-----------------------------------------------------------------------------
                            Functions code
 -----------------------------------------------------------------------------*/
/**@{*/
static int
uves_cal_cd_align_define_parameters(cpl_parameterlist *parameters)
{
    const char *subcontext = NULL;
    const char *recipe_id = make_str(UVES_CD_ALIGN_ID);

    /*****************
     *    General    *
     *****************/
    if (uves_define_global_parameters(parameters) != CPL_ERROR_NONE)
        {
            return -1;
        }
    
    /* stepsize */
    uves_par_new_range("steps",
               CPL_TYPE_INT,
               "Step size in pixels",
               100, 1, INT_MAX);
    
    /* xborder */
    uves_par_new_range("xborder",
               CPL_TYPE_INT,
               "Exclude a border region of this size (pixels)",
               200, 0, INT_MAX);
    
    /* window */
    uves_par_new_range("window",
               CPL_TYPE_INT,
               "The half window height used for Gaussian fitting",
               50, 1, INT_MAX);
    
    return (cpl_error_get_code() != CPL_ERROR_NONE);
}

/*----------------------------------------------------------------------------*/
/**
  @brief    calculates the average Y difference of two CD alignment test frames.
  @param    im1          first CD alignment frames
  @param    im2          second CD alignment frames
  @param    rotated_header1  first image header
  @param    rotated_header2  second image header
  @param    steps        step size (pixels)
  @param    xborder      exclude this border region from the measurement (pixels)
  @param    window       fitting window height is (2*window+1)
  @param    debug_mode        save images to disk?
  @param    chip         CCD chip
  @return  CD align table

  Adopted from uves_cal_cdalign.prg, Burkhard Wolff, ESO-DMD, December 2004,
  contains part of a MIDAS script by Andreas Kaufer
 */
/*----------------------------------------------------------------------------*/
cpl_table *
uves_cd_align_process(const cpl_image *im1,
    const cpl_image *im2,
    const uves_propertylist *rotated_header1,
    const uves_propertylist *rotated_header2,
    int steps,
    int xborder,
    int window,
    bool debug_mode,
    enum uves_chip chip)
{
    cpl_table *result = NULL;
    int row = 0;             /* number of table rows used */
    const cpl_image *images[2];
    cpl_image *rows = NULL;
    cpl_size max_row[2];          /* image row with max flux */
    int nx, ny, x;
    cpl_size num_fits, fit_succeeded;

    images[0] = im1;
    images[1] = im2;    
    nx = cpl_image_get_size_x(images[0]);
    ny = cpl_image_get_size_y(images[0]);
    
    if (debug_mode) check( uves_save_image_local("CD alignment frame", "cd_align1", 
                        images[0], chip, -1, -1, 
                        rotated_header1, true),
              "Error saving 1st CD aligment frame");
    
    if (debug_mode) check( uves_save_image_local("CD alignment frame", "cd_align2", 
                        images[1], chip, -1, -1, 
                        rotated_header2, true),
              "Error saving 2nd CD aligment frame");
    
    assure( cpl_image_get_size_x(images[0]) == cpl_image_get_size_x(images[1]) &&
        cpl_image_get_size_y(images[0]) == cpl_image_get_size_y(images[1]),
        CPL_ERROR_INCOMPATIBLE_INPUT,
        "Images sizes: %" CPL_SIZE_FORMAT "x%" CPL_SIZE_FORMAT " and %" CPL_SIZE_FORMAT "x%" CPL_SIZE_FORMAT "",
        cpl_image_get_size_x(images[0]),
        cpl_image_get_size_y(images[0]),
        cpl_image_get_size_x(images[1]),
        cpl_image_get_size_y(images[1]) );


    result = cpl_table_new(nx); row = 0;
    cpl_table_new_column(result, "X"    , CPL_TYPE_INT);
    cpl_table_new_column(result, "YCEN1", CPL_TYPE_DOUBLE);
    cpl_table_new_column(result, "YCEN2", CPL_TYPE_DOUBLE);
    cpl_table_new_column(result, "SIGMA1", CPL_TYPE_DOUBLE);
    cpl_table_new_column(result, "SIGMA2", CPL_TYPE_DOUBLE);
    cpl_table_new_column(result, "BACK1", CPL_TYPE_DOUBLE);
    cpl_table_new_column(result, "BACK2", CPL_TYPE_DOUBLE);
    cpl_table_new_column(result, "NORM1", CPL_TYPE_DOUBLE);
    cpl_table_new_column(result, "NORM2", CPL_TYPE_DOUBLE);

    cpl_table_set_column_unit(result,"X","pix");
    cpl_table_set_column_unit(result,"YCEN1","pix");
    cpl_table_set_column_unit(result,"YCEN2","pix");
    cpl_table_set_column_unit(result,"SIGMA1","pix");
    cpl_table_set_column_unit(result,"SIGMA2","pix");
    cpl_table_set_column_unit(result,"BACK1","ADU");
    cpl_table_set_column_unit(result,"BACK2","ADU");
    cpl_table_set_column_unit(result,"NORM1","ADU");
    cpl_table_set_column_unit(result,"NORM2","ADU");

    assure_mem( result );
    
    /* Find row of max accumulated flux (i.e. position of the order) */
    {
    int im;
    for (im = 0; im < 2; im++)
        {
        int direction = 1; /* To get image of single column */
        cpl_size max_col;
        
        uves_free_image(&rows);
        rows = cpl_image_collapse_create(images[im], direction);

        cpl_image_get_maxpos(rows, &max_col, &(max_row[im]));
        uves_msg("Row of max flux (%" CPL_SIZE_FORMAT ". image) = %" CPL_SIZE_FORMAT "", (cpl_size)im+1, max_row[im]);

        assure( max_col == 1, CPL_ERROR_ILLEGAL_OUTPUT,
            "Something went wrong, max_col in collapsed image is = %" CPL_SIZE_FORMAT "", max_col);
        }
    }


    num_fits = 0;         /* Number of measure points */
    fit_succeeded = 0;   /* Number of successful Gauss fits */
    for (x = 1 + xborder; x <= nx - xborder; x += steps)
    {
        int im;
        for (im = 0; im < 2; im++)
        {
            bool horizontal = false;
            bool fix_background = false;
            bool fit_background = false;
            int number_of_parameters = 4;
            double y_0, sigma, norm, background;
            int ylow  = uves_max_int(1, uves_min_int(ny, max_row[im] - window));
            int yhigh = uves_max_int(1, uves_min_int(ny, max_row[im] + window));
        
            uves_fit_1d_image(images[im], 
                      NULL, NULL, /* errors, bpm */
                      horizontal, fix_background, fit_background,
                      ylow, yhigh, x,
                      &y_0, &sigma, &norm, &background, NULL, /* slope */
                      NULL, NULL, /* mse, red_chisq */
                      NULL,       /* Covariance */
                      uves_gauss, uves_gauss_derivative, 
                      number_of_parameters);

            num_fits += 1;
            if (cpl_error_get_code() == CPL_ERROR_CONTINUE)
            {
                uves_error_reset();
                
                uves_msg_warning("Fitting window (%" CPL_SIZE_FORMAT ", %" CPL_SIZE_FORMAT ") - (%" CPL_SIZE_FORMAT ", %" CPL_SIZE_FORMAT ") failed",
                                 (cpl_size)x, (cpl_size)ylow, 
                                 (cpl_size)x, (cpl_size)yhigh);
            }
            else
            {
                fit_succeeded += 1;

                assure( cpl_error_get_code() == CPL_ERROR_NONE,
                    cpl_error_get_code(),
                    "Gaussian fitting failed");
                
                cpl_table_set_int   (result, "X"    , row, x);
                cpl_table_set_double(result, (im == 0) ? "YCEN1" : "YCEN2", row, y_0);
                cpl_table_set_double(result, (im == 0) ? "SIGMA1": "SIGMA2", row, sigma);
                cpl_table_set_double(result, (im == 0) ? "BACK1" : "BACK2", row, norm);
                cpl_table_set_double(result, (im == 0) ? "NORM1" : "NORM2", row, background);
            }
        }
        row++;
    }

    cpl_table_set_size(result, row);

    uves_msg_low("Was able to fit %" CPL_SIZE_FORMAT " of %" CPL_SIZE_FORMAT " columns", fit_succeeded, num_fits);

    check(( cpl_table_duplicate_column(result, "YDIFF", result, "YCEN2"),
        cpl_table_subtract_columns(result, "YDIFF", "YCEN1")),
      "Error calculating residuals of fit");
    cpl_table_set_column_unit(result,"YDIFF","pix");
    {
    cpl_size num_valid = cpl_table_get_nrow(result) - cpl_table_count_invalid(result, "YDIFF");
    
    assure( num_valid >= 1, CPL_ERROR_ILLEGAL_OUTPUT,
        "Only %" CPL_SIZE_FORMAT " valid YDIFF value(s), 1 or more needed",
        num_valid);
    }
    

  cleanup:
    uves_free_image(&rows);
    return result;
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Compute QC
  @param    cdalign          result table
  @param    raw_header       input FITS header
  @param    chip             CCD chip
  @return   QC table

 */
/*----------------------------------------------------------------------------*/

static cpl_table*
cd_align_qclog(const cpl_table *cdalign,
           const uves_propertylist *raw_header,
           enum uves_chip chip)
{
    cpl_table *qclog = NULL;
    double mean, sigma, median;

    check( qclog = uves_qclog_init(raw_header, chip),
       "Error during QC initialization");
    
    mean   = cpl_table_get_column_mean  (cdalign, "YDIFF");
    sigma  = cpl_table_get_column_stdev (cdalign, "YDIFF");
    median = cpl_table_get_column_median(cdalign, "YDIFF");

    uves_qclog_add_string(qclog,
              "QC TEST1 ID",
              "Test-of-CD-Alignment",
              "Name of QC test",
              "%s");
    
    uves_qclog_add_double(qclog,
              "QC YDIFFAVG",
              mean,
              "Average Y difference",
              "%8.4f");
  
    uves_qclog_add_double(qclog,
              "QC YDIFFMED",
              median,
              "Median Y difference",
              "%8.4f");

    uves_qclog_add_double(qclog,
              "QC YDIFFRMS",
              sigma,
              "RMS Y difference",
              "%8.4f");
  

    uves_msg("Average shift = %.4f +- %.4f pixels",
         mean, sigma);
         

  cleanup:
    return qclog;
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Show flux in image after bias subtraction
  @param    parameters  the parameters list
  @param    frames      the frames list
  @return   CPL_ERROR_NONE if everything is ok

 */
/*----------------------------------------------------------------------------*/
static double
avg_flux(const cpl_image *im)
{
    double result = 0;
    cpl_image *median_filt = NULL;
    bool extrapolate_border = true;
    
    /* Report total flux after bias subtraction.
       Bias is estimated as the median value
       
       Note that: total flux  -  nx*ny*median =
       nx*ny(mean - median)
       
       so just report (mean - median)
    */

    /* First apply a small window (3x3) median filter to
       get a bit robust avg, but without destroying the echelle order signal
    */
    
    median_filt = cpl_image_duplicate(im);
    assure_mem( median_filt );

    uves_filter_image_median(&median_filt, 1, 1,
                 extrapolate_border);

    result =
    cpl_image_get_mean  (median_filt) -
    cpl_image_get_median(median_filt);
    
  cleanup:
    uves_free_image(&median_filt);
    return result;
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Get the command line options and execute the data reduction
  @param    parameters  the parameters list
  @param    frames      the frames list
  @return   CPL_ERROR_NONE if everything is ok

 */
/*----------------------------------------------------------------------------*/
static void
uves_cal_cd_align_exe(cpl_frameset *frames, const cpl_parameterlist *parameters,
         const char *starttime)
{
    /* Input */
    cpl_image *raw_images[2][2] = {{NULL, NULL}, {NULL, NULL}};
    
    /* This 2x2 array of images contain:
       
       raw_images[0][0]:  First frame, REDL or BLUE chip 
       raw_images[0][1]:  First frame, REDU or NULL
       raw_images[1][0]:  Second frame, REDL or BLUE chip 
       raw_images[1][1]:  Second frame, REDU or NULL

       etc. for the following arrays
    */

    uves_propertylist *raw_headers[2][2]     = {{NULL, NULL}, {NULL, NULL}};
    uves_propertylist *rotated_headers[2][2] = {{NULL, NULL}, {NULL, NULL}};

    cpl_table* qclog[2] = {NULL, NULL};

    /* Output */
    uves_propertylist *product_header = NULL;
    uves_propertylist *table_header = NULL;
    cpl_table *cd_align = NULL;
    
    /* Parameters */
    int steps, xborder, window;
    bool debug_mode;

    /* Local variables */
    const char *product_filename = NULL;
    bool blue;
    enum uves_chip chip;
    const char *raw_filename[2];
    int raw_index;

    const char* PROCESS_CHIP=NULL;

    check( uves_get_parameter(parameters, NULL, "uves", "debug", 
                  CPL_TYPE_BOOL, &debug_mode), "Could not read parameter");

    check( uves_get_parameter(parameters, NULL, "uves", "process_chip", CPL_TYPE_STRING, &PROCESS_CHIP),
               "Could not read parameter");
    uves_string_toupper((char*)PROCESS_CHIP);

    check( uves_get_parameter(parameters, NULL, make_str(UVES_CD_ALIGN_ID), "steps",
                  CPL_TYPE_INT   , &steps), "Could not read parameter");
    check( uves_get_parameter(parameters, NULL, make_str(UVES_CD_ALIGN_ID), "xborder",
                  CPL_TYPE_INT   , &xborder), "Could not read parameter");
    check( uves_get_parameter(parameters, NULL, make_str(UVES_CD_ALIGN_ID), "window",
                  CPL_TYPE_INT   , &window), "Could not read parameter");
    

    check( uves_load_cd_align(frames,
                  &raw_filename[0], 
                  &raw_filename[1], 
                  raw_images[0],
                  raw_images[1],
                  raw_headers[0],
                  raw_headers[1],
                  rotated_headers[0],
                  rotated_headers[1],
                  &blue), 
       "Error loading raw frame");

    uves_msg("Using %s", raw_filename[0]);
    uves_msg("Using %s", raw_filename[1]);

    
    if (blue)
    {
        chip = UVES_CHIP_BLUE;
    }
    else
    {
        if (debug_mode)
        {
            int raw_index_l = uves_chip_get_index(UVES_CHIP_REDL);
            int raw_index_u = uves_chip_get_index(UVES_CHIP_REDU);
            
            uves_msg("1. REDL average flux per pixel = %f ADU", avg_flux(raw_images[0][raw_index_l]));
            uves_msg("2. REDL average flux per pixel = %f ADU", avg_flux(raw_images[1][raw_index_l]));
            
            uves_msg("1. REDU average flux per pixel = %f ADU", avg_flux(raw_images[0][raw_index_u]));
            uves_msg("2. REDU average flux per pixel = %f ADU", avg_flux(raw_images[1][raw_index_u]));
        }
        
        chip = UVES_CHIP_REDL; /* Process only lower red chip */
    }
    
    raw_index = uves_chip_get_index(chip);
    
    uves_msg("Processing %s chip",
         uves_chip_tostring_upper(chip));
    
    check( cd_align = uves_cd_align_process(raw_images[0][raw_index],
                  raw_images[1][raw_index],
                  rotated_headers[0][raw_index],
                  rotated_headers[1][raw_index],
                  steps,
                  xborder,
                  window,
                  debug_mode,
                  chip),
       "Error during processing");

    check( qclog[0] = cd_align_qclog(cd_align,
                     raw_headers[0][raw_index], /* of first frame */
                     chip),
       "Could not compute QC");

    product_header = uves_propertylist_new();

    product_filename = uves_cd_align_filename(chip);
    check( uves_frameset_insert(frames,
                cd_align,
                CPL_FRAME_GROUP_PRODUCT,
                CPL_FRAME_TYPE_TABLE,
                CPL_FRAME_LEVEL_FINAL,
                product_filename,
                UVES_CD_ALIGN_TABLE(blue),
                raw_headers[0][raw_index],
                product_header,
                NULL,       /* table header */
                parameters,
                make_str(UVES_CD_ALIGN_ID),
                PACKAGE "/" PACKAGE_VERSION,
                qclog, /* No QC */
                starttime, true,
                0),
       "Could not add CD align table %s to frameset", product_filename);

    uves_msg("CD align table %s (%s) added to frameset",
         product_filename, UVES_CD_ALIGN_TABLE(blue));

  cleanup:
    uves_free_image(&raw_images[0][0]);
    uves_free_image(&raw_images[0][1]);
    uves_free_image(&raw_images[1][0]);
    uves_free_image(&raw_images[1][1]);
    uves_free_propertylist(&raw_headers[0][0]);
    uves_free_propertylist(&raw_headers[0][1]);
    uves_free_propertylist(&raw_headers[1][0]);
    uves_free_propertylist(&raw_headers[1][1]);
    uves_free_propertylist(&rotated_headers[0][0]);
    uves_free_propertylist(&rotated_headers[0][1]);
    uves_free_propertylist(&rotated_headers[1][0]);
    uves_free_propertylist(&rotated_headers[1][1]);

    uves_free_table(&qclog[0]);
    uves_free_string_const(&product_filename);
    uves_free_table(&cd_align);
    uves_free_propertylist(&product_header);

    return;
}


/**@}*/
