/*===========================================================================
  Copyright (C) 2001 European Southern Observatory (ESO)

  This program is free software; you can redistribute it and/or 
  modify it under the terms of the GNU General Public License as 
  published by the Free Software Foundation; either version 2 of 
  the License, or (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public 
  License along with this program; if not, write to the Free 
  Software Foundation, Inc., 675 Massachusetss Ave, Cambridge, 
  MA 02139, USA.

  Corresponding concerning ESO-MIDAS should be addressed as follows:
    Internet e-mail: midas@eso.org
    Postal address: European Southern Observatory
            Data Management Division 
            Karl-Schwarzschild-Strasse 2
            D 85748 Garching bei Muenchen 
            GERMANY
===========================================================================*/
/* Program  : scatter.c                                                    */
/* Author   : G. Mulas  -  ITAL_FLAMES Consortium                          */
/* Date     :                                                              */
/*                                                                         */
/* Purpose  : Missing                                                      */
/*                                                                         */
/*                                                                         */
/* Input:  see interface                                                   */ 
/*                                                                      */
/* Output:                                                              */
/*                                                                         */
/* DRS Functions called:                                                   */
/* none                                                                    */ 
/*                                                                         */ 
/* Pseudocode:                                                             */
/* Missing                                                                 */ 
/*                                                                         */ 
/* Version  :                                                              */
/* Last modification date: 2002/08/05                                      */
/* Who     When        Why                Where                            */
/* AMo     02-08-05   Add header         header                            */
/*-------------------------------------------------------------------------*/

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif


#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <string.h>
#include <flames.h>
#include <flames_midas_def.h>
#include <flames_def_drs_par.h>
#include <flames_scatter.h>
#include <flames_dfs.h>
#include <flames_uves.h>
#include <flames_gauss_jordan.h>
#include <flames_newmatrix.h>
#include <flames_trimback.h>
#include <uves_msg.h>


typedef struct _deviation {
    double zdev;
    int32_t zind;
} deviation;

typedef struct _pixel {
    frame_data value;
    int32_t ix;
    int32_t iy;
} pixel;


static int 
devcomp(const void *zdev1, const void *zdev2);

static int 
pixelcomp(const void *pixel1, const void *pixel2);


flames_err 
scatter(flames_frame *Frame, 
        orderpos *Order, 
        scatterswitch bkgswitch, 
        scatterswitch2 bkgswitch2,
        int badwinxsize, 
        int badwinysize,
        double badfracthres, 
        int badtotthres, 
        double kappa2,
        int32_t maxbackiters, 
        double maxdiscardfract,
        int *OUTPUTI)
{
    flames_err status;

    char output[70];
    int k=0;
    int l=0;
    int i=0;
    int j=0;
    int npar=0;
    int freed_sw=0;
    int qq;
    int* mid_unit=0;

    double di;

    double xscale;
    double yscale;
    double zdiff;
    double **d;
    double **x;

    int drs_bkg_pix;
    int32_t goodpixels;
    int32_t goodpixelsorig;
    int32_t newgoodpixels;
    int32_t niters;
    int32_t i2;
    int32_t j2;

    int32_t goodwindows=0;
    frame_mask windowisgood=0;

    double **xmatrix;
    double **ymatrix;
    double **xmatrixnew;
    double **ymatrixnew;

    double *zvector;
    double *zsigma;
    double *zvectornew;
    double *zsigmanew;
    double *zfitted;





    int32_t maxrejects=0;

    double *xbuf;
    double *xbuf1;
    double *xbuf2;
    double *ybuf;
    double *ybuf1;
    double *ybuf2;

    int32_t *Window_x_low=0;
    int32_t *Window_x_up=0;
    int32_t *Window_y_low=0;
    int32_t *Window_y_up=0;

    pixel *pixellist;

    int32_t medianindex=0;
    int32_t i1=0;
    int32_t j1=0;
    int32_t minix=0;
    int32_t miniy=0;

    frame_data minvalue=0;
    frame_data medianvalue=0;

    double normfactor=0;
    double normweight=0;
    double averix=0;
    double averiy=0;
    double avervalue=0;
    //double aversigma=0;

    frame_data *fdvecbuf1=0;
    frame_data *fdvecbuf2=0;
    frame_mask *fmvecbuf1=0;

    int32_t ijoffset=0;
    int32_t ijindex=0;
    int32_t i1j1index=0;
    int32_t i2j2index=0;
    int actvals=0;
    char drs_verbosity[10];
    int mid_stat=0;


    memset(output, '\0', 70);

    uves_msg_debug("bkgswitch=%d",bkgswitch);
    uves_msg_debug("bkgswitch2=%d",bkgswitch2);
    uves_msg_debug("badwinxsize=%d",badwinxsize);
    uves_msg_debug("badwinysize=%d",badwinysize);
    uves_msg_debug("badfracthres=%f",badfracthres);
    uves_msg_debug("badtotthres=%d",badtotthres);
    uves_msg_debug("kappa2=%f",kappa2);
    uves_msg_debug("maxbackiters=%d",maxbackiters);
    uves_msg_debug("maxdiscardfract=%f",maxdiscardfract);


    fdvecbuf1 = Frame->frame_array[0];
    fdvecbuf2 = Frame->frame_sigma[0];
    fmvecbuf1 = Frame->badpixel[0];


#if FLAMES_DEBUG
    {
        int i;
        for (i = 0; i < 100; i++){
            fprintf(stderr, "%d %f %f %d\n",
                            i,
                            fdvecbuf1[200*50 + i*100],
                            fdvecbuf2[200*50 + i*100],
                            fmvecbuf1[200*50 + i*100]);
        }
    }
#endif



    if (Frame->substepx == 0) {
        SCTPUT("Error: the x step must be non zero");
        SCSEPI();
        return flames_midas_fail();
    }
    if (Frame->substepy == 0) {
        SCTPUT("Error: the y step must be non zero");
        SCSEPI();
        return flames_midas_fail();
    }


    memset(drs_verbosity, 0, 10);
    if ((mid_stat=SCKGETC(DRS_VERBOSITY, 1, 3, &actvals, drs_verbosity))
                    != 0) {
        /* the keyword seems undefined, protest... */
        return(MAREMMA);
    }



    /* trim the background table to avoid pixels containing light from the
    (possibly shifted) fibres */
    sprintf(output,
            "trimming background table from %d windows...",
            Frame->back.Window_Number);
    SCTPUT(output);




    uves_msg_debug("processing frame=%s\n",Frame->framename);
    uves_msg_debug("bkgswitch=%d\n",bkgswitch);
    uves_msg_debug("bkgswitch2=%d\n",bkgswitch2);
    uves_msg_debug("badwinxsize=%d\n", badwinxsize);
    uves_msg_debug("badwinysize=%d\n", badwinysize);
    uves_msg_debug("badfracthres=%f\n", badfracthres);
    uves_msg_debug("badtotthres=%d\n",badtotthres);
    uves_msg_debug("kappa2=%f\n",kappa2);
    uves_msg_debug("maxbackiters=%d\n",maxbackiters);
    uves_msg_debug("maxdiscardfract=%f\n",maxdiscardfract);

    if ((status=trimback(Frame, Order, &Window_x_low, &Window_x_up,
                    &Window_y_low, &Window_y_up, bkgswitch2, badwinxsize,
                    badwinysize, badfracthres, badtotthres))!=NOERR) {
        SCTPUT("Error trimming background table");
        return flames_midas_fail();
    }
    /* remember that trimback allocates the Window_* vectors, we will have
    to free them before returning */
    sprintf(output, "...%d windows left\n", Frame->back.Window_Number);
    SCTPUT(output);

    /* put together the initial data for the fit, now */
    /* we want to have several possible options for this, depending
    on the trade off between speed and quality which we want to 
    achieve */

    /* was neighborhood bad pixel scanning required? */


    goodpixels = 0;

    for (qq=1; qq<=Frame->back.Window_Number; qq++) {
        for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
            ijoffset = i*Frame->subcols;

            for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
                ijindex = ijoffset+j;
                //uves_msg("fmvecbuf1[ijindex]=%d",fmvecbuf1[ijindex]);
                if (fmvecbuf1[ijindex] == 0) {
                    /* good pixel, use it in the fit */
                    //uves_msg("good pix");
                    goodpixels++;
                }
            }
        }
    }

    /* did we find any good pixels? */
    if (goodpixels==0) {
        SCTPUT("Background estimation impossible: no usable pixels!\n");
        return flames_midas_fail();
    }
    npar = (int32_t)((Frame->back.xdegree+1)*(Frame->back.ydegree+1));
    xscale = Frame->subcols>=2 ? (double)Frame->subcols-1 : 1;
    yscale = Frame->subrows>=2 ? (double)Frame->subrows-1 : 1;

    /* we want to have several possible options for this, depending
    on the trade off between speed and quality that we want to 
    achieve */
    switch(bkgswitch) {

    case USEALL:
    default:
        /* use all the pixels in all the windows */
        /* did we find enough pixels? */
        if (goodpixels<npar) {
            SCTPUT("Background estimation impossible1:\n less usable pixels than \
parameters\n");
            return flames_midas_fail();
        }
        sprintf(output, "Using %d pixels for background fitting...\n",goodpixels);
        SCTPUT(output);

        drs_bkg_pix = goodpixels;
        SCKWRI(OUTPUTI,&drs_bkg_pix,4,1,mid_unit);

        xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodpixels);
        ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodpixels);
        zvector = dvector(1,goodpixels);
        zsigma = dvector(1,goodpixels);
        goodpixels = 0;

        for (qq=1; qq<=Frame->back.Window_Number; qq++) {
            for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
                ijoffset = i*Frame->subcols;
                di = ((double) i)/yscale;
                for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
                    ijindex = ijoffset+j;
                    if (fmvecbuf1[ijindex] == 0) {
                        /* good pixel, use it in the fit */
                        goodpixels++;
                        xmatrix[1][goodpixels] = ((double) j)/xscale;
                        ymatrix[1][goodpixels] = di;
                        zvector[goodpixels] = fdvecbuf1[ijindex];
                        zsigma[goodpixels] = fdvecbuf2[ijindex];
                    }
                }
            }
        }
        break;

    case USEMEDIAN:

        /* find the median in each window and use only that */
        /* count the windows containing at least one good pixel */
        goodwindows = 0;
        goodpixels = 0;
        for (qq=1; qq<=Frame->back.Window_Number; qq++) {
            windowisgood = FALSE;
            for (i=Window_y_low[qq]; windowisgood==FALSE && i<=Window_y_up[qq]; i++){
                ijoffset = i*Frame->subcols;
                for (j=Window_x_low[qq]; windowisgood==FALSE && j<=Window_x_up[qq];
                                j++){
                    ijindex = ijoffset+j;
                    if (fmvecbuf1[ijindex]==0) windowisgood = TRUE;
                }
            }
            if (windowisgood == TRUE) {
                goodwindows++;
                newgoodpixels = (Window_y_up[qq]-Window_y_low[qq]+1)*
                                (Window_x_up[qq]-Window_x_low[qq]+1);
                if (newgoodpixels>goodpixels) goodpixels=newgoodpixels;
            }
        }
        /* ok, we can now define the arrays with the correct size */
        /* did we find enough pixels? */

        if (goodwindows<npar) {
            SCTPUT("Background estimation impossible2:\n less usable windows than \
parameters\n");
            return flames_midas_fail();
        }
        pixellist = ((pixel *) calloc((size_t) goodpixels, sizeof(pixel)));
        sprintf(output, "Using %d pixels for background fitting...\n",goodwindows);
        SCTPUT(output);
        drs_bkg_pix = goodwindows;
        SCKWRI(OUTPUTI,&drs_bkg_pix,1,1,mid_unit);


        xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodwindows);
        ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodwindows);
        zvector = dvector(1,goodwindows);
        zsigma = dvector(1,goodwindows);
        goodwindows = 0;
        for (qq=1; qq<=Frame->back.Window_Number; qq++) {
            windowisgood = FALSE;
            goodpixels = 0;
            for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
                ijoffset = i*Frame->subcols;
                for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
                    ijindex = ijoffset+j;
                    if (fmvecbuf1[ijindex] == 0) {
                        /* good pixel, use it in the fit */
                        pixellist[goodpixels].iy = i;
                        pixellist[goodpixels].ix = j;
                        pixellist[goodpixels].value = fdvecbuf1[ijindex];
                        windowisgood = TRUE;
                        goodpixels++;
                    }
                }
            }

            /* is this a good window? */
            if (windowisgood==TRUE) {
                /* to find the median, sort the pixels */
                qsort(pixellist, (size_t) goodpixels, sizeof(pixel), pixelcomp);

                /* now that the pixels are sorted, pick the median */
                if (2*(goodpixels/2) != goodpixels) {
                    /* goodpixels is odd, just pick the right index */
                    medianindex = (goodpixels-1)/2;
                    i = pixellist[medianindex].iy;
                    j = pixellist[medianindex].ix;
                    ijindex = (i*Frame->subcols)+j;
                    goodwindows++;
                    xmatrix[1][goodwindows] = ((double) j)/xscale;
                    ymatrix[1][goodwindows] = ((double) i)/yscale;
                    zvector[goodwindows] = fdvecbuf1[ijindex];
                    zsigma[goodwindows] = fdvecbuf2[ijindex];
                }
                else {
                    /* goodpixels is even, compute the average of the two
        median values */
                    medianindex = goodpixels/2;
                    i1 = pixellist[medianindex-1].iy;
                    j1 = pixellist[medianindex-1].ix;
                    i1j1index = (i1*Frame->subcols)+j1;
                    i2 = pixellist[medianindex].iy;
                    j2 = pixellist[medianindex].ix;
                    i2j2index = (i2*Frame->subcols)+j2;
                    goodwindows++;
                    xmatrix[1][goodwindows] = .5*((double)j1+(double)j2)/xscale;
                    ymatrix[1][goodwindows] = .5*((double)i1+(double)i2)/yscale;
                    zvector[goodwindows] =
                                    .5*(fdvecbuf1[i1j1index]+fdvecbuf1[i2j2index]);
                    zsigma[goodwindows] =
                                    .5*(fdvecbuf2[i1j1index]+fdvecbuf2[i2j2index]);
                }
            }
        }
        free(pixellist);
        goodpixels = goodwindows;
        break;

    case USEMINIMUM:

        /* use the pixel with the minimum value in each window */
        /* count the windows containing at least one good pixel */
        goodwindows = 0;
        goodpixels = 0;
        for (qq=1; qq<=Frame->back.Window_Number; qq++) {
            windowisgood = FALSE;
            for (i=Window_y_low[qq]; windowisgood==FALSE && i<=Window_y_up[qq]; i++) {
                ijoffset = i*Frame->subcols;
                for (j=Window_x_low[qq]; windowisgood==FALSE && j<=Window_x_up[qq];
                                j++) {
                    ijindex = ijoffset+j;
                    if (fmvecbuf1[ijindex]==0) windowisgood = TRUE;
                }
            }
            if (windowisgood == TRUE) goodwindows++;
        }
        /* ok, we can now define the arrays with the correct size */
        /* did we find enough pixels? */
        if (goodwindows<npar) {
            SCTPUT("Background estimation impossible3:\n less usable windows than \
parameters\n");
            return flames_midas_fail();
        }
        sprintf(output, "Using %d pixels for background fitting...\n",goodwindows);
        SCTPUT(output);

        drs_bkg_pix = goodwindows;

        SCKWRI(OUTPUTI,&drs_bkg_pix,3,1,mid_unit);

        xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodwindows);
        ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodwindows);
        zvector = dvector(1,goodwindows);
        zsigma = dvector(1,goodwindows);
        goodwindows = 0;
        for (qq=1; qq<=Frame->back.Window_Number; qq++) {
            windowisgood = FALSE;
            for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
                ijoffset = i*Frame->subcols;
                for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
                    ijindex = ijoffset+j;
                    if (fmvecbuf1[ijindex] == 0) {
                        /* good pixel, use it in the fit */
                        if (windowisgood==FALSE) {
                            /* this is the first good pixel we found in this window */
                            minix = j;
                            miniy = i;
                            minvalue = fdvecbuf1[ijindex];
                            windowisgood = TRUE;
                        }
                        else {
                            /* this is not the first good pixel in this window */
                            if (fdvecbuf1[ijindex]<minvalue) {
                                minix = j;
                                miniy = i;
                                minvalue = fdvecbuf1[ijindex];
                            }
                        }
                    }
                }
            }
            /* is this a good window? */
            if (windowisgood==TRUE) {
                goodwindows++;
                xmatrix[1][goodwindows] = ((double) minix)/xscale;
                ymatrix[1][goodwindows] = ((double) miniy)/yscale;
                ijindex = (miniy*Frame->subcols)+minix;
                zvector[goodwindows] = fdvecbuf1[ijindex];
                zsigma[goodwindows] = fdvecbuf2[ijindex];
            }
        }
        goodpixels = goodwindows;
        break;

    case USEAVERAGE:

        /* find the median in each window and that to sigma-clip other pixels
      in the same window*/
        /* count the windows containing at least one good pixel */
        goodwindows = 0;
        goodpixels = 0;
        for (qq=1; qq<=Frame->back.Window_Number; qq++) {
            windowisgood = FALSE;
            for (i=Window_y_low[qq]; windowisgood==FALSE && i<=Window_y_up[qq]; i++){
                ijoffset = i*Frame->subcols;
                for (j=Window_x_low[qq]; windowisgood==FALSE && j<=Window_x_up[qq];
                                j++) {
                    ijindex = ijoffset+j;
                    if (fmvecbuf1[ijindex]==0) windowisgood = TRUE;
                }
            }
            if (windowisgood == TRUE) {
                goodwindows++;
                newgoodpixels = (Window_y_up[qq]-Window_y_low[qq]+1)*
                                (Window_x_up[qq]-Window_x_low[qq]+1);
                if (newgoodpixels>goodpixels) goodpixels=newgoodpixels;
            }
        }
        /* ok, we can now define the arrays with the correct size */
        /* did we find enough pixels? */

        if (goodwindows<npar) {
            SCTPUT("Background estimation impossible4:\n less usable windows than \
parameters\n");
            return flames_midas_fail();
        }
        pixellist = ((pixel *) calloc((size_t) goodpixels, sizeof(pixel)));
        sprintf(output, "Using %d pixels for background fitting...\n",goodwindows);
        SCTPUT(output);

        drs_bkg_pix = goodwindows;

        SCKWRI(OUTPUTI,&drs_bkg_pix,3,1,mid_unit);

        xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodwindows);
        ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodwindows);
        zvector = dvector(1,goodwindows);
        zsigma = dvector(1,goodwindows);
        goodwindows = 0;
        for (qq=1; qq<=Frame->back.Window_Number; qq++) {
            windowisgood = FALSE;
            goodpixels = 0;
            for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
                ijoffset = i*Frame->subcols;
                for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
                    ijindex = ijoffset+j;
                    if (fmvecbuf1[ijindex] == 0) {
                        /* good pixel, use it in the fit */
                        pixellist[goodpixels].iy = i;
                        pixellist[goodpixels].ix = j;
                        pixellist[goodpixels].value = fdvecbuf1[ijindex];
                        windowisgood = TRUE;
                        goodpixels++;
                    }
                }
            }
            /* is this a good window? */
            if (windowisgood==TRUE) {
                /* to find the median, sort the pixels */
                qsort(pixellist, (size_t) goodpixels, sizeof(pixel), pixelcomp);
                /* now that the pixels are sorted, pick the median */
                if (2*(goodpixels/2) != goodpixels) {
                    /* goodpixels is odd, just pick the right index */
                    medianindex = (goodpixels-1)/2;
                    i = pixellist[medianindex].iy;
                    j = pixellist[medianindex].ix;
                    ijindex = (i*Frame->subcols)+j;
                    medianvalue = fdvecbuf1[ijindex];
                }
                else {
                    /* goodpixels is even, compute the average of the two
        median values */
                    medianindex = goodpixels/2;
                    i1 = pixellist[medianindex-1].iy;
                    j1 = pixellist[medianindex-1].ix;
                    i1j1index = (i1*Frame->subcols)+j1;
                    i2 = pixellist[medianindex].iy;
                    j2 = pixellist[medianindex].ix;
                    i2j2index = (i2*Frame->subcols)+j2;
                    medianvalue = .5*(fdvecbuf1[i1j1index]+fdvecbuf1[i2j2index]);
                }
                /* ok, now build the average with the pixels which do not exceed the
      cutting threshold */       
                normfactor=0;
                averix=0;
                averiy=0;
                avervalue=0;
                //aversigma=0;
                for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
                    ijoffset = i*Frame->subcols;
                    for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
                        ijindex=ijoffset+j;
                        if (fmvecbuf1[ijindex]==0 &&
                                        fabs((double)((fdvecbuf1[ijindex]-medianvalue)/
                                                        fdvecbuf2[ijindex]))<kappa2) {
                            /* good pixel, use it in the fit */
                            normweight = (double) 1/fdvecbuf2[ijindex];
                            averix += normweight*(double)j;
                            averiy += normweight*(double)i;
                            avervalue += normweight*(double)fdvecbuf1[ijindex];
                            normfactor += normweight;
                        }
                    }
                }
                goodwindows++;
                xmatrix[1][goodwindows] = averix/(normfactor*xscale);
                ymatrix[1][goodwindows] = averiy/(normfactor*yscale);
                zvector[goodwindows] = avervalue/normfactor;
                zsigma[goodwindows] = 1/normfactor;
            }
        }
        free(pixellist);
        goodpixels = goodwindows;
        break;
    }

    /* free the Window_* boundary vectors here, we do not need them further */
    free_lvector(Window_x_low, 1, Frame->back.Window_Number);
    free_lvector(Window_x_up, 1, Frame->back.Window_Number);
    free_lvector(Window_y_low, 1, Frame->back.Window_Number);
    free_lvector(Window_y_up, 1, Frame->back.Window_Number);

    xbuf1 = xmatrix[1];
    for (j=2; j<=2*Frame->back.xdegree; j++) {
        xbuf = xmatrix[j];
        xbuf2 = xmatrix[j-1];
        //double dj = (double) j;
        for (qq=1; qq<=goodpixels; qq++)
            xbuf[qq] = xbuf1[qq]*xbuf2[qq];
    }
    ybuf1 = ymatrix[1];
    for (i=2; i<=2*Frame->back.ydegree; i++) {
        ybuf = ymatrix[i];
        ybuf2 = ymatrix[i-1];
        di = (double) i;
        for (qq=1; qq<=goodpixels; qq++)
            ybuf[qq] = ybuf1[qq]*ybuf2[qq];
    }
    goodpixelsorig = newgoodpixels = goodpixels;

    niters = 0;
    x=dmatrix(1,npar,1,1);
    d=dmatrix(1,npar,1,npar);

    /* start the fitting/kappa-sigma-clipping loop */
    do {
        niters++;

        if ( strcmp(drs_verbosity,"LOW") == 0 ){
        } else {

            sprintf(output, "background fitting iteration %d running...", niters);
            SCTPUT(output);

        }
        goodpixels = newgoodpixels;
        maxrejects = (int32_t) ceil(maxdiscardfract*goodpixels);
        if (goodpixels<maxrejects) maxrejects=goodpixels;
        else if (maxrejects<1) maxrejects=1;

        zfitted = dvector(1,goodpixels);
        deviation* zdevs = ((deviation *) calloc((size_t) (goodpixels),  sizeof(deviation)))-1;
        char* zmask = cvector(1,goodpixels);
        for (k=1; k<=npar; k++) {
            x[k][1] = 0;
            for (l=1; l<=npar; l++) {
                d[k][l] = 0;
            }
        }
        /* the first running index is k, the second is l: here k = l = 1 */
        k = l = 1;
        for (qq=1; qq<=goodpixels; qq++) {
            /* i = i2 = j = j2 = 0 */
            x[1][1] += zvector[qq]/zsigma[qq];
            d[1][1] += 1/zsigma[qq];
        }
        /* span all the l>1 values for k = 1; */
        for (j2=1; j2<=Frame->back.xdegree; j2++){
            /* i = i2 = j = 0, j2 > 0 */
            l++;
            xbuf = xmatrix[j2];
            for (qq=1; qq<=goodpixels; qq++)
                d[1][l] += xbuf[qq]/zsigma[qq];
        }
        for (i2=1; i2<=Frame->back.ydegree; i2++){
            /* i = j = 0, i2>0, j2 = 0 */
            l++;
            ybuf = ymatrix[i2];
            for (qq=1; qq<=goodpixels; qq++)
                d[1][l] += ybuf[qq]/zsigma[qq];
            for (j2=1; j2<=Frame->back.xdegree; j2++){
                /* i = j = 0, i2>0, j2>0 */
                l++;
                xbuf = xmatrix[j2];
                for (qq=1; qq<=goodpixels; qq++)
                    d[1][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
            }
        }
        /* run over the k values for which the polynomial has 0 degree in y */
        for (j=1; j<=Frame->back.xdegree; j++){
            /* i = 0, j>0, i2 = 0, j2 = j */
            k++;
            xbuf2 = xmatrix[2*j];
            xbuf = xmatrix[j];
            for (qq=1; qq<=goodpixels; qq++) {
                x[k][1] += xbuf[qq]*zvector[qq]/zsigma[qq];
                d[k][k] += xbuf2[qq]/zsigma[qq];
            }
            /* run over non diagonal terms of d */
            l=k;
            /* first the l values corresponding to 0 degree in y */
            for (j2=j+1; j2<=Frame->back.xdegree; j2++){
                /* i = 0, j>0, i2 = 0, j2>j */
                l++;
                xbuf = xmatrix[j+j2];
                for (qq=1; qq<=goodpixels; qq++)
                    d[k][l] += xbuf[qq]/zsigma[qq];
            }
            /* run over l values corresponding to y degree > 0 */
            for (i2=1; i2<=Frame->back.ydegree; i2++){
                /* i = 0, j > 0, i2>i, j2 = 0 */
                l++;
                xbuf = xmatrix[j];
                ybuf = ymatrix[i2];
                for (qq=1; qq<=goodpixels; qq++)
                    d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
                for (j2=1; j2<=Frame->back.xdegree; j2++){
                    /* i = 0, j>0, i2>i, j2>0 */
                    l++;
                    xbuf = xmatrix[j+j2];
                    for (qq=1; qq<=goodpixels; qq++)
                        d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
                }
            }
        }

        /* run over k values corresponding to y degree > 0 */
        for (i=1; i<=Frame->back.ydegree; i++){
            /* i>0, j = 0, i2=i, j2=0 */
            k++;
            ybuf = ymatrix[i];
            ybuf2 = ymatrix[2*i];
            for (qq=1; qq<=goodpixels; qq++) {
                x[k][1] += ybuf[qq]*zvector[qq]/zsigma[qq];
                d[k][k] += ybuf2[qq]/zsigma[qq];
            }
            l=k;
            for (j2=1; j2<=Frame->back.xdegree; j2++) {
                /* i>0, j=0, i2=i, j2>0 */
                l++;
                xbuf = xmatrix[j2];
                for (qq=1; qq<=goodpixels; qq++)
                    d[k][l] += ybuf2[qq]*xbuf[qq]/zsigma[qq];
            }
            /* run over terms with i2>i */
            for (i2=i+1; i2<=Frame->back.ydegree; i2++){
                /* i>0, j=0, i2>i, j2=0 */
                l++;
                ybuf = ymatrix[i+i2];
                for (qq=1; qq<=goodpixels; qq++)
                    d[k][l] += ybuf[qq]/zsigma[qq];
                for (j2=1; j2<=Frame->back.xdegree; j2++){
                    /* i>0, j=0, i2>i, j2>0 */
                    l++;
                    xbuf = xmatrix[j2];
                    for (qq=1; qq<=goodpixels; qq++)
                        d[k][l] += ybuf[qq]*xbuf[qq]/zsigma[qq];
                }
            }
            for (j=1; j<=Frame->back.xdegree; j++){
                /* i>0, j>0, i2=i, j2=j */
                k++;
                xbuf = xmatrix[j];
                xbuf2 = xmatrix[2*j];
                ybuf = ymatrix[i];
                ybuf2 = ymatrix[2*i];
                for (qq=1; qq<=goodpixels; qq++) {
                    x[k][1] += ybuf[qq]*xbuf[qq]*zvector[qq]/zsigma[qq];
                    d[k][k] += xbuf2[qq]*ybuf2[qq]/zsigma[qq];
                }
                l=k;
                for (j2=j+1; j2<=Frame->back.xdegree; j2++){
                    /* i>0, j>0, i2=i, j2>j */
                    l++;
                    xbuf = xmatrix[j+j2];
                    for (qq=1; qq<=goodpixels; qq++)
                        d[k][l] += xbuf[qq]*ybuf2[qq]/zsigma[qq];
                }
                for (i2=i+1; i2<=Frame->back.ydegree; i2++){
                    /* i>0, j>0, i2>i, j2=0 */
                    l++;
                    xbuf = xmatrix[j];
                    ybuf = ymatrix[i+i2];
                    for (qq=1; qq<=goodpixels; qq++)
                        d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
                    for (j2=1; j2<=Frame->back.xdegree; j2++){
                        /* i>0, j>0, i2>i, j2>0 */
                        l++;
                        xbuf = xmatrix[j+j2];
                        for (qq=1; qq<=goodpixels; qq++)
                            d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
                    }
                }
            }
        }

        /* now take advantage of the simmetry */
        for (k=1; k<=npar; k++) {
            for (l=k+1; l<=npar; l++) {
                d[l][k] = d[k][l];
            }
        }

        /* perform the actual fit */
        flames_gauss_jordan(d,npar,x,1);

        /* compute the fitted points on the whole points set */
        /* zero degree terms first */
        /* i = j = 0 */
        for (qq=1; qq<=goodpixels; qq++) zfitted[qq] = x[1][1];
        k=1;
        for (j=1; j<=Frame->back.xdegree; j++) {
            /* i = 0, j>0 */
            k++;
            xbuf = xmatrix[j];
            for(qq=1; qq<=goodpixels; qq++) zfitted[qq] += x[k][1]*xbuf[qq];
        }
        for (i=1; i<=Frame->back.ydegree; i++) {
            /* i>0, j=0 */
            k++;
            ybuf = ymatrix[i];
            for (qq=1; qq<=goodpixels; qq++) zfitted[qq] += x[k][1]*ybuf[qq];
            for (j=1; j<=Frame->back.xdegree; j++) {
                /* i>0, j>0 */
                k++;
                xbuf = xmatrix[j];
                for (qq=1; qq<=goodpixels; qq++)
                    zfitted[qq] += x[k][1]*xbuf[qq]*ybuf[qq];
            }
        }
        /* select and count the pixels which pass the sigma clipping */
        /* compute normalised deviations */
        for (qq=1; qq<=goodpixels; qq++) {
            zdiff = zfitted[qq]-zvector[qq];
            zdevs[qq].zdev = (zdiff*zdiff)/(kappa2*zsigma[qq]);
            zdevs[qq].zind = qq;
        }
        /* sort deviations */
        qsort(zdevs+1, (size_t) goodpixels, sizeof(deviation), devcomp);
        /* initialise the whole mask first */
        for (qq=1; qq<=goodpixels; qq++) zmask[qq]=TRUE;
        /* now run over the worst pixels and mark the bad ones */
        newgoodpixels = goodpixels;
        for (qq=1; (qq<=maxrejects)&&(zdevs[qq].zdev>1); qq++) {
            zmask[zdevs[qq].zind]=FALSE;
            newgoodpixels--;
        }
        /* build the new arrays with the remaining good pixels */
        if (newgoodpixels != goodpixels) {

            if ( strcmp(drs_verbosity,"LOW") == 0 ){
            } else {

                sprintf(output, "sigma-clipped %d pixels on %d",
                                goodpixels-newgoodpixels, goodpixels);
                SCTPUT(output);
            }
            /* allocate the new arrays */
            xmatrixnew = dmatrix(1,2*Frame->back.xdegree,1,newgoodpixels);
            ymatrixnew = dmatrix(1,2*Frame->back.ydegree,1,newgoodpixels);
            zvectornew = dvector(1,newgoodpixels);
            zsigmanew = dvector(1,newgoodpixels);
            newgoodpixels = 0;
            for (qq=1; qq<=goodpixels; qq++) {
                if (zmask[qq] == TRUE) {
                    newgoodpixels++;
                    for (i=1; i<=2*Frame->back.ydegree; i++)
                        ymatrixnew[i][newgoodpixels] = ymatrix[i][qq];
                    for (j=1; j<=2*Frame->back.xdegree; j++)
                        xmatrixnew[j][newgoodpixels] = xmatrix[j][qq];
                    zvectornew[newgoodpixels] = zvector[qq];
                    zsigmanew[newgoodpixels] = zsigma[qq];
                }
            }

            /* free the old arrays */
            free_dmatrix(xmatrix,1,2*Frame->back.xdegree,1,goodpixels);
            xmatrix = xmatrixnew;
            free_dmatrix(ymatrix,1,2*Frame->back.ydegree,1,goodpixels);
            ymatrix = ymatrixnew;
            free_dvector(zvector,1,goodpixels);
            zvector = zvectornew;
            free_dvector(zsigma,1,goodpixels);
            zsigma = zsigmanew;

            free_dvector(zfitted,1,goodpixels);
            free(zdevs+1);
            free_cvector(zmask,1,goodpixels);
            freed_sw=1;
        }
        if(freed_sw==0) {
            free(zdevs+1);
            free_dvector(zfitted,1,goodpixels);
            free_cvector(zmask,1,goodpixels);
        }

        if ( strcmp(drs_verbosity,"LOW") == 0 ){
        } else {
            sprintf(output, "background fitting iteration %d finished\n", niters);
            SCTPUT(output);
        }
    } while ((newgoodpixels!=goodpixels) && (niters<maxbackiters));


    /* free the arrays holding the points to be used for the fit */
    free_dmatrix(xmatrix, 1,2*Frame->back.xdegree,1,goodpixels);
    free_dmatrix(ymatrix, 1,2*Frame->back.xdegree,1,goodpixels);
    free_dvector(zvector, 1,goodpixels);
    free_dvector(zsigma, 1,goodpixels);

    /* the following check should be suppressed, else code crushes with double delete of same pointer
 if(freed_sw != 1) free_dvector(zfitted, 1,goodpixels);
 if(freed_sw != 1) free_cvector(zmask, 1,goodpixels);
 if(freed_sw != 1) free(zdevs+1);
     */

    if(!(Frame->back.coeff)) Frame->back.coeff=dvector(1,npar);

    for (k=1; k<=npar; k++) Frame->back.coeff[k]=x[k][1];

    /* free locally allocated arrays */
    free_dmatrix(x,1,npar,1,1);
    free_dmatrix(d,1,npar,1,npar);

    return 0;
}


static int 
devcomp(const void *d1, const void *d2)
{
    const deviation *zdev1= (const deviation*) d1;
    const deviation *zdev2= (const deviation*) d2;
    if ((*zdev1).zdev > (*zdev2).zdev) return -1;
    else if ((*zdev1).zdev < (*zdev2).zdev) return 1;
    else return 0;
}

static int 
pixelcomp(const void *p1, const void *p2)
{
    const pixel* pixel1 = (const pixel*) p1;
    const pixel* pixel2 = (const pixel*) p2;

    if (pixel1->value > pixel2->value) return -1;
    else if (pixel1->value < pixel2->value) return 1;
    else return 0;
}
