vircam_linearity_analyse.c

00001 /* $Id: vircam_linearity_analyse.c,v 1.65 2012/01/16 12:32:18 jim Exp $
00002  *
00003  * This file is part of the VIRCAM Pipeline
00004  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: jim $
00023  * $Date: 2012/01/16 12:32:18 $
00024  * $Revision: 1.65 $
00025  * $Name: vcam-1_3_0 $
00026  */
00027 
00028 /* Includes */
00029 
00030 #ifdef HAVE_CONFIG_H
00031 #include <config.h>
00032 #endif
00033 
00034 #include <stdio.h>
00035 #include <cpl.h>
00036 #include <math.h>
00037 
00038 #include "vircam_utils.h"
00039 #include "vircam_pfits.h"
00040 #include "vircam_dfs.h"
00041 #include "vircam_mods.h"
00042 #include "vircam_channel.h"
00043 #include "vircam_stats.h"
00044 #include "vircam_paf.h"
00045 
00046 /* Function prototypes */
00047 
00048 static int vircam_linearity_analyse_create(cpl_plugin *) ;
00049 static int vircam_linearity_analyse_exec(cpl_plugin *) ;
00050 static int vircam_linearity_analyse_destroy(cpl_plugin *) ;
00051 static int vircam_linearity_analyse(cpl_parameterlist *, cpl_frameset *) ;
00052 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
00053                                             cpl_parameterlist *parlist);
00054 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
00055                                          cpl_parameterlist *parlist);
00056 static int vircam_linearity_analyse_domedark_groups(void);
00057 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
00058                                                 parquet *p, int np);
00059 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
00060                                          int nchan, double *facrng, 
00061                                          double *maxdiff);
00062 static void vircam_mjdsort(double **fdata, double *mjd, int n);
00063 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows);
00064 static void vircam_linearity_analyse_init(void);
00065 static void vircam_linearity_analyse_tidy(int level);
00066 
00067 /* Static global variables */
00068 
00069 static struct {
00070 
00071     /* Input */
00072 
00073     int     norder;
00074     float   lthr;
00075     float   hthr;
00076     int     maxbpmfr;
00077     int     adjust;
00078     int     diagnostic;
00079     int     extenum;
00080 
00081     /* Output */
00082 
00083     float   linearity;
00084     float   linerror;
00085     float   bad_pixel_stat;
00086     int     bad_pixel_num;
00087     float   facrng;
00088     float   maxdiff;
00089 
00090 } vircam_linearity_analyse_config;
00091 
00092 typedef struct {
00093     cpl_frameset   *darks;
00094     cpl_frameset   *domes;
00095     int            ndarks;
00096     int            ndomes;
00097     float          exptime;
00098     unsigned char  flag;
00099     vir_fits       **proc;
00100 } ddgrp;
00101 
00102 #define OK_FLAG       0
00103 #define SATURATE_FLAG 1
00104 
00105 #define SUBSET 128
00106 #define SUBSET2 (SUBSET/2)
00107 
00108 static struct {
00109     cpl_size         *labels;
00110     cpl_frameset     *domelist;
00111     int              ndomes;
00112     cpl_frameset     *darklist;
00113     int              ndarks;
00114     cpl_frameset     *domecheck;
00115     int              ndomecheck;
00116     cpl_frameset     *darkcheck;
00117     int              ndarkcheck;
00118     cpl_frame        *inherit;
00119     cpl_frame        *chanfrm;
00120     vir_tfits        *chantab;
00121     cpl_table        *lchantab;
00122     cpl_array        *bpm_array;
00123     ddgrp            *ddg;
00124     int              nddg;
00125     vir_fits         **flatlist;
00126     int              nflatlist;
00127     cpl_propertylist *plist;
00128     cpl_propertylist *elist;
00129     int              nx;
00130     int              ny;
00131     cpl_propertylist *phupaf;
00132     cpl_table        *diag1;
00133     cpl_table        *diag2;
00134     int              nfdata;
00135     int              nuse;
00136 } ps;
00137 
00138 static int isfirst;
00139 static int dummy;
00140 static float sat;
00141 static cpl_frame *product_frame_chantab = NULL;
00142 static cpl_frame *product_frame_bpm = NULL;
00143 static cpl_frame *product_frame_diag1 = NULL;
00144 static cpl_frame *product_frame_diag2 = NULL;
00145 
00146 static char vircam_linearity_analyse_description[] =
00147 "vircam_linearity_analyse -- VIRCAM linearity mapping recipe.\n\n"
00148 "Form master dark images from the input raw frames and use these to\n"
00149 "dark correct a series of dome flat exposures Using the dark\n"
00150 "corrected dome flat series, work out linearity coefficients for\n"
00151 "each data channel. The program expects the following files in the SOF\n"
00152 "    Tag                   Description\n"
00153 "    -----------------------------------------------------------------------\n"
00154 "    %-21s A list of raw dome flat images\n"
00155 "    %-21s A list of raw dark images\n"
00156 "    %-21s The channel table\n"
00157 "    %-21s A list of raw dome flat images at the monitor exposure time\n"
00158 "    %-21s A list of raw dark images at the monitor exposure time\n"
00159 "The first three of these are required. The last two are only required if"
00160 "the light source monitoring algorithm is to be used"
00161 "\n";
00162 
00284 /* Function code */
00285 
00286 /*---------------------------------------------------------------------------*/
00294 /*---------------------------------------------------------------------------*/
00295 
00296 int cpl_plugin_get_info(cpl_pluginlist *list) {
00297     cpl_recipe  *recipe = cpl_calloc(1,sizeof(*recipe));
00298     cpl_plugin  *plugin = &recipe->interface;
00299     char alldesc[SZ_ALLDESC];
00300     (void)snprintf(alldesc,SZ_ALLDESC,vircam_linearity_analyse_description,
00301                    VIRCAM_LIN_DOME_RAW,VIRCAM_LIN_DARK_RAW,
00302                    VIRCAM_CAL_CHANTAB_INIT,VIRCAM_LIN_DOME_CHECK,
00303                    VIRCAM_LIN_DARK_CHECK);
00304 
00305     cpl_plugin_init(plugin,
00306                     CPL_PLUGIN_API,
00307                     VIRCAM_BINARY_VERSION,
00308                     CPL_PLUGIN_TYPE_RECIPE,
00309                     "vircam_linearity_analyse",
00310                     "VIRCAM linearity analysis recipe",
00311                     alldesc,
00312                     "Jim Lewis",
00313                     "jrl@ast.cam.ac.uk",
00314                     vircam_get_license(),
00315                     vircam_linearity_analyse_create,
00316                     vircam_linearity_analyse_exec,
00317                     vircam_linearity_analyse_destroy);
00318 
00319     cpl_pluginlist_append(list,plugin);
00320 
00321     return(0);
00322 }
00323 
00324 /*---------------------------------------------------------------------------*/
00333 /*---------------------------------------------------------------------------*/
00334 
00335 static int vircam_linearity_analyse_create(cpl_plugin *plugin) {
00336     cpl_recipe      *recipe;
00337     cpl_parameter   *p;
00338 
00339     /* Get the recipe out of the plugin */
00340 
00341     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00342         recipe = (cpl_recipe *)plugin;
00343     else
00344         return(-1);
00345 
00346     /* Create the parameters list in the cpl_recipe object */
00347 
00348     recipe->parameters = cpl_parameterlist_new();
00349 
00350     /* Fill in the parameters. First the polynomial order */
00351 
00352     p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.norder",
00353                                 CPL_TYPE_INT,
00354                                 "Order of polynomial fit",
00355                                 "vircam.vircam_linearity_analyse",
00356                                 4,1,6);
00357     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nord");
00358     cpl_parameterlist_append(recipe->parameters,p);
00359 
00360     /* The lower threshold */
00361 
00362     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.lthr",
00363                                 CPL_TYPE_DOUBLE,
00364                                 "Lower bad pixel threshold",
00365                                 "vircam.vircam_linearity_analyse",8.0);
00366     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"lthr");
00367     cpl_parameterlist_append(recipe->parameters,p);
00368 
00369     /* The upper threshold */
00370 
00371     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.hthr",
00372                                 CPL_TYPE_DOUBLE,
00373                                 "Upper bad pixel threshold",
00374                                 "vircam.vircam_linearity_analyse",8.0);
00375     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"hthr");
00376     cpl_parameterlist_append(recipe->parameters,p);
00377 
00378     /* The maximum number of frames to be used in forming the bad pixel mask */
00379 
00380     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.maxbpmfr",
00381                                 CPL_TYPE_INT,
00382                                 "Maximum # frames used in bpm analysis",
00383                                 "vircam.vircam_linearity_analyse",10);
00384     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"maxbpmfr");
00385     cpl_parameterlist_append(recipe->parameters,p);
00386 
00387     /* The flag to allow statistics of the dome flat frames to be adjusted
00388        using a set of monitor exposures */
00389 
00390     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.adjust",
00391                                 CPL_TYPE_BOOL,
00392                                 "Adjust stats with monitor set",
00393                                 "vircam.vircam_linearity_analyse",1);
00394     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"adjust");
00395     cpl_parameterlist_append(recipe->parameters,p);
00396 
00397     /* The flag to allow diagnostic curves to be written out */
00398 
00399     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.diagnostic",
00400                                 CPL_TYPE_BOOL,
00401                                 "Write out diagnostic tables",
00402                                 "vircam.vircam_linearity_analyse",0);
00403     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"diagnostic");
00404     cpl_parameterlist_append(recipe->parameters,p);
00405 
00406     /* Extension number of input frames to use */
00407 
00408     p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.extenum",
00409                                 CPL_TYPE_INT,
00410                                 "Extension number to be done, 0 == all",
00411                                 "vircam.vircam_linearity_analyse",
00412                                 1,0,16);
00413     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
00414     cpl_parameterlist_append(recipe->parameters,p);
00415 
00416     /* Get out of here */
00417 
00418     return(0);
00419 }
00420 
00421 /*---------------------------------------------------------------------------*/
00427 /*---------------------------------------------------------------------------*/
00428 
00429 static int vircam_linearity_analyse_exec(cpl_plugin *plugin) {
00430     cpl_recipe  *recipe;
00431 
00432     /* Get the recipe out of the plugin */
00433 
00434     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00435         recipe = (cpl_recipe *)plugin;
00436     else
00437         return(-1);
00438 
00439     return(vircam_linearity_analyse(recipe->parameters,recipe->frames));
00440 }
00441 
00442 /*---------------------------------------------------------------------------*/
00448 /*---------------------------------------------------------------------------*/
00449 
00450 static int vircam_linearity_analyse_destroy(cpl_plugin *plugin) {
00451     cpl_recipe *recipe ;
00452 
00453     /* Get the recipe out of the plugin */
00454 
00455     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00456         recipe = (cpl_recipe *)plugin;
00457     else
00458         return(-1);
00459 
00460     cpl_parameterlist_delete(recipe->parameters);
00461     return(0);
00462 }
00463 
00464 /*---------------------------------------------------------------------------*/
00471 /*---------------------------------------------------------------------------*/
00472 
00473 static int vircam_linearity_analyse(cpl_parameterlist *parlist,
00474                                     cpl_frameset *framelist) {
00475     const char *fctid="vircam_linearity_analyse";
00476     char colname[16];
00477     int i,jst,jfn,j,status,k,nbad,ngood,krem,n;
00478     int ndarks,ndomes,kk,live,ngood_flats,*bpm,nalloc,adjust,ndit;
00479     cpl_size nlab;
00480     long np;
00481     float med,mindit,*exps,badfrac,expt;
00482     unsigned char *rejmask,*rejplus;
00483     double *dexps,**fdata,*d,*mjds,*mjdcheck,mjd,**cdata,*cf,fac;
00484     double facrng,maxdiff,*lindata;
00485     vir_fits **darks,**domes,*test,*outdark,*fframe;
00486     parquet *pp;
00487     cpl_parameter *p;
00488     cpl_propertylist *drs,*plist;
00489     cpl_image *outimage;
00490     cpl_frame *frame;
00491     
00492     /* Check validity of input frameset */
00493 
00494     if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
00495         cpl_msg_error(fctid,"Input framelist NULL or has no input data");
00496         return(-1);
00497     }
00498 
00499     /* Check the files in the frameset */
00500 
00501     if (vircam_frameset_fexists(framelist) != VIR_OK) {
00502         cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
00503         return(-1);
00504     }
00505 
00506     /* Initialise a few things */
00507 
00508     vircam_linearity_analyse_init();
00509 
00510     /* Get the parameters */
00511 
00512     p = cpl_parameterlist_find(parlist,
00513                                "vircam.vircam_linearity_analyse.norder");
00514     vircam_linearity_analyse_config.norder = cpl_parameter_get_int(p);
00515     p = cpl_parameterlist_find(parlist,
00516                                "vircam.vircam_linearity_analyse.lthr");
00517     vircam_linearity_analyse_config.lthr = (float)cpl_parameter_get_double(p);
00518     p = cpl_parameterlist_find(parlist,
00519                                "vircam.vircam_linearity_analyse.hthr");
00520     vircam_linearity_analyse_config.hthr = (float)cpl_parameter_get_double(p);
00521     p = cpl_parameterlist_find(parlist,
00522                                "vircam.vircam_linearity_analyse.maxbpmfr");
00523     vircam_linearity_analyse_config.maxbpmfr = cpl_parameter_get_int(p);
00524     p = cpl_parameterlist_find(parlist,
00525                                "vircam.vircam_linearity_analyse.adjust");
00526     vircam_linearity_analyse_config.adjust = cpl_parameter_get_bool(p);
00527     p = cpl_parameterlist_find(parlist,
00528                                "vircam.vircam_linearity_analyse.diagnostic");
00529     vircam_linearity_analyse_config.diagnostic = cpl_parameter_get_bool(p);
00530     p = cpl_parameterlist_find(parlist,
00531                                "vircam.vircam_linearity_analyse.extenum");
00532     vircam_linearity_analyse_config.extenum = cpl_parameter_get_int(p);
00533 
00534     /* Sort out raw from calib frames */
00535 
00536     if (vircam_dfs_set_groups(framelist) != VIR_OK) {
00537         cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
00538         vircam_linearity_analyse_tidy(2);
00539         return(-1);
00540     }
00541 
00542     /* Get framelist labels */
00543 
00544     if ((ps.labels = cpl_frameset_labelise(framelist,vircam_compare_tags,
00545                                            &nlab)) == NULL) {
00546         cpl_msg_error(fctid,"Cannot labelise the input frames");
00547         vircam_linearity_analyse_tidy(2);
00548         return(-1);
00549     }
00550 
00551     /* Get the dome flat frames */
00552 
00553     if ((ps.domelist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00554                                                 VIRCAM_LIN_DOME_RAW)) == NULL) {
00555         cpl_msg_error(fctid,"Cannot find dome flat frames in input frameset");
00556         vircam_linearity_analyse_tidy(2);
00557         return(-1);
00558     }
00559     ps.ndomes = cpl_frameset_get_size(ps.domelist);
00560     ps.inherit = cpl_frameset_get_first(ps.domelist);
00561 
00562     /* Check to make sure that NDIT == 1 */
00563 
00564     plist = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_frame(ps.domelist,0)),0);
00565     (void)vircam_pfits_get_ndit(plist,&ndit);
00566     freepropertylist(plist);
00567     if (ndit != 1) {
00568         cpl_msg_error(fctid,
00569                       "NDIT=%" CPL_SIZE_FORMAT ". Recipe requires that ndit == 1",
00570                       (cpl_size)ndit);
00571         vircam_linearity_analyse_tidy(2);
00572         return(-1);
00573     }
00574 
00575     /* Get the dark frames */
00576 
00577     if ((ps.darklist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00578                                                 VIRCAM_LIN_DARK_RAW)) == NULL) {
00579         cpl_msg_error(fctid,"Cannot find dark frames in input frameset");
00580         vircam_linearity_analyse_tidy(2);
00581         return(-1);
00582     }
00583     ps.ndarks = cpl_frameset_get_size(ps.darklist);
00584 
00585     /* If you are planning to adjust the stats for the dome frames by using
00586        a monitor exposure set, then read that frameset in now */
00587 
00588     if (vircam_linearity_analyse_config.adjust) {
00589         if ((ps.domecheck = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00590                                                      VIRCAM_LIN_DOME_CHECK)) == NULL) {
00591             cpl_msg_info(fctid,"No monitor frames found in sof. No adjustments made to stats");
00592             vircam_linearity_analyse_config.adjust = 0;
00593             ps.ndomecheck = 0;
00594         } else {
00595             ps.ndomecheck = cpl_frameset_get_size(ps.domecheck);
00596             if ((ps.darkcheck = vircam_frameset_subgroup(framelist,ps.labels,
00597                                                          nlab,VIRCAM_LIN_DARK_CHECK)) == NULL) {
00598                 cpl_msg_info(fctid,"No darks for monitor frames found in sof. No adjustments made to stats");
00599                 vircam_linearity_analyse_config.adjust = 0;
00600                 ps.ndomecheck = 0;
00601                 freeframeset(ps.domecheck);
00602                 ps.ndarkcheck = 0;
00603             } else {
00604                 ps.ndarkcheck = cpl_frameset_get_size(ps.darkcheck);
00605             }
00606         }       
00607     }
00608     
00609     /* Check to see if there is a channel table. If so, then read it */
00610 
00611     if ((ps.chanfrm = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00612                                                  VIRCAM_CAL_CHANTAB_INIT)) == NULL) {
00613         cpl_msg_error(fctid,"No initial channel table found");
00614         vircam_linearity_analyse_tidy(2);
00615         return(-1);
00616     }
00617 
00618     /* Group the domes and darks by exposure time */
00619 
00620     if (vircam_linearity_analyse_domedark_groups() != 0) {
00621         vircam_linearity_analyse_tidy(2);
00622         return(-1);
00623     }
00624 
00625     /* See if the number of exposure times is too small for the order of the
00626        polynomial you want to fit. If it is, the adjust the order */
00627 
00628     if (ps.nddg < vircam_linearity_analyse_config.norder+1) {
00629         cpl_msg_warning(fctid,
00630                         "Number of exposure times is too small: %" CPL_SIZE_FORMAT ", order: %" CPL_SIZE_FORMAT "\nTaking fit down to order %" CPL_SIZE_FORMAT,
00631                         (cpl_size)(ps.nddg),
00632                         (cpl_size)(vircam_linearity_analyse_config.norder),
00633                         (cpl_size)(ps.nddg-1));
00634         vircam_linearity_analyse_config.norder = ps.nddg - 1;
00635     }
00636 
00637     /* Now, how many image extensions do we want to do? If the extension
00638        number is zero, then we loop for all possible extensions. If it
00639        isn't then we just do the extension specified */
00640 
00641     vircam_exten_range(vircam_linearity_analyse_config.extenum,
00642                        (const cpl_frame *)cpl_frameset_get_frame(ps.ddg[0].darks,0),
00643                        &jst,&jfn);
00644     if (jst == -1 || jfn == -1) {
00645         cpl_msg_error(fctid,"Unable to continue");
00646         vircam_linearity_analyse_tidy(2);
00647         return(-1);
00648     }
00649 
00650     /* Now loop for all the extensions and do the BPM analysis... */
00651 
00652     for (j = jst; j <= jfn; j++) {
00653         cpl_msg_info(fctid,"Beginning BPM work on extension %" CPL_SIZE_FORMAT,
00654                      (cpl_size)j);
00655         isfirst = (j == jst);
00656         dummy = 0;
00657         vircam_linearity_analyse_config.bad_pixel_stat = 0.0;
00658         vircam_linearity_analyse_config.bad_pixel_num = 0;
00659         vircam_linearity_analyse_config.linerror = 0.0;
00660         vircam_linearity_analyse_config.linearity = 0.0;
00661         vircam_linearity_analyse_config.facrng = 0.0;
00662         vircam_linearity_analyse_config.maxdiff = 0.0;
00663 
00664         /* Get some standard info in case we need it for dummy products */
00665 
00666         test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[0].domes),
00667                                 CPL_TYPE_FLOAT,j);
00668         ps.plist = cpl_propertylist_duplicate(vircam_fits_get_phu(test));
00669         ps.elist = cpl_propertylist_duplicate(vircam_fits_get_ehu(test));
00670         ps.nx = (int)cpl_image_get_size_x(vircam_fits_get_image(test));
00671         ps.ny = (int)cpl_image_get_size_y(vircam_fits_get_image(test));
00672         vircam_fits_delete(test);
00673 
00674         /* Load up the channel table for this detector and verify it. Get 
00675            saturation level for this detector from FITS header */
00676 
00677         ps.chantab = vircam_tfits_load(ps.chanfrm,j);
00678         if (ps.chantab == NULL) {
00679             cpl_msg_error(fctid,
00680                           "Channel table extension %" CPL_SIZE_FORMAT " failed to load",
00681                           (cpl_size)j);
00682             dummy = 1;
00683             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00684                 return(-1);
00685             continue;
00686         } else if (vircam_chantab_verify(vircam_tfits_get_table(ps.chantab)) 
00687                    != VIR_OK) {
00688             cpl_msg_error(fctid,
00689                           "Channel table extension %" CPL_SIZE_FORMAT " has errors",
00690                           (cpl_size)j);
00691             dummy = 1;
00692             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00693                 return(-1);
00694             continue;
00695         }
00696         if (vircam_pfits_get_saturation(vircam_tfits_get_ehu(ps.chantab),
00697                                         &sat) != VIR_OK) {
00698             cpl_msg_error(fctid,
00699                           "Channel table extension header %" CPL_SIZE_FORMAT " missing saturation info",
00700                           (cpl_size)j);
00701             dummy = 1;
00702             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00703                 return(-1);
00704             vircam_linearity_analyse_tidy(1);
00705             continue;
00706         }
00707 
00708         /* Get the channel structure */
00709 
00710         vircam_chan_fill(vircam_tfits_get_table(ps.chantab),&pp,&np);
00711 
00712         /* If doing diagnostics, then create the tables now */
00713 
00714         if (vircam_linearity_analyse_config.diagnostic) {
00715             ps.diag1 = vircam_linearity_analyse_diagtab_init(np,ps.ndomes);
00716             if (vircam_linearity_analyse_config.adjust)
00717                 ps.diag2 = vircam_linearity_analyse_diagtab_init(np,ps.ndomecheck);
00718         }
00719 
00720         /* Check DETLIVE for this extension */
00721 
00722         if (vircam_pfits_get_detlive((const cpl_propertylist *)ps.elist,&live) 
00723             != VIR_OK) {
00724             cpl_msg_error(fctid,"No DET LIVE keyword in this extension");
00725             dummy = 1;
00726             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0) 
00727                 return(-1);
00728             vircam_linearity_analyse_tidy(1);
00729             continue;
00730         }
00731         if (! live) {
00732             cpl_msg_info(fctid,"Detector flagged dead");
00733             dummy = 1;
00734             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0) 
00735                 return(-1);
00736             vircam_linearity_analyse_tidy(1);
00737             continue;
00738         }
00739 
00740         /* Get the value of MINDIT so that you can work out the rough level
00741            of the flats before the reset frame was subtracted off */
00742 
00743         if (vircam_pfits_get_mindit((const cpl_propertylist *)ps.elist,
00744                                     &mindit) != VIR_OK) {
00745             cpl_msg_error(fctid,
00746                           "No value of MINDIT found in extension %" CPL_SIZE_FORMAT,
00747                           (cpl_size)j);
00748             dummy = 1;
00749             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00750                 return(-1);
00751             continue;
00752         }
00753 
00754         /* Get a representative from each exposure group and check if it
00755            is saturated. If it is, then reject the group from further 
00756            analysis */
00757 
00758         ngood = 0;
00759         exps = cpl_malloc(ps.nddg*sizeof(float));
00760         ngood_flats = 0;
00761         for (i = 0; i < ps.nddg; i++) {
00762             test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[i].domes),
00763                                     CPL_TYPE_FLOAT,j);
00764             med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
00765             med *= (1.0 + mindit/ps.ddg[i].exptime);
00766             if (med > sat) {
00767                 ps.ddg[i].flag = SATURATE_FLAG;
00768             } else {
00769                 ngood++;
00770                 exps[ngood-1] = ps.ddg[i].exptime;
00771                 ngood_flats += ps.ddg[i].ndomes;
00772                 ps.ddg[i].flag = OK_FLAG;
00773             }
00774             vircam_fits_delete(test);
00775         }
00776         exps = cpl_realloc(exps,ngood*sizeof(float));
00777                
00778         /* Are there enough non-saturated exposures for linearity fit? */
00779 
00780         if (ngood <  vircam_linearity_analyse_config.norder+1) {
00781             cpl_msg_info(fctid,
00782                          "Too few unsaturated flats for linearity fit for extension %" CPL_SIZE_FORMAT,
00783                          (cpl_size)j);
00784             dummy = 1;
00785             cpl_free(exps);
00786             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0) 
00787                 return(-1);
00788             continue;
00789         }           
00790 
00791         /* Sort the exposure array */
00792 
00793         vircam_sort(&exps,ngood,1);
00794 
00795         /* Loop for each exposure time. When you have enough flats, then
00796            you can quit this loop */
00797 
00798         ps.nuse = min(vircam_linearity_analyse_config.maxbpmfr,ngood_flats);
00799         ps.nflatlist = 0;
00800         ps.flatlist = cpl_malloc(ps.nuse*sizeof(vir_fits *));
00801         for (i = ngood-1; i >= 0; i--) {
00802             krem = -1;
00803             for (k = 0; k <= ps.nddg; k++) {
00804                 if (ps.ddg[k].exptime == exps[i]) {
00805                     krem = k;
00806                     break;
00807                 }
00808             }
00809 
00810             /* Load the dark frames from this exposure time */
00811 
00812             darks = vircam_fits_load_list(ps.ddg[krem].darks,CPL_TYPE_FLOAT,j);
00813             ndarks = ps.ddg[krem].ndarks;
00814             if (darks == NULL) {
00815                 cpl_msg_error(fctid,
00816                               "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
00817                               (cpl_size)j,ps.ddg[krem].exptime);
00818                 continue;
00819             }
00820 
00821             /* Form a mean dark for this exposure time. If there is only one
00822                then don't bother, just load up that one frame. */
00823 
00824             if (ndarks == 1) {
00825                 outdark = vircam_fits_duplicate(darks[0]);
00826             } else {
00827                 status = VIR_OK;
00828                 (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,&outimage,
00829                                        &rejmask,&rejplus,&drs,&status);
00830                 freespace(rejmask);
00831                 freespace(rejplus);
00832                 freepropertylist(drs);
00833                 if (status != VIR_OK) {
00834                     cpl_msg_error(fctid,
00835                                   "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
00836                                   (cpl_size)j,ps.ddg[krem].exptime);
00837                     freefitslist(darks,ndarks);
00838                     continue;
00839                 }
00840                 outdark = vircam_fits_wrap(outimage,darks[0],NULL,NULL);
00841             }
00842             freefitslist(darks,ndarks);
00843 
00844             /* Load the flats for this group */
00845 
00846             domes = vircam_fits_load_list(ps.ddg[krem].domes,CPL_TYPE_FLOAT,j);
00847             ndomes = ps.ddg[krem].ndomes;
00848             if (domes == NULL) {
00849                 cpl_msg_error(fctid,
00850                               "Error loading domes extension %" CPL_SIZE_FORMAT ", exptime %g",
00851                               (cpl_size)j,ps.ddg[i].exptime);
00852                 freefits(outdark);
00853                 continue;
00854             }
00855             
00856             /* Now loop for each flat in this group or until you have
00857                filled the flats buffer */
00858 
00859             for (kk = 0; kk < ndomes; kk++) {
00860                 status = VIR_OK;
00861                 vircam_darkcor(domes[kk],outdark,1.0,&status);
00862                 ps.flatlist[ps.nflatlist] = vircam_fits_duplicate(domes[kk]);
00863                 ps.ddg[krem].proc[kk] = ps.flatlist[ps.nflatlist];
00864                 ps.nflatlist++;
00865                 if (ps.nflatlist == ps.nuse)
00866                     break;
00867             }
00868 
00869             /* Tidy up a bit */
00870 
00871             freefitslist(domes,ndomes);
00872             freefits(outdark);
00873 
00874             /* Do we have enough yet? */
00875 
00876             if (ps.nflatlist == ps.nuse) 
00877                 break;
00878         }
00879         freespace(exps);
00880         ps.flatlist = cpl_realloc(ps.flatlist,
00881                                   (ps.nflatlist)*sizeof(vir_fits *));
00882         
00883         /* Generate a bad pixel mask now */
00884 
00885         status = VIR_OK;
00886         (void)vircam_genbpm(ps.flatlist,ps.nflatlist,
00887                             vircam_linearity_analyse_config.lthr,
00888                             vircam_linearity_analyse_config.hthr,
00889                             &(ps.bpm_array),&nbad,&badfrac,&status);
00890         bpm = cpl_array_get_data_int(ps.bpm_array);
00891 
00892         /* Store away some useful info */
00893 
00894         vircam_linearity_analyse_config.bad_pixel_num = nbad;
00895         vircam_linearity_analyse_config.bad_pixel_stat = badfrac;
00896 
00897         /* Right. Free the pointer for flatlist, but don't delete the 
00898            vir_fits structures in the list because they are copied into
00899            the ddg structure */
00900 
00901         freespace(ps.flatlist);
00902         ps.nflatlist = 0;
00903 
00904         /* Get an initial allocation of space to hold the stats */
00905 
00906         nalloc = 16;
00907         fdata = cpl_malloc(nalloc*sizeof(double *));
00908         dexps = cpl_malloc(nalloc*sizeof(double));
00909         mjds = cpl_malloc(nalloc*sizeof(double));
00910 
00911         /* Loop through the ddg structure, missing out any that have
00912            overexposed flats. Work out the stats for the images that 
00913            remain. */
00914 
00915         cpl_msg_info(fctid,
00916                      "Beginning linearity work on extension %" CPL_SIZE_FORMAT,
00917                      (cpl_size)j);
00918         ps.nfdata = 0;
00919         outdark = NULL;
00920         for (i = 0; i < ps.nddg; i++) {
00921             if (ps.ddg[i].flag == SATURATE_FLAG)
00922                 continue;
00923             for (k = 0; k < ps.ddg[i].ndomes; k++) {
00924 
00925                 /* If this particular frame wasn't processed, then you need
00926                    to form a dark for this group. */
00927 
00928                 if (ps.ddg[i].proc[k] == NULL) {
00929                     if (outdark == NULL) {
00930 
00931                         /* Load the dark frames from this exposure time */
00932 
00933                         darks = vircam_fits_load_list(ps.ddg[i].darks,
00934                                                       CPL_TYPE_FLOAT,j);
00935                         ndarks = ps.ddg[i].ndarks;
00936                         if (darks == NULL) {
00937                             cpl_msg_error(fctid,
00938                                           "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
00939                                           (cpl_size)j,ps.ddg[i].exptime);
00940                             continue;
00941                         }
00942 
00943                         /* Form a mean dark for this exposure time. If there 
00944                            is only one then don't bother, just load up that 
00945                            one frame. */
00946 
00947                         if (ps.ddg[i].ndarks == 1) {
00948                             outdark = vircam_fits_duplicate(darks[0]);
00949                         } else {
00950                             status = VIR_OK;
00951                             (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
00952                                                    &outimage,&rejmask,&rejplus,
00953                                                    &drs,&status);
00954                             freespace(rejmask);
00955                             freespace(rejplus);
00956                             freepropertylist(drs);
00957                             if (status != VIR_OK) {
00958                                 cpl_msg_error(fctid,
00959                                               "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
00960                                               (cpl_size)j,ps.ddg[i].exptime);
00961                                 freefitslist(darks,ndarks);
00962                                 continue;
00963                             }
00964                             outdark = vircam_fits_wrap(outimage,darks[0],
00965                                                        NULL,NULL);
00966                         }
00967                         freefitslist(darks,ndarks);
00968                     }
00969 
00970                     /* Load the flat and dark correct it */
00971 
00972                     frame = cpl_frameset_get_frame(ps.ddg[i].domes,k);
00973                     fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
00974                     vircam_darkcor(fframe,outdark,1.0,&status);
00975 
00976                 /* If this frame has already been corrected, then use it */
00977 
00978                 } else {
00979                     fframe = ps.ddg[i].proc[k];
00980                 }
00981 
00982                 /* Generate the stats for this frame and store it */
00983 
00984                 d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
00985                 if (ps.nfdata >= nalloc) {
00986                     nalloc += 16;
00987                     fdata = cpl_realloc(fdata,nalloc*sizeof(double *));
00988                     dexps = cpl_realloc(dexps,nalloc*sizeof(double));
00989                     mjds = cpl_realloc(mjds,nalloc*sizeof(double));
00990                 }
00991                 (void)vircam_pfits_get_mjd(vircam_fits_get_phu(fframe),&mjd);
00992                 mjds[ps.nfdata] = mjd;
00993                 dexps[ps.nfdata] = (double)(ps.ddg[i].exptime);         
00994                 fdata[ps.nfdata] = d;           
00995 
00996                 /* If doing diagnostic curves, then add the relevant
00997                    information to the first table now */
00998 
00999                 if (ps.diag1 != NULL) {
01000                     cpl_table_set_string(ps.diag1,"filename",
01001                                          (cpl_size)(ps.nfdata),
01002                                          vircam_fits_get_filename(fframe));
01003                     cpl_table_set_double(ps.diag1,"exptime",
01004                                          (cpl_size)ps.nfdata,
01005                                          dexps[ps.nfdata]);
01006                     cpl_table_set_double(ps.diag1,"mjd",(cpl_size)(ps.nfdata),
01007                                          mjd);
01008                     for (n = 1; n <= np; n++) {
01009                         snprintf(colname,16,"rawflux_%02d",n);
01010                         cpl_table_set_double(ps.diag1,colname,
01011                                              (cpl_size)(ps.nfdata),d[n-1]);
01012                     }
01013                 }
01014                 if (ps.ddg[i].proc[k] != NULL) { 
01015                     freefits(ps.ddg[i].proc[k]);
01016                 } else {
01017                     freefits(fframe);
01018                 }
01019                 ps.nfdata++;
01020             }
01021             freefits(outdark);
01022         }
01023         freefits(outdark);
01024         if (ps.diag1 != NULL) 
01025             cpl_table_set_size(ps.diag1,(cpl_size)(ps.nfdata));
01026 
01027         /* Now, if we are going to tweak the stats using the monitor exposure
01028            then we should do that now */
01029 
01030         if (vircam_linearity_analyse_config.adjust) {
01031 
01032             /* Get some workspace for the data array */
01033 
01034             cdata = cpl_malloc(ps.ndomecheck*sizeof(double *));
01035 
01036             /* Get the exposure time for the monitor set and make sure
01037                that the set isn't saturated */
01038 
01039             test = vircam_fits_load(cpl_frameset_get_first(ps.domecheck),
01040                                     CPL_TYPE_FLOAT,j);
01041             (void)vircam_pfits_get_exptime(vircam_fits_get_phu(test),&expt);
01042             med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
01043             med *= (1.0 + mindit/expt);
01044             adjust = 1;
01045             if (med > sat) {
01046                 cpl_msg_info(fctid,"Monitor exposures saturated. No drift adjustment made");
01047                 adjust = 0;
01048             }
01049             vircam_fits_delete(test);
01050 
01051             /* Ok assuming all that's going well, then form a mean dark */
01052 
01053             if (adjust) {
01054 
01055                 darks = vircam_fits_load_list(ps.darkcheck,CPL_TYPE_FLOAT,j);
01056                 ndarks = ps.ndarkcheck;
01057                 if (darks == NULL) {
01058                     cpl_msg_error(fctid,
01059                                   "Error loading check darks extension %" CPL_SIZE_FORMAT,
01060                                   (cpl_size)j);
01061                     continue;
01062                 }
01063 
01064                 /* Form a mean dark for this exposure time. If there 
01065                    is only one then don't bother, just load up that 
01066                    one frame. */
01067 
01068                 if (ndarks == 1) {
01069                     outdark = vircam_fits_duplicate(darks[0]);
01070                 } else {
01071                     status = VIR_OK;
01072                     (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
01073                                            &outimage,&rejmask,&rejplus,
01074                                            &drs,&status);
01075                     freespace(rejmask);
01076                     freespace(rejplus);
01077                     freepropertylist(drs);
01078                     if (status != VIR_OK) {
01079                         cpl_msg_error(fctid,
01080                                       "Combine failure extension %" CPL_SIZE_FORMAT " monitor",
01081                                       (cpl_size)j);
01082                         freefitslist(darks,ndarks);
01083                         continue;
01084                     }
01085                     outdark = vircam_fits_wrap(outimage,darks[0],
01086                                                NULL,NULL);
01087                 }
01088                 freefitslist(darks,ndarks);             
01089 
01090                 /* Now, loop through the monitor domes, dark correct and then
01091                    do the stats */
01092                 
01093                 mjdcheck = cpl_malloc(ps.ndomecheck*sizeof(double));
01094                 for (i = 0; i < ps.ndomecheck; i++) {
01095                     frame = cpl_frameset_get_frame(ps.domecheck,i);
01096                     fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
01097                     vircam_darkcor(fframe,outdark,1.0,&status);
01098                     d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
01099                     cdata[i] = d;
01100                     vircam_pfits_get_mjd(vircam_fits_get_phu(fframe),&mjd);
01101                     vircam_pfits_get_exptime(vircam_fits_get_phu(fframe),&expt);
01102                     mjdcheck[i] = mjd;
01103 
01104                     /* If doing diagnostics then fill in the table now */
01105 
01106                     if (ps.diag2 != NULL) {
01107                         cpl_table_set_string(ps.diag2,"filename",(cpl_size)i,
01108                                              vircam_fits_get_filename(fframe));
01109                         cpl_table_set_double(ps.diag2,"exptime",
01110                                              (cpl_size)i,(double)expt);
01111                         cpl_table_set_double(ps.diag2,"mjd",
01112                                              (cpl_size)i,mjd);
01113                         for (n = 1; n <= np; n++) {
01114                             snprintf(colname,16,"rawflux_%02d",n);
01115                             cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
01116                                                  d[n-1]);
01117                             snprintf(colname,16,"linflux_%02d",n);
01118                             cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
01119                                                  d[n-1]);
01120                         }
01121                     }
01122                     freefits(fframe);
01123                 }
01124                 freefits(outdark);
01125 
01126                 /* Generate the correction factors now */
01127 
01128                 cf = vircam_linearity_tweakfac(cdata,mjdcheck,ps.ndomecheck,
01129                                                np,&facrng,&maxdiff);
01130                 vircam_linearity_analyse_config.facrng = 100.0*(float)facrng;
01131                 vircam_linearity_analyse_config.maxdiff = 100.0*(float)maxdiff;
01132                 if (ps.diag2 != NULL) {
01133                     for (i = 0; i < ps.ndomecheck; i++) 
01134                         cpl_table_set_double(ps.diag2,"adjust_fac",
01135                                              (cpl_size)i,cf[i]);
01136                 }
01137 
01138                 /* Ok, now do the correction for each of the linearity
01139                    sequence frames */
01140 
01141                 for (i = 0; i < ps.nfdata; i++) {
01142                     mjd = mjds[i];
01143                     krem = -1;
01144                     for (k = 0; k < ps.ndomecheck; k++) {
01145                         if (mjd < mjdcheck[k]) {
01146                             krem = k;
01147                             break;
01148                         }
01149                     }
01150                     if (krem == -1) {
01151                         fac = cf[ps.ndomecheck-1];
01152                     } else if (krem == 0) {
01153                         fac = cf[0];
01154                     } else {
01155                         fac = 0.5*(cf[krem -1]  + cf[krem]);
01156                     }
01157                     for (k = 0; k < np; k++)
01158                         fdata[i][k] /= fac;
01159                     if (ps.diag1 != NULL) 
01160                         cpl_table_set_double(ps.diag1,"adjust_fac",
01161                                              (cpl_size)i,fac);
01162                 }
01163 
01164                 /* Get rid of some stuff now */
01165 
01166                 freespace2(cdata,ps.ndomecheck);
01167                 freespace(cf);
01168                 freespace(mjdcheck);
01169             }
01170         }
01171 
01172         /* Do some intermediate tidying */
01173         
01174         freespace(mjds);
01175         vircam_chan_free(np,&pp);
01176 
01177         /* Right, there should be no images left in memory now. Do the
01178            linearity analysis now. */
01179 
01180         (void)vircam_genlincur(fdata,ps.nfdata,dexps,(double)mindit,ps.chantab,
01181                                vircam_linearity_analyse_config.norder,
01182                                &(ps.lchantab),&lindata,&status);
01183         if (ps.diag1 != NULL) {
01184             for (i = 0; i < ps.nfdata; i++) {
01185                 for (n = 0; n < np; n++) {
01186                     snprintf(colname,16,"linflux_%02d",n+1);
01187                     cpl_table_set_double(ps.diag1,colname,
01188                                          (cpl_size)i,lindata[i*np+n]);
01189                 }
01190             }
01191         }
01192         freespace2(fdata,ps.nfdata);
01193         freespace(dexps);
01194         freespace(lindata);
01195         if (status != VIR_OK) {
01196             cpl_msg_error(fctid,
01197                           "Linearity curve fit failed extension %" CPL_SIZE_FORMAT,
01198                           (cpl_size)j);
01199             dummy = 1;
01200             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
01201                 return(-1);
01202             vircam_linearity_analyse_tidy(1);
01203             continue;
01204         }
01205         
01206         /* Get some QC1 parameters by finding the average fit error and
01207            the average percentage non-linearity */
01208 
01209         vircam_linearity_analyse_config.linearity = 
01210             (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000");
01211         vircam_linearity_analyse_config.linerror = 
01212             (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000_err");
01213 
01214         /* Save new linearity info */
01215 
01216         if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
01217             return(-1);
01218     }
01219 
01220     /* Get out of here */
01221 
01222     vircam_linearity_analyse_tidy(2);
01223     return(0);
01224 }
01225 
01226 
01227 /*---------------------------------------------------------------------------*/
01234 /*---------------------------------------------------------------------------*/
01235 
01236 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
01237                                          cpl_parameterlist *parlist) {
01238     cpl_propertylist *plist,*elist,*pafprop;
01239     cpl_image *outimg;
01240     const char *outtab = "lchantab.fits";
01241     const char *outbpm = "bpm.fits";
01242     const char *outtabpaf = "lchantab";
01243     const char *outbpmpaf = "bpm";
01244     const char *outdiag1 = "ldiag1.fits";
01245     const char *outdiag2 = "ldiag2.fits";
01246     const char *fctid = "vircam_linearity_analyse_save";
01247     const char *recipeid = "vircam_linearity_analyse";
01248     int nx,ny,nord,*bpm,i;
01249 
01250     /* Do some stuff for the first extension to set up the frame */
01251 
01252     nord = vircam_linearity_analyse_config.norder;
01253     if (isfirst) {
01254 
01255         /* Set up the output frame */
01256 
01257         product_frame_chantab = cpl_frame_new();
01258         cpl_frame_set_filename(product_frame_chantab,outtab);
01259         cpl_frame_set_tag(product_frame_chantab,VIRCAM_PRO_CHANTAB);
01260         cpl_frame_set_type(product_frame_chantab,CPL_FRAME_TYPE_TABLE);
01261         cpl_frame_set_group(product_frame_chantab,CPL_FRAME_GROUP_PRODUCT);
01262         cpl_frame_set_level(product_frame_chantab,CPL_FRAME_LEVEL_FINAL);
01263 
01264         /* Set up the PRO keywords for primary */
01265         
01266         ps.phupaf = vircam_paf_phu_items(ps.plist);
01267         plist = cpl_propertylist_duplicate(ps.plist);
01268         vircam_dfs_set_product_primary_header(plist,product_frame_chantab,
01269                                               framelist,parlist,
01270                                               (char *)recipeid,
01271                                               "PRO-1.15",ps.inherit,0);
01272 
01273         /* Now define the table propertylist and give it an extension name and 
01274            PRO keywords */
01275 
01276         elist = cpl_propertylist_duplicate(ps.elist);
01277         cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
01278         vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
01279                                             framelist,parlist,(char *)recipeid,
01280                                             "PRO-1.15",ps.inherit);
01281         
01282         /* Add QC1 info */
01283 
01284         cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
01285                                       vircam_linearity_analyse_config.linearity);
01286         cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
01287                                      "% non-linearity at 10000 ADU");
01288         cpl_propertylist_update_float(elist,"ESO QC LINERROR",
01289                                       vircam_linearity_analyse_config.linerror);
01290         cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
01291                                      "% error non-linearity at 10000 ADU");
01292         cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
01293                                       vircam_linearity_analyse_config.facrng);
01294         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
01295                                      "total % range in screen variation");
01296         cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
01297                                       vircam_linearity_analyse_config.maxdiff);
01298         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
01299                                      "maximum % step in screen variation");
01300         cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
01301 
01302         /* Set up a dummy table if necessary */
01303 
01304         if (dummy == 1) {
01305             vircam_dummy_property(elist);
01306             if (ps.lchantab == NULL) 
01307                 ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
01308         }
01309 
01310         /* And finally save the table */
01311  
01312         if (cpl_table_save(ps.lchantab,plist,elist,outtab,CPL_IO_DEFAULT)
01313             != CPL_ERROR_NONE) {
01314             cpl_msg_error(fctid,"Cannot save product table extension");
01315             freepropertylist(plist);
01316             freepropertylist(elist);
01317             return(-1);
01318         }
01319         cpl_frameset_insert(framelist,product_frame_chantab);
01320 
01321         /* Write PAF */
01322 
01323         pafprop = vircam_paf_req_items(elist);
01324         vircam_merge_propertylists(pafprop,ps.phupaf);
01325         vircam_paf_append(pafprop,plist,"ESO INS FILT1 NAME");
01326         vircam_paf_append(pafprop,elist,"ESO PRO CATG");
01327         vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
01328         if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
01329                              "QC file",pafprop) != VIR_OK)
01330             cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
01331         cpl_propertylist_delete(pafprop);
01332 
01333         /* Quick tidy */
01334 
01335         freepropertylist(plist);
01336         freepropertylist(elist);
01337 
01338         /* Set up the output bad pixel mask primary */
01339 
01340         product_frame_bpm = cpl_frame_new();
01341         cpl_frame_set_filename(product_frame_bpm,outbpm);
01342         cpl_frame_set_tag(product_frame_bpm,VIRCAM_PRO_BPM);
01343         cpl_frame_set_type(product_frame_bpm,CPL_FRAME_TYPE_IMAGE);
01344         cpl_frame_set_group(product_frame_bpm,CPL_FRAME_GROUP_PRODUCT);
01345         cpl_frame_set_level(product_frame_bpm,CPL_FRAME_LEVEL_FINAL);
01346 
01347         /* Set up the PRO keywords for primary header in the bpm */
01348 
01349         plist = ps.plist;
01350         vircam_dfs_set_product_primary_header(plist,product_frame_bpm,
01351                                               framelist,parlist,
01352                                               (char *)recipeid,"PRO-1.15",
01353                                               ps.inherit,0);
01354 
01355         /* Now save the PHU 'image' */
01356 
01357         if (cpl_image_save(NULL,outbpm,CPL_TYPE_UCHAR,plist,
01358                            CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
01359             cpl_msg_error(fctid,"Cannot save product PHU");
01360             cpl_frame_delete(product_frame_bpm);
01361             return(-1);
01362         }
01363         cpl_frameset_insert(framelist,product_frame_bpm);
01364 
01365         /* Section for saving diagnostic tables. First the linearity
01366            sequence diagnostics */
01367 
01368         if (ps.diag1 != NULL) {
01369 
01370             /* Set up the output frame */
01371 
01372             product_frame_diag1 = cpl_frame_new();
01373             cpl_frame_set_filename(product_frame_diag1,outdiag1);
01374             cpl_frame_set_tag(product_frame_diag1,VIRCAM_PRO_LIN_DIAG1);
01375             cpl_frame_set_type(product_frame_diag1,CPL_FRAME_TYPE_TABLE);
01376             cpl_frame_set_group(product_frame_diag1,CPL_FRAME_GROUP_PRODUCT);
01377             cpl_frame_set_level(product_frame_diag1,CPL_FRAME_LEVEL_FINAL);
01378 
01379             /* Set up the PRO keywords for primary */
01380 
01381             plist = cpl_propertylist_duplicate(ps.plist);
01382             vircam_dfs_set_product_primary_header(plist,product_frame_diag1,
01383                                                   framelist,parlist,
01384                                                   (char *)recipeid,
01385                                                   "PRO-1.15",ps.inherit,0);
01386 
01387             /* Now define the table propertylist and give it an extension name and 
01388                PRO keywords */
01389 
01390             elist = cpl_propertylist_duplicate(ps.elist);
01391             vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
01392                                                 framelist,parlist,
01393                                                 (char *)recipeid,"PRO-1.15",
01394                                                 ps.inherit);
01395 
01396             /* Set up a dummy property if necessary */
01397 
01398             if (dummy == 1) 
01399                 vircam_dummy_property(elist);
01400 
01401             /* And finally save the table */
01402 
01403             if (cpl_table_save(ps.diag1,plist,elist,outdiag1,CPL_IO_DEFAULT)
01404                 != CPL_ERROR_NONE) {
01405                 cpl_msg_error(fctid,"Cannot save product table extension");
01406                 freepropertylist(plist);
01407                 freepropertylist(elist);
01408                 return(-1);
01409             }
01410             cpl_frameset_insert(framelist,product_frame_diag1);
01411             freepropertylist(plist);
01412             freepropertylist(elist);
01413         }
01414 
01415         /* Now the monitor sequence diagnostics */
01416 
01417         if (ps.diag2 != NULL) {
01418 
01419             /* Set up the output frame */
01420 
01421             product_frame_diag2 = cpl_frame_new();
01422             cpl_frame_set_filename(product_frame_diag2,outdiag2);
01423             cpl_frame_set_tag(product_frame_diag2,VIRCAM_PRO_LIN_DIAG2);
01424             cpl_frame_set_type(product_frame_diag2,CPL_FRAME_TYPE_TABLE);
01425             cpl_frame_set_group(product_frame_diag2,CPL_FRAME_GROUP_PRODUCT);
01426             cpl_frame_set_level(product_frame_diag2,CPL_FRAME_LEVEL_FINAL);
01427 
01428             /* Set up the PRO keywords for primary */
01429 
01430             plist = cpl_propertylist_duplicate(ps.plist);
01431             vircam_dfs_set_product_primary_header(plist,product_frame_diag2,
01432                                                   framelist,parlist,
01433                                                   (char *)recipeid,
01434                                                   "PRO-1.15",ps.inherit,0);
01435 
01436             /* Now define the table propertylist and give it an extension name 
01437                and PRO keywords */
01438 
01439             elist = cpl_propertylist_duplicate(ps.elist);
01440             vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
01441                                                 framelist,parlist,
01442                                                 (char *)recipeid,"PRO-1.15",
01443                                                 ps.inherit);
01444 
01445             /* Set up a dummy property if necessary */
01446 
01447             if (dummy == 1) 
01448                 vircam_dummy_property(elist);
01449 
01450             /* And finally save the table */
01451 
01452             if (cpl_table_save(ps.diag2,plist,elist,outdiag2,CPL_IO_DEFAULT)
01453                 != CPL_ERROR_NONE) {
01454                 cpl_msg_error(fctid,"Cannot save product table extension");
01455                 freepropertylist(plist);
01456                 freepropertylist(elist);
01457                 return(-1);
01458             }
01459             cpl_frameset_insert(framelist,product_frame_diag2);
01460             freepropertylist(plist);
01461             freepropertylist(elist);
01462         }
01463 
01464     /* Section for all other extensions */
01465 
01466     } else {
01467 
01468         /* Do the table extension PRO keywords */
01469 
01470         elist = cpl_propertylist_duplicate(ps.elist);
01471         cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
01472         vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
01473                                             framelist,parlist,
01474                                             (char *)recipeid,
01475                                             "PRO-1.15",ps.inherit);
01476 
01477         /* Add QC1 info */
01478 
01479         cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
01480                                       vircam_linearity_analyse_config.linearity);
01481         cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
01482                                      "% non-linearity at 10000 ADU");
01483         cpl_propertylist_update_float(elist,"ESO QC LINERROR",
01484                                       vircam_linearity_analyse_config.linerror);
01485         cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
01486                                      "% error non-linearity at 10000 ADU");
01487         cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
01488                                       vircam_linearity_analyse_config.facrng);
01489         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
01490                                      "total % range in screen variation");
01491         cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
01492                                       vircam_linearity_analyse_config.maxdiff);
01493         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
01494                                      "maximum % step in screen variation");
01495         cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
01496 
01497         /* Set up a dummy table if necessary */
01498 
01499         if (dummy == 1) {
01500             vircam_dummy_property(elist);
01501             if (ps.lchantab == NULL) 
01502                 ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
01503         }
01504 
01505         /* And finally save the table */
01506  
01507         if (cpl_table_save(ps.lchantab,NULL,elist,outtab,CPL_IO_EXTEND)
01508             != CPL_ERROR_NONE) {
01509             cpl_msg_error(fctid,"Cannot save product table extension");
01510             freepropertylist(elist);
01511             return(-1);
01512         }
01513 
01514         /* Write PAF */
01515 
01516         pafprop = vircam_paf_req_items(elist);
01517         vircam_merge_propertylists(pafprop,ps.phupaf);
01518         vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
01519         vircam_paf_append(pafprop,elist,"ESO PRO CATG");
01520         vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
01521         if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
01522                              "QC file",pafprop) != VIR_OK)
01523             cpl_msg_warning(fctid,"Unable to save PAF for BPM");
01524         cpl_propertylist_delete(pafprop);
01525 
01526         /* Quick tidy */
01527 
01528         freepropertylist(elist);
01529 
01530         /* Now the diagnostic tables */
01531 
01532         if (ps.diag1 != NULL) {
01533             elist = cpl_propertylist_duplicate(ps.elist);
01534             vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
01535                                                 framelist,parlist,
01536                                                 (char *)recipeid,"PRO-1.15",
01537                                                 ps.inherit);
01538 
01539             /* Set up a dummy property if necessary */
01540 
01541             if (dummy == 1) 
01542                 vircam_dummy_property(elist);
01543 
01544             /* And finally save the table */
01545 
01546             if (cpl_table_save(ps.diag1,NULL,elist,outdiag1,CPL_IO_EXTEND)
01547                 != CPL_ERROR_NONE) {
01548                 cpl_msg_error(fctid,"Cannot save product table extension");
01549                 freepropertylist(elist);
01550                 return(-1);
01551             }
01552             freepropertylist(elist);
01553         }
01554         if (ps.diag2 != NULL) {
01555             elist = cpl_propertylist_duplicate(ps.elist);
01556             vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
01557                                                 framelist,parlist,
01558                                                 (char *)recipeid,"PRO-1.15",
01559                                                 ps.inherit);
01560 
01561             /* Set up a dummy property if necessary */
01562 
01563             if (dummy == 1) 
01564                 vircam_dummy_property(elist);
01565 
01566             /* And finally save the table */
01567 
01568             if (cpl_table_save(ps.diag2,NULL,elist,outdiag2,CPL_IO_EXTEND)
01569                 != CPL_ERROR_NONE) {
01570                 cpl_msg_error(fctid,"Cannot save product table extension");
01571                 freepropertylist(elist);
01572                 return(-1);
01573             }
01574             freepropertylist(elist);
01575         }
01576 
01577     }
01578 
01579     /* Save the bpm extension now */
01580 
01581     plist = ps.elist;
01582     nx = ps.nx;
01583     ny = ps.ny;
01584     if (dummy && ps.bpm_array == NULL) {
01585         ps.bpm_array = cpl_array_new((cpl_size)(nx*ny),CPL_TYPE_INT);
01586         bpm = cpl_array_get_data_int(ps.bpm_array);
01587         for (i = 0; i < nx*ny; i++)
01588             bpm[i] = 0;
01589     }
01590     bpm = cpl_array_get_data_int(ps.bpm_array);
01591     vircam_dfs_set_product_exten_header(plist,product_frame_bpm,
01592                                         framelist,parlist,(char *)recipeid,
01593                                         "PRO-1.15",ps.inherit);
01594     cpl_propertylist_update_float(plist,"ESO QC BAD_PIXEL_STAT",
01595                                   vircam_linearity_analyse_config.bad_pixel_stat);
01596     cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_STAT",
01597                                  "Fraction of pixels that are bad");
01598     cpl_propertylist_update_int(plist,"ESO QC BAD_PIXEL_NUM",
01599                                 vircam_linearity_analyse_config.bad_pixel_num);
01600     cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_NUM",
01601                                  "Number of pixels that are bad");
01602     cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps.nuse);
01603     if (dummy)
01604         vircam_dummy_property(plist);
01605     outimg = cpl_image_wrap_int((cpl_size)nx,(cpl_size)ny,bpm);
01606     if (cpl_image_save(outimg,outbpm,CPL_TYPE_UCHAR,plist,
01607                        CPL_IO_EXTEND) != CPL_ERROR_NONE) {
01608         cpl_msg_error(fctid,"Cannot save product image extension");
01609         return(-1);
01610     }
01611 
01612     /* Write PAF */
01613 
01614     pafprop = vircam_paf_req_items(plist);
01615     vircam_merge_propertylists(pafprop,ps.phupaf);
01616     vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
01617     vircam_paf_append(pafprop,ps.plist,"ESO PRO CATG");
01618     vircam_paf_append(pafprop,plist,"ESO PRO DATANCOM");
01619     if (vircam_paf_print((char *)outbpmpaf,"VIRCAM/vircam_linearity_analyse",
01620                          "QC file",pafprop) != VIR_OK)
01621         cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
01622     cpl_propertylist_delete(pafprop);
01623 
01624     /* Quick tidy */
01625 
01626     cpl_image_unwrap(outimg);
01627 
01628     return(0);
01629 }
01630 
01631 /*---------------------------------------------------------------------------*/
01639 /*---------------------------------------------------------------------------*/
01640 
01641 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
01642                                             cpl_parameterlist *parlist) {
01643     int retval;
01644     const char *fctid = "vircam_linearity_analyse_lastbit";
01645 
01646     /* Save the new channel table and bad pixel map */
01647 
01648     cpl_msg_info(fctid,
01649                  "Saving linearity table and bpm for extension %" CPL_SIZE_FORMAT,
01650                  (cpl_size)jext);
01651     retval = vircam_linearity_analyse_save(framelist,parlist);
01652     if (retval != 0) {
01653         vircam_linearity_analyse_tidy(2);
01654         return(-1);
01655     }
01656 
01657     /* Do some intermediate tidying */
01658 
01659     vircam_linearity_analyse_tidy(1);
01660     return(0);
01661 }
01662 
01663 /*---------------------------------------------------------------------------*/
01667 /*---------------------------------------------------------------------------*/
01668 
01669 static int vircam_linearity_analyse_domedark_groups(void) {
01670     int i,j,found;
01671     float texp;
01672     cpl_frame *frame;
01673     cpl_propertylist *plist;
01674     const char *fctid = "vircam_linearity_analyse_domedark_groups";
01675 
01676     /* Start by getting the memory for the domedark groups */
01677 
01678     ps.ddg = cpl_calloc(ps.ndomes,sizeof(ddgrp));
01679     ps.nddg = 0;
01680 
01681     /* Loop for each of the dome frames and get its exposure time. If this
01682        doesn't exist, then signal an error and go on */
01683 
01684     for (i = 0; i < ps.ndomes; i++) {
01685         frame = cpl_frameset_get_frame(ps.domelist,i);
01686         plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
01687         if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
01688             cpl_msg_warning(fctid,"No exposure time found in %s",
01689                             cpl_frame_get_filename(frame));
01690             cpl_propertylist_delete(plist);
01691             continue;
01692         }
01693         cpl_propertylist_delete(plist);
01694 
01695         /* Search the domedark groups to see if this exposure time has already 
01696            been used. If not, then create a new group. If it has then just add 
01697            this frame to the correct group */
01698 
01699         found = 0;
01700         for (j = 0; j < ps.nddg; j++) {
01701             if (ps.ddg[j].exptime == texp) {
01702                 found = 1;
01703                 break;
01704             }
01705         }
01706         if (found) {
01707             cpl_frameset_insert(ps.ddg[j].domes,cpl_frame_duplicate(frame));
01708             ps.ddg[j].ndomes += 1;
01709         } else {
01710             ps.ddg[ps.nddg].exptime = texp;
01711             ps.ddg[ps.nddg].darks = cpl_frameset_new();
01712             ps.ddg[ps.nddg].domes = cpl_frameset_new();
01713             ps.ddg[ps.nddg].ndarks = 0;
01714             ps.ddg[ps.nddg].ndomes = 1;
01715             ps.ddg[ps.nddg].flag = OK_FLAG;
01716             cpl_frameset_insert(ps.ddg[ps.nddg].domes,
01717                                cpl_frame_duplicate(frame));
01718             ps.nddg += 1;
01719         }
01720     }
01721             
01722     /* Right, now loop through all the darks and get their exposure times */
01723 
01724     for (i = 0; i < ps.ndarks; i++) {
01725         frame = cpl_frameset_get_frame(ps.darklist,i);
01726         plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
01727         if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
01728             cpl_msg_warning(fctid,"No exposure time found in %s",
01729                             cpl_frame_get_filename(frame));
01730             cpl_propertylist_delete(plist);
01731             continue;
01732         }
01733         cpl_propertylist_delete(plist);
01734 
01735         /* Search the domedark groups to see if this dark fits into one of
01736            the defined groups. If not, then ignore it. If it does, then
01737            add it into the dark frameset */
01738 
01739         found = 0;
01740         for (j = 0; j < ps.nddg; j++) {
01741             if (ps.ddg[j].exptime == texp) {
01742                 found = 1;
01743                 break;
01744             }
01745         }
01746         if (found) {
01747             cpl_frameset_insert(ps.ddg[j].darks,cpl_frame_duplicate(frame));
01748             ps.ddg[j].ndarks += 1;
01749         }
01750     }
01751 
01752     /* Now go through the domedark groups and ditch any that don't have any
01753        dark frames */
01754 
01755     i = 0;
01756     while (i < ps.nddg) {
01757         if (ps.ddg[i].ndarks == 0) {
01758             cpl_msg_warning(fctid,
01759                             "No dark frames exist for exposure %g\nThrowing these away",
01760                             ps.ddg[i].exptime);
01761             freeframeset(ps.ddg[i].darks);
01762             freeframeset(ps.ddg[i].domes);
01763             for (j = i+1; j < ps.nddg; j++)
01764                 ps.ddg[j-1] = ps.ddg[j];
01765             ps.nddg -= 1;
01766         } else
01767             i++;
01768     }
01769 
01770     /* Allocate some space for vir_fits arrays for processed domes */
01771 
01772     for (i = 0; i < ps.nddg; i++) {
01773         ps.ddg[i].proc = cpl_malloc(ps.ddg[i].ndomes*sizeof(vir_fits *));
01774         for (j = 0; j < ps.ddg[i].ndomes; j++) 
01775             ps.ddg[i].proc[j] = NULL;
01776     }                
01777 
01778     /* Resize the output array and return so long as there is anything
01779        left. If there isn't then signal a major error */
01780 
01781     if (ps.nddg > 0) {
01782         ps.ddg = cpl_realloc(ps.ddg,ps.nddg*sizeof(ddgrp));
01783         return(0);
01784     } else {
01785         cpl_msg_error(fctid,"There are no darks defined for input domes");
01786         return(-1);
01787     }
01788 }
01789 
01790 /*---------------------------------------------------------------------------*/
01799 /*---------------------------------------------------------------------------*/
01800 
01801 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
01802                                                 parquet *p, int np) {
01803     int i,ist,ifn,jst,jfn,n,jind2,iind2,jj,nx,ii;
01804     parquet *pp;
01805     double *d;
01806     float *tmp,*data;
01807 
01808     /* Get the workspace for the output result */
01809 
01810     d = cpl_malloc(np*sizeof(double));
01811 
01812     /* Get the input data array */
01813 
01814     nx = (int)cpl_image_get_size_x(vircam_fits_get_image(fframe));
01815     data = cpl_image_get_data_float(vircam_fits_get_image(fframe));
01816 
01817     /* Get some workspace for doing the median calculations */
01818 
01819     tmp = cpl_malloc(SUBSET*SUBSET*sizeof(float));
01820 
01821     /* Loop for each channel in the parquet structure */
01822 
01823     for (i = 0; i < np; i++) {
01824         pp = p + i;
01825 
01826         /* Take the central part of the channel */
01827 
01828         ist = ((pp->delta_i)/2 - SUBSET2);
01829         ifn = ist + SUBSET - 1;
01830         jst = ((pp->delta_j)/2 - SUBSET2);
01831         jfn = jst + SUBSET - 1;
01832 
01833         /* Put the data into the workspace and do a median */
01834 
01835         n = 0;
01836         for (jj = jst; jj <= jfn; jj++) {
01837             jind2 = (jj + pp->iymin - 1)*nx;
01838             for (ii = ist; ii <= ifn; ii++) {
01839                 iind2 = jind2 + ii + pp->ixmin - 1;
01840                 if (bpm[iind2] == 0) 
01841                     tmp[n++] = data[iind2];
01842             }
01843         }
01844         d[i] = (double)vircam_med(tmp,NULL,(long)n);
01845     }
01846 
01847     /* Tidy and get out of here */
01848 
01849     freespace(tmp);
01850     return(d);
01851 }
01852 
01853 /*---------------------------------------------------------------------------*/
01865 /*---------------------------------------------------------------------------*/
01866 
01867 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
01868                                          int nchan, double *facrng, 
01869                                          double *maxdiff) {
01870     int i,ist,ifn,j;
01871     double *factors,sum,midval,minfac,maxfac;
01872 
01873     /* Get some memory for the output array */
01874 
01875     factors = cpl_malloc(nim*sizeof(double));
01876 
01877     /* First sort the data into order of mjd */
01878 
01879     vircam_mjdsort(fdata,mjd,nim);
01880 
01881     /* Which index is the midpoint? */
01882 
01883     if (nim % 2 == 0) {
01884         ist = nim/2 - 1;
01885         ifn = ist + 1;
01886     } else {
01887         ist = nim/2;
01888         ifn = ist;
01889     }
01890 
01891     /* Loop for each channel */
01892 
01893     for (i = 0; i < nchan; i++) {
01894         
01895         /* Get midpoint value */
01896 
01897         midval = 0.5*(fdata[ist][i] + fdata[ifn][i]);
01898 
01899         /* Now normalise all the ith channels by this value */
01900 
01901         for (j = 0; j < nim; j++) 
01902             fdata[j][i] /= midval;
01903     }
01904 
01905     /* Now loop for each image and average the values for all the channels in
01906        in image */
01907 
01908     *maxdiff = 0.0;
01909     maxfac = 0.0;
01910     minfac = 0.0;
01911     for (j = 0; j < nim; j++) {
01912         sum = 0.0;
01913         for (i = 0; i < nchan; i++)
01914             sum += fdata[j][i];
01915         factors[j] = sum/(double)nchan;
01916         if (j == 0) {
01917             maxfac = factors[j];
01918             minfac = factors[j];
01919         } else {
01920             minfac = min(minfac,factors[j]);
01921             maxfac = max(maxfac,factors[j]);
01922             *maxdiff = max(*maxdiff,fabs(factors[j]-factors[j-1]));
01923         }
01924     }
01925     *facrng = maxfac - minfac;
01926 
01927     /* Get out of here */
01928 
01929     return(factors);
01930 }
01931     
01932 /*---------------------------------------------------------------------------*/
01940 /*---------------------------------------------------------------------------*/
01941 
01942 static void vircam_mjdsort(double **fdata, double *mjd, int n) {
01943     int iii,ii,i,ifin,j;
01944     double tmpmjd,*tmpdata;
01945 
01946 
01947     iii = 2;
01948     while (iii < n)
01949         iii *= 2;
01950     iii = min(n,(3*iii)/4 - 1);
01951 
01952     while (iii > 1) {
01953         iii /= 2;
01954         ifin = n - iii;
01955         for (ii = 0; ii < ifin; ii++) {
01956             i = ii;
01957             j = i + iii;
01958             if (mjd[i] > mjd[j]) {
01959                 tmpmjd = mjd[j];
01960                 tmpdata = fdata[j];
01961                 while (1) {
01962                     mjd[j] = mjd[i];
01963                     fdata[j] = fdata[i];
01964                     j = i;
01965                     i = i - iii;
01966                     if (i < 0 || mjd[0] <= tmpmjd) 
01967                         break;
01968                 }
01969                 mjd[j] = tmpmjd;
01970                 fdata[j] = tmpdata;
01971             }
01972         }
01973     }
01974 }
01975 
01976 /*---------------------------------------------------------------------------*/
01983 /*---------------------------------------------------------------------------*/
01984 
01985 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows) {
01986     int i;
01987     char colname[16];
01988     cpl_table *t;
01989 
01990     /* Create a new table */
01991 
01992     t = cpl_table_new(nrows);
01993 
01994     /* Add the first few columns */
01995 
01996     cpl_table_new_column(t,"filename",CPL_TYPE_STRING);
01997     cpl_table_new_column(t,"exptime",CPL_TYPE_DOUBLE);
01998     cpl_table_set_column_unit(t,"exptime","seconds");
01999     cpl_table_new_column(t,"mjd",CPL_TYPE_DOUBLE);
02000     cpl_table_set_column_unit(t,"mjd","days");
02001 
02002     /* Add columns for each of the channels' raw median flux and linearised 
02003        median flux */
02004 
02005     for (i = 1; i <= np; i++) {
02006         (void)snprintf(colname,16,"rawflux_%02d",i);
02007         cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
02008         cpl_table_set_column_unit(t,colname,"ADU");
02009         (void)snprintf(colname,16,"linflux_%02d",i);
02010         cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
02011         cpl_table_set_column_unit(t,colname,"ADU");
02012     }
02013 
02014     /* Finally add the correction factors that were used */
02015 
02016     cpl_table_new_column(t,"adjust_fac",CPL_TYPE_DOUBLE);
02017 
02018     /* Right, get out of here */
02019 
02020     return(t);
02021 }
02022 
02023 /*---------------------------------------------------------------------------*/
02027 /*---------------------------------------------------------------------------*/
02028 
02029 static void vircam_linearity_analyse_init(void) {
02030     ps.labels = NULL;
02031     ps.domelist = NULL;
02032     ps.darklist = NULL;
02033     ps.domecheck = NULL;
02034     ps.darkcheck = NULL;
02035     ps.ndomes = 0;
02036     ps.ndarks = 0;
02037     ps.ndomecheck = 0;
02038     ps.ndarkcheck = 0;
02039     ps.chanfrm = NULL;
02040     ps.chantab = NULL;
02041     ps.lchantab = NULL;
02042     ps.flatlist = NULL;
02043     ps.bpm_array = NULL;
02044     ps.ddg = NULL;
02045     ps.plist = NULL;
02046     ps.elist = NULL;
02047     ps.phupaf = NULL;
02048     ps.diag1 = NULL;
02049     ps.diag2 = NULL;
02050     ps.inherit = NULL;
02051 }
02052 
02053 /*---------------------------------------------------------------------------*/
02057 /*---------------------------------------------------------------------------*/
02058 
02059 static void vircam_linearity_analyse_tidy(int level) {
02060     int i;
02061 
02062     freetfits(ps.chantab);
02063     freearray(ps.bpm_array);
02064     freefitslist(ps.flatlist,ps.nflatlist);
02065     freetable(ps.lchantab);
02066     freepropertylist(ps.plist);
02067     freepropertylist(ps.elist);
02068     freetable(ps.diag1);
02069     freetable(ps.diag2);
02070     if (level == 1)
02071         return;
02072     
02073     freespace(ps.labels);
02074     freeframeset(ps.domelist);
02075     freeframeset(ps.darklist);
02076     freeframeset(ps.domecheck);
02077     freeframeset(ps.darkcheck);
02078     freeframe(ps.chanfrm);
02079     if (ps.ddg != NULL) {
02080         for (i = 0; i < ps.nddg; i++) {
02081             freeframeset(ps.ddg[i].darks);
02082             freeframeset(ps.ddg[i].domes);
02083             freefitslist(ps.ddg[i].proc,ps.ddg[i].ndomes);
02084         }
02085         freespace(ps.ddg);
02086     }
02087     freepropertylist(ps.phupaf);
02088 }
02089 
02092 /* 
02093 
02094 $Log: vircam_linearity_analyse.c,v $
02095 Revision 1.65  2012/01/16 12:32:18  jim
02096 A few more changes to fit in with cpl6
02097 
02098 Revision 1.64  2012/01/15 17:40:09  jim
02099 Minor modifications to take into accout the changes in cpl API for v6
02100 
02101 Revision 1.63  2010/12/09 13:20:26  jim
02102 Default polynomial order is now 4
02103 
02104 Revision 1.62  2010/07/02 07:17:35  jim
02105 Fixed typo
02106 
02107 Revision 1.61  2010/06/30 12:42:00  jim
02108 A few fixes to stop compiler compaints
02109 
02110 Revision 1.60  2010/03/09 14:29:55  jim
02111 Now modified ESO PRO DATANCOM to reflect the number of images used in
02112 the analysis
02113 
02114 Revision 1.59  2009/09/09 09:50:21  jim
02115 Modified to try and get headers right
02116 
02117 Revision 1.58  2009/07/13 08:17:57  jim
02118 Fixed bug that meant saturation level wasn't being propogated
02119 
02120 Revision 1.57  2009/06/23 05:22:26  jim
02121 Adds saturation back into the channel table header
02122 
02123 Revision 1.56  2008/12/08 06:32:42  jim
02124 Changed 'missing dark frame' error to warning
02125 
02126 Revision 1.55  2008/12/05 13:28:32  jim
02127 Fixed save routine so that the correct version of PRO CATG is written to the
02128 paf file
02129 
02130 Revision 1.54  2008/10/01 04:59:13  jim
02131 Added call to vircam_frameset_fexists to check input frameset
02132 
02133 Revision 1.53  2008/09/30 11:33:00  jim
02134 Fixed bug where saturation flag wasn't being set to OK.
02135 
02136 Revision 1.52  2008/09/29 11:23:00  jim
02137 Minor fix to docs
02138 
02139 Revision 1.51  2008/01/22 19:47:56  jim
02140 New version to implement new algorithm
02141 
02142 Revision 1.50  2007/11/26 09:58:49  jim
02143 Now fails if given observation files done with NDIT != 1
02144 
02145 Revision 1.49  2007/11/23 18:34:28  jim
02146 fixed memory allocation bug
02147 
02148 Revision 1.48  2007/11/22 12:36:55  jim
02149 Modified to create diagnostic tables
02150 
02151 Revision 1.47  2007/11/20 09:41:13  jim
02152 Added ability to alter dome sequence stats by using the monitoring exposures
02153 
02154 Revision 1.46  2007/11/14 10:42:25  jim
02155 Substantial changes to incorporate new linearity analysis algorithm and to
02156 restrict the amount of memory required to do the analysis (especially
02157 the BPM work)
02158 
02159 Revision 1.45  2007/09/07 13:32:12  jim
02160 uses a sorted framelist to ensure that the correct information is given
02161 to the output product header
02162 
02163 Revision 1.44  2007/09/06 21:37:53  jim
02164 fixed call to vircam_dfs_setup_product_ routines to use the full input
02165 frameset
02166 
02167 Revision 1.43  2007/08/29 09:20:33  jim
02168 Primary header is now derived from the same header that forms the PAF rather
02169 than starting off empty and allowing CPL to copy everything it thinks you
02170 want...
02171 
02172 Revision 1.42  2007/08/23 09:02:03  jim
02173 Modified to check domes for DETLIVE before checking darks
02174 
02175 Revision 1.41  2007/07/09 13:21:55  jim
02176 Modified to use new version of vircam_exten_range
02177 
02178 Revision 1.40  2007/06/13 08:11:27  jim
02179 Modified docs to reflect changes in DFS tags
02180 
02181 Revision 1.39  2007/04/30 09:40:17  jim
02182 Added more stuff to paf files
02183 
02184 Revision 1.38  2007/04/04 10:36:07  jim
02185 Fixed typo preventing output of main PAF. Also modified to use dfs tags
02186 
02187 Revision 1.37  2007/03/29 12:19:38  jim
02188 Little changes to improve documentation
02189 
02190 Revision 1.36  2007/03/01 12:41:49  jim
02191 Modified slightly after code checking
02192 
02193 Revision 1.35  2007/02/19 21:13:04  jim
02194 added bad pixel number QC parameter
02195 
02196 Revision 1.34  2007/02/15 11:54:09  jim
02197 Modified to make a distinction between initial channel table and one that
02198 has the proper linearity information
02199 
02200 Revision 1.33  2007/02/15 06:59:38  jim
02201 Added ability to write QC paf files
02202 
02203 Revision 1.32  2007/02/07 10:12:40  jim
02204 Removed calls to vircam_ndit_correct as this is now no longer necessary
02205 
02206 Revision 1.31  2007/02/06 13:11:12  jim
02207 Fixed entry for PRO dictionary in cpl_dfs_set_product_header
02208 
02209 Revision 1.30  2006/12/13 11:45:36  jim
02210 Fixed scaling of sigma error
02211 
02212 Revision 1.29  2006/12/11 22:47:12  jim
02213 Fixed subtle bug in the way that stats were being done.
02214 
02215 Revision 1.28  2006/11/27 12:15:08  jim
02216 changed calls to cpl_propertylist_append to cpl_propertylist_update
02217 
02218 Revision 1.27  2006/11/10 09:23:46  jim
02219 Fixed save routine so to use a new version of vircam_chantab_new
02220 
02221 Revision 1.26  2006/10/31 10:27:27  jim
02222 Fixed a few bugs and modified to make sure than an extension name appear
02223 in each fits extension
02224 
02225 Revision 1.25  2006/09/09 16:49:40  jim
02226 Header comment update
02227 
02228 Revision 1.24  2006/09/08 09:20:22  jim
02229 major upgrade to main processing routine: to deal with bad input better; to
02230 write out dummy results in the case of failure; to combine raw darks on the
02231 fly for use in dark correction, rather than using master darks;
02232 
02233 Revision 1.23  2006/08/03 13:26:44  jim
02234 fixed another typo
02235 
02236 Revision 1.22  2006/08/03 10:36:32  jim
02237 Fixed typo
02238 
02239 Revision 1.21  2006/06/20 19:06:38  jim
02240 Added correction for ndit. Now adjusts the value of norder if the number
02241 of frames given is too small for the order of polynomial requested
02242 
02243 Revision 1.20  2006/06/15 09:58:58  jim
02244 Minor changes to docs
02245 
02246 Revision 1.19  2006/06/06 13:03:42  jim
02247 Fixed scaling that was causing funny stats
02248 
02249 Revision 1.18  2006/05/27 21:40:06  jim
02250 Bad pixels are now defined by a number of sigma above or below the mean
02251 
02252 Revision 1.17  2006/05/09 09:30:47  jim
02253 Fixed _save routine so that bad pixel mask is saved with unsigned char
02254 data type
02255 
02256 Revision 1.16  2006/05/08 12:32:12  jim
02257 Changed default calling parameters for vircam_imcombine
02258 
02259 Revision 1.15  2006/05/04 11:53:15  jim
02260 Fixed the way the _save routine works to be more consistent with the
02261 standard CPL way of doing things
02262 
02263 Revision 1.14  2006/05/03 12:55:17  jim
02264 Fixed some memory leaks
02265 
02266 Revision 1.13  2006/05/02 13:26:32  jim
02267 fixed bug where the wrong amount of memory was being allocated for the dark
02268 exposure times
02269 
02270 Revision 1.12  2006/05/02 11:36:29  jim
02271 fixed illegal propertylist_delete calls
02272 
02273 Revision 1.11  2006/04/27 09:46:01  jim
02274 Modified DFS frame types to conform to new dictionary
02275 
02276 Revision 1.10  2006/04/25 13:45:57  jim
02277 Fixed to adhere to new calling sequence for vircam_dfs routines
02278 
02279 Revision 1.9  2006/04/24 12:12:59  jim
02280 Fixed --help documentation and sorted out filename extension problem
02281 (.fit -> .fits)
02282 
02283 Revision 1.8  2006/04/20 11:31:34  jim
02284 Added bad pixel masking
02285 
02286 Revision 1.7  2006/03/23 21:18:45  jim
02287 Minor changes mainly to comment headers
02288 
02289 Revision 1.6  2006/03/22 14:02:51  jim
02290 cosmetic changes to keep lint happy
02291 
02292 Revision 1.5  2006/03/22 12:13:51  jim
02293 Modified to use new vircam_mask capability
02294 
02295 Revision 1.4  2006/03/15 10:43:40  jim
02296 Fixed a few things
02297 
02298 Revision 1.3  2006/03/03 14:29:06  jim
02299 Now calls routines with vir_fits.
02300 
02301 Revision 1.2  2006/02/22 10:01:22  jim
02302 Added full documentation
02303 
02304 Revision 1.1  2006/02/18 11:49:58  jim
02305 new file
02306 
02307 
02308 */

Generated on 15 Mar 2012 for VIRCAM Pipeline by  doxygen 1.6.1