00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030 #ifdef HAVE_CONFIG_H
00031 #include <config.h>
00032 #endif
00033
00034 #include <stdio.h>
00035 #include <cpl.h>
00036 #include <math.h>
00037
00038 #include "vircam_utils.h"
00039 #include "vircam_pfits.h"
00040 #include "vircam_dfs.h"
00041 #include "vircam_mods.h"
00042 #include "vircam_channel.h"
00043 #include "vircam_stats.h"
00044 #include "vircam_paf.h"
00045
00046
00047
00048 static int vircam_linearity_analyse_create(cpl_plugin *) ;
00049 static int vircam_linearity_analyse_exec(cpl_plugin *) ;
00050 static int vircam_linearity_analyse_destroy(cpl_plugin *) ;
00051 static int vircam_linearity_analyse(cpl_parameterlist *, cpl_frameset *) ;
00052 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
00053 cpl_parameterlist *parlist);
00054 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
00055 cpl_parameterlist *parlist);
00056 static int vircam_linearity_analyse_domedark_groups(void);
00057 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
00058 parquet *p, int np);
00059 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
00060 int nchan, double *facrng,
00061 double *maxdiff);
00062 static void vircam_mjdsort(double **fdata, double *mjd, int n);
00063 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows);
00064 static void vircam_linearity_analyse_init(void);
00065 static void vircam_linearity_analyse_tidy(int level);
00066
00067
00068
00069 static struct {
00070
00071
00072
00073 int norder;
00074 float lthr;
00075 float hthr;
00076 int maxbpmfr;
00077 int adjust;
00078 int diagnostic;
00079 int extenum;
00080
00081
00082
00083 float linearity;
00084 float linerror;
00085 float bad_pixel_stat;
00086 int bad_pixel_num;
00087 float facrng;
00088 float maxdiff;
00089
00090 } vircam_linearity_analyse_config;
00091
00092 typedef struct {
00093 cpl_frameset *darks;
00094 cpl_frameset *domes;
00095 int ndarks;
00096 int ndomes;
00097 float exptime;
00098 unsigned char flag;
00099 vir_fits **proc;
00100 } ddgrp;
00101
00102 #define OK_FLAG 0
00103 #define SATURATE_FLAG 1
00104
00105 #define SUBSET 128
00106 #define SUBSET2 (SUBSET/2)
00107
00108 static struct {
00109 cpl_size *labels;
00110 cpl_frameset *domelist;
00111 int ndomes;
00112 cpl_frameset *darklist;
00113 int ndarks;
00114 cpl_frameset *domecheck;
00115 int ndomecheck;
00116 cpl_frameset *darkcheck;
00117 int ndarkcheck;
00118 cpl_frame *inherit;
00119 cpl_frame *chanfrm;
00120 vir_tfits *chantab;
00121 cpl_table *lchantab;
00122 cpl_array *bpm_array;
00123 ddgrp *ddg;
00124 int nddg;
00125 vir_fits **flatlist;
00126 int nflatlist;
00127 cpl_propertylist *plist;
00128 cpl_propertylist *elist;
00129 int nx;
00130 int ny;
00131 cpl_propertylist *phupaf;
00132 cpl_table *diag1;
00133 cpl_table *diag2;
00134 int nfdata;
00135 int nuse;
00136 } ps;
00137
00138 static int isfirst;
00139 static int dummy;
00140 static float sat;
00141 static cpl_frame *product_frame_chantab = NULL;
00142 static cpl_frame *product_frame_bpm = NULL;
00143 static cpl_frame *product_frame_diag1 = NULL;
00144 static cpl_frame *product_frame_diag2 = NULL;
00145
00146 static char vircam_linearity_analyse_description[] =
00147 "vircam_linearity_analyse -- VIRCAM linearity mapping recipe.\n\n"
00148 "Form master dark images from the input raw frames and use these to\n"
00149 "dark correct a series of dome flat exposures Using the dark\n"
00150 "corrected dome flat series, work out linearity coefficients for\n"
00151 "each data channel. The program expects the following files in the SOF\n"
00152 " Tag Description\n"
00153 " -----------------------------------------------------------------------\n"
00154 " %-21s A list of raw dome flat images\n"
00155 " %-21s A list of raw dark images\n"
00156 " %-21s The channel table\n"
00157 " %-21s A list of raw dome flat images at the monitor exposure time\n"
00158 " %-21s A list of raw dark images at the monitor exposure time\n"
00159 "The first three of these are required. The last two are only required if"
00160 "the light source monitoring algorithm is to be used"
00161 "\n";
00162
00284
00285
00286
00294
00295
00296 int cpl_plugin_get_info(cpl_pluginlist *list) {
00297 cpl_recipe *recipe = cpl_calloc(1,sizeof(*recipe));
00298 cpl_plugin *plugin = &recipe->interface;
00299 char alldesc[SZ_ALLDESC];
00300 (void)snprintf(alldesc,SZ_ALLDESC,vircam_linearity_analyse_description,
00301 VIRCAM_LIN_DOME_RAW,VIRCAM_LIN_DARK_RAW,
00302 VIRCAM_CAL_CHANTAB_INIT,VIRCAM_LIN_DOME_CHECK,
00303 VIRCAM_LIN_DARK_CHECK);
00304
00305 cpl_plugin_init(plugin,
00306 CPL_PLUGIN_API,
00307 VIRCAM_BINARY_VERSION,
00308 CPL_PLUGIN_TYPE_RECIPE,
00309 "vircam_linearity_analyse",
00310 "VIRCAM linearity analysis recipe",
00311 alldesc,
00312 "Jim Lewis",
00313 "jrl@ast.cam.ac.uk",
00314 vircam_get_license(),
00315 vircam_linearity_analyse_create,
00316 vircam_linearity_analyse_exec,
00317 vircam_linearity_analyse_destroy);
00318
00319 cpl_pluginlist_append(list,plugin);
00320
00321 return(0);
00322 }
00323
00324
00333
00334
00335 static int vircam_linearity_analyse_create(cpl_plugin *plugin) {
00336 cpl_recipe *recipe;
00337 cpl_parameter *p;
00338
00339
00340
00341 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00342 recipe = (cpl_recipe *)plugin;
00343 else
00344 return(-1);
00345
00346
00347
00348 recipe->parameters = cpl_parameterlist_new();
00349
00350
00351
00352 p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.norder",
00353 CPL_TYPE_INT,
00354 "Order of polynomial fit",
00355 "vircam.vircam_linearity_analyse",
00356 4,1,6);
00357 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nord");
00358 cpl_parameterlist_append(recipe->parameters,p);
00359
00360
00361
00362 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.lthr",
00363 CPL_TYPE_DOUBLE,
00364 "Lower bad pixel threshold",
00365 "vircam.vircam_linearity_analyse",8.0);
00366 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"lthr");
00367 cpl_parameterlist_append(recipe->parameters,p);
00368
00369
00370
00371 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.hthr",
00372 CPL_TYPE_DOUBLE,
00373 "Upper bad pixel threshold",
00374 "vircam.vircam_linearity_analyse",8.0);
00375 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"hthr");
00376 cpl_parameterlist_append(recipe->parameters,p);
00377
00378
00379
00380 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.maxbpmfr",
00381 CPL_TYPE_INT,
00382 "Maximum # frames used in bpm analysis",
00383 "vircam.vircam_linearity_analyse",10);
00384 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"maxbpmfr");
00385 cpl_parameterlist_append(recipe->parameters,p);
00386
00387
00388
00389
00390 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.adjust",
00391 CPL_TYPE_BOOL,
00392 "Adjust stats with monitor set",
00393 "vircam.vircam_linearity_analyse",1);
00394 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"adjust");
00395 cpl_parameterlist_append(recipe->parameters,p);
00396
00397
00398
00399 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.diagnostic",
00400 CPL_TYPE_BOOL,
00401 "Write out diagnostic tables",
00402 "vircam.vircam_linearity_analyse",0);
00403 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"diagnostic");
00404 cpl_parameterlist_append(recipe->parameters,p);
00405
00406
00407
00408 p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.extenum",
00409 CPL_TYPE_INT,
00410 "Extension number to be done, 0 == all",
00411 "vircam.vircam_linearity_analyse",
00412 1,0,16);
00413 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
00414 cpl_parameterlist_append(recipe->parameters,p);
00415
00416
00417
00418 return(0);
00419 }
00420
00421
00427
00428
00429 static int vircam_linearity_analyse_exec(cpl_plugin *plugin) {
00430 cpl_recipe *recipe;
00431
00432
00433
00434 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00435 recipe = (cpl_recipe *)plugin;
00436 else
00437 return(-1);
00438
00439 return(vircam_linearity_analyse(recipe->parameters,recipe->frames));
00440 }
00441
00442
00448
00449
00450 static int vircam_linearity_analyse_destroy(cpl_plugin *plugin) {
00451 cpl_recipe *recipe ;
00452
00453
00454
00455 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00456 recipe = (cpl_recipe *)plugin;
00457 else
00458 return(-1);
00459
00460 cpl_parameterlist_delete(recipe->parameters);
00461 return(0);
00462 }
00463
00464
00471
00472
00473 static int vircam_linearity_analyse(cpl_parameterlist *parlist,
00474 cpl_frameset *framelist) {
00475 const char *fctid="vircam_linearity_analyse";
00476 char colname[16];
00477 int i,jst,jfn,j,status,k,nbad,ngood,krem,n;
00478 int ndarks,ndomes,kk,live,ngood_flats,*bpm,nalloc,adjust,ndit;
00479 cpl_size nlab;
00480 long np;
00481 float med,mindit,*exps,badfrac,expt;
00482 unsigned char *rejmask,*rejplus;
00483 double *dexps,**fdata,*d,*mjds,*mjdcheck,mjd,**cdata,*cf,fac;
00484 double facrng,maxdiff,*lindata;
00485 vir_fits **darks,**domes,*test,*outdark,*fframe;
00486 parquet *pp;
00487 cpl_parameter *p;
00488 cpl_propertylist *drs,*plist;
00489 cpl_image *outimage;
00490 cpl_frame *frame;
00491
00492
00493
00494 if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
00495 cpl_msg_error(fctid,"Input framelist NULL or has no input data");
00496 return(-1);
00497 }
00498
00499
00500
00501 if (vircam_frameset_fexists(framelist) != VIR_OK) {
00502 cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
00503 return(-1);
00504 }
00505
00506
00507
00508 vircam_linearity_analyse_init();
00509
00510
00511
00512 p = cpl_parameterlist_find(parlist,
00513 "vircam.vircam_linearity_analyse.norder");
00514 vircam_linearity_analyse_config.norder = cpl_parameter_get_int(p);
00515 p = cpl_parameterlist_find(parlist,
00516 "vircam.vircam_linearity_analyse.lthr");
00517 vircam_linearity_analyse_config.lthr = (float)cpl_parameter_get_double(p);
00518 p = cpl_parameterlist_find(parlist,
00519 "vircam.vircam_linearity_analyse.hthr");
00520 vircam_linearity_analyse_config.hthr = (float)cpl_parameter_get_double(p);
00521 p = cpl_parameterlist_find(parlist,
00522 "vircam.vircam_linearity_analyse.maxbpmfr");
00523 vircam_linearity_analyse_config.maxbpmfr = cpl_parameter_get_int(p);
00524 p = cpl_parameterlist_find(parlist,
00525 "vircam.vircam_linearity_analyse.adjust");
00526 vircam_linearity_analyse_config.adjust = cpl_parameter_get_bool(p);
00527 p = cpl_parameterlist_find(parlist,
00528 "vircam.vircam_linearity_analyse.diagnostic");
00529 vircam_linearity_analyse_config.diagnostic = cpl_parameter_get_bool(p);
00530 p = cpl_parameterlist_find(parlist,
00531 "vircam.vircam_linearity_analyse.extenum");
00532 vircam_linearity_analyse_config.extenum = cpl_parameter_get_int(p);
00533
00534
00535
00536 if (vircam_dfs_set_groups(framelist) != VIR_OK) {
00537 cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
00538 vircam_linearity_analyse_tidy(2);
00539 return(-1);
00540 }
00541
00542
00543
00544 if ((ps.labels = cpl_frameset_labelise(framelist,vircam_compare_tags,
00545 &nlab)) == NULL) {
00546 cpl_msg_error(fctid,"Cannot labelise the input frames");
00547 vircam_linearity_analyse_tidy(2);
00548 return(-1);
00549 }
00550
00551
00552
00553 if ((ps.domelist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00554 VIRCAM_LIN_DOME_RAW)) == NULL) {
00555 cpl_msg_error(fctid,"Cannot find dome flat frames in input frameset");
00556 vircam_linearity_analyse_tidy(2);
00557 return(-1);
00558 }
00559 ps.ndomes = cpl_frameset_get_size(ps.domelist);
00560 ps.inherit = cpl_frameset_get_first(ps.domelist);
00561
00562
00563
00564 plist = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_frame(ps.domelist,0)),0);
00565 (void)vircam_pfits_get_ndit(plist,&ndit);
00566 freepropertylist(plist);
00567 if (ndit != 1) {
00568 cpl_msg_error(fctid,
00569 "NDIT=%" CPL_SIZE_FORMAT ". Recipe requires that ndit == 1",
00570 (cpl_size)ndit);
00571 vircam_linearity_analyse_tidy(2);
00572 return(-1);
00573 }
00574
00575
00576
00577 if ((ps.darklist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00578 VIRCAM_LIN_DARK_RAW)) == NULL) {
00579 cpl_msg_error(fctid,"Cannot find dark frames in input frameset");
00580 vircam_linearity_analyse_tidy(2);
00581 return(-1);
00582 }
00583 ps.ndarks = cpl_frameset_get_size(ps.darklist);
00584
00585
00586
00587
00588 if (vircam_linearity_analyse_config.adjust) {
00589 if ((ps.domecheck = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00590 VIRCAM_LIN_DOME_CHECK)) == NULL) {
00591 cpl_msg_info(fctid,"No monitor frames found in sof. No adjustments made to stats");
00592 vircam_linearity_analyse_config.adjust = 0;
00593 ps.ndomecheck = 0;
00594 } else {
00595 ps.ndomecheck = cpl_frameset_get_size(ps.domecheck);
00596 if ((ps.darkcheck = vircam_frameset_subgroup(framelist,ps.labels,
00597 nlab,VIRCAM_LIN_DARK_CHECK)) == NULL) {
00598 cpl_msg_info(fctid,"No darks for monitor frames found in sof. No adjustments made to stats");
00599 vircam_linearity_analyse_config.adjust = 0;
00600 ps.ndomecheck = 0;
00601 freeframeset(ps.domecheck);
00602 ps.ndarkcheck = 0;
00603 } else {
00604 ps.ndarkcheck = cpl_frameset_get_size(ps.darkcheck);
00605 }
00606 }
00607 }
00608
00609
00610
00611 if ((ps.chanfrm = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00612 VIRCAM_CAL_CHANTAB_INIT)) == NULL) {
00613 cpl_msg_error(fctid,"No initial channel table found");
00614 vircam_linearity_analyse_tidy(2);
00615 return(-1);
00616 }
00617
00618
00619
00620 if (vircam_linearity_analyse_domedark_groups() != 0) {
00621 vircam_linearity_analyse_tidy(2);
00622 return(-1);
00623 }
00624
00625
00626
00627
00628 if (ps.nddg < vircam_linearity_analyse_config.norder+1) {
00629 cpl_msg_warning(fctid,
00630 "Number of exposure times is too small: %" CPL_SIZE_FORMAT ", order: %" CPL_SIZE_FORMAT "\nTaking fit down to order %" CPL_SIZE_FORMAT,
00631 (cpl_size)(ps.nddg),
00632 (cpl_size)(vircam_linearity_analyse_config.norder),
00633 (cpl_size)(ps.nddg-1));
00634 vircam_linearity_analyse_config.norder = ps.nddg - 1;
00635 }
00636
00637
00638
00639
00640
00641 vircam_exten_range(vircam_linearity_analyse_config.extenum,
00642 (const cpl_frame *)cpl_frameset_get_frame(ps.ddg[0].darks,0),
00643 &jst,&jfn);
00644 if (jst == -1 || jfn == -1) {
00645 cpl_msg_error(fctid,"Unable to continue");
00646 vircam_linearity_analyse_tidy(2);
00647 return(-1);
00648 }
00649
00650
00651
00652 for (j = jst; j <= jfn; j++) {
00653 cpl_msg_info(fctid,"Beginning BPM work on extension %" CPL_SIZE_FORMAT,
00654 (cpl_size)j);
00655 isfirst = (j == jst);
00656 dummy = 0;
00657 vircam_linearity_analyse_config.bad_pixel_stat = 0.0;
00658 vircam_linearity_analyse_config.bad_pixel_num = 0;
00659 vircam_linearity_analyse_config.linerror = 0.0;
00660 vircam_linearity_analyse_config.linearity = 0.0;
00661 vircam_linearity_analyse_config.facrng = 0.0;
00662 vircam_linearity_analyse_config.maxdiff = 0.0;
00663
00664
00665
00666 test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[0].domes),
00667 CPL_TYPE_FLOAT,j);
00668 ps.plist = cpl_propertylist_duplicate(vircam_fits_get_phu(test));
00669 ps.elist = cpl_propertylist_duplicate(vircam_fits_get_ehu(test));
00670 ps.nx = (int)cpl_image_get_size_x(vircam_fits_get_image(test));
00671 ps.ny = (int)cpl_image_get_size_y(vircam_fits_get_image(test));
00672 vircam_fits_delete(test);
00673
00674
00675
00676
00677 ps.chantab = vircam_tfits_load(ps.chanfrm,j);
00678 if (ps.chantab == NULL) {
00679 cpl_msg_error(fctid,
00680 "Channel table extension %" CPL_SIZE_FORMAT " failed to load",
00681 (cpl_size)j);
00682 dummy = 1;
00683 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00684 return(-1);
00685 continue;
00686 } else if (vircam_chantab_verify(vircam_tfits_get_table(ps.chantab))
00687 != VIR_OK) {
00688 cpl_msg_error(fctid,
00689 "Channel table extension %" CPL_SIZE_FORMAT " has errors",
00690 (cpl_size)j);
00691 dummy = 1;
00692 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00693 return(-1);
00694 continue;
00695 }
00696 if (vircam_pfits_get_saturation(vircam_tfits_get_ehu(ps.chantab),
00697 &sat) != VIR_OK) {
00698 cpl_msg_error(fctid,
00699 "Channel table extension header %" CPL_SIZE_FORMAT " missing saturation info",
00700 (cpl_size)j);
00701 dummy = 1;
00702 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00703 return(-1);
00704 vircam_linearity_analyse_tidy(1);
00705 continue;
00706 }
00707
00708
00709
00710 vircam_chan_fill(vircam_tfits_get_table(ps.chantab),&pp,&np);
00711
00712
00713
00714 if (vircam_linearity_analyse_config.diagnostic) {
00715 ps.diag1 = vircam_linearity_analyse_diagtab_init(np,ps.ndomes);
00716 if (vircam_linearity_analyse_config.adjust)
00717 ps.diag2 = vircam_linearity_analyse_diagtab_init(np,ps.ndomecheck);
00718 }
00719
00720
00721
00722 if (vircam_pfits_get_detlive((const cpl_propertylist *)ps.elist,&live)
00723 != VIR_OK) {
00724 cpl_msg_error(fctid,"No DET LIVE keyword in this extension");
00725 dummy = 1;
00726 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00727 return(-1);
00728 vircam_linearity_analyse_tidy(1);
00729 continue;
00730 }
00731 if (! live) {
00732 cpl_msg_info(fctid,"Detector flagged dead");
00733 dummy = 1;
00734 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00735 return(-1);
00736 vircam_linearity_analyse_tidy(1);
00737 continue;
00738 }
00739
00740
00741
00742
00743 if (vircam_pfits_get_mindit((const cpl_propertylist *)ps.elist,
00744 &mindit) != VIR_OK) {
00745 cpl_msg_error(fctid,
00746 "No value of MINDIT found in extension %" CPL_SIZE_FORMAT,
00747 (cpl_size)j);
00748 dummy = 1;
00749 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00750 return(-1);
00751 continue;
00752 }
00753
00754
00755
00756
00757
00758 ngood = 0;
00759 exps = cpl_malloc(ps.nddg*sizeof(float));
00760 ngood_flats = 0;
00761 for (i = 0; i < ps.nddg; i++) {
00762 test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[i].domes),
00763 CPL_TYPE_FLOAT,j);
00764 med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
00765 med *= (1.0 + mindit/ps.ddg[i].exptime);
00766 if (med > sat) {
00767 ps.ddg[i].flag = SATURATE_FLAG;
00768 } else {
00769 ngood++;
00770 exps[ngood-1] = ps.ddg[i].exptime;
00771 ngood_flats += ps.ddg[i].ndomes;
00772 ps.ddg[i].flag = OK_FLAG;
00773 }
00774 vircam_fits_delete(test);
00775 }
00776 exps = cpl_realloc(exps,ngood*sizeof(float));
00777
00778
00779
00780 if (ngood < vircam_linearity_analyse_config.norder+1) {
00781 cpl_msg_info(fctid,
00782 "Too few unsaturated flats for linearity fit for extension %" CPL_SIZE_FORMAT,
00783 (cpl_size)j);
00784 dummy = 1;
00785 cpl_free(exps);
00786 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00787 return(-1);
00788 continue;
00789 }
00790
00791
00792
00793 vircam_sort(&exps,ngood,1);
00794
00795
00796
00797
00798 ps.nuse = min(vircam_linearity_analyse_config.maxbpmfr,ngood_flats);
00799 ps.nflatlist = 0;
00800 ps.flatlist = cpl_malloc(ps.nuse*sizeof(vir_fits *));
00801 for (i = ngood-1; i >= 0; i--) {
00802 krem = -1;
00803 for (k = 0; k <= ps.nddg; k++) {
00804 if (ps.ddg[k].exptime == exps[i]) {
00805 krem = k;
00806 break;
00807 }
00808 }
00809
00810
00811
00812 darks = vircam_fits_load_list(ps.ddg[krem].darks,CPL_TYPE_FLOAT,j);
00813 ndarks = ps.ddg[krem].ndarks;
00814 if (darks == NULL) {
00815 cpl_msg_error(fctid,
00816 "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
00817 (cpl_size)j,ps.ddg[krem].exptime);
00818 continue;
00819 }
00820
00821
00822
00823
00824 if (ndarks == 1) {
00825 outdark = vircam_fits_duplicate(darks[0]);
00826 } else {
00827 status = VIR_OK;
00828 (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,&outimage,
00829 &rejmask,&rejplus,&drs,&status);
00830 freespace(rejmask);
00831 freespace(rejplus);
00832 freepropertylist(drs);
00833 if (status != VIR_OK) {
00834 cpl_msg_error(fctid,
00835 "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
00836 (cpl_size)j,ps.ddg[krem].exptime);
00837 freefitslist(darks,ndarks);
00838 continue;
00839 }
00840 outdark = vircam_fits_wrap(outimage,darks[0],NULL,NULL);
00841 }
00842 freefitslist(darks,ndarks);
00843
00844
00845
00846 domes = vircam_fits_load_list(ps.ddg[krem].domes,CPL_TYPE_FLOAT,j);
00847 ndomes = ps.ddg[krem].ndomes;
00848 if (domes == NULL) {
00849 cpl_msg_error(fctid,
00850 "Error loading domes extension %" CPL_SIZE_FORMAT ", exptime %g",
00851 (cpl_size)j,ps.ddg[i].exptime);
00852 freefits(outdark);
00853 continue;
00854 }
00855
00856
00857
00858
00859 for (kk = 0; kk < ndomes; kk++) {
00860 status = VIR_OK;
00861 vircam_darkcor(domes[kk],outdark,1.0,&status);
00862 ps.flatlist[ps.nflatlist] = vircam_fits_duplicate(domes[kk]);
00863 ps.ddg[krem].proc[kk] = ps.flatlist[ps.nflatlist];
00864 ps.nflatlist++;
00865 if (ps.nflatlist == ps.nuse)
00866 break;
00867 }
00868
00869
00870
00871 freefitslist(domes,ndomes);
00872 freefits(outdark);
00873
00874
00875
00876 if (ps.nflatlist == ps.nuse)
00877 break;
00878 }
00879 freespace(exps);
00880 ps.flatlist = cpl_realloc(ps.flatlist,
00881 (ps.nflatlist)*sizeof(vir_fits *));
00882
00883
00884
00885 status = VIR_OK;
00886 (void)vircam_genbpm(ps.flatlist,ps.nflatlist,
00887 vircam_linearity_analyse_config.lthr,
00888 vircam_linearity_analyse_config.hthr,
00889 &(ps.bpm_array),&nbad,&badfrac,&status);
00890 bpm = cpl_array_get_data_int(ps.bpm_array);
00891
00892
00893
00894 vircam_linearity_analyse_config.bad_pixel_num = nbad;
00895 vircam_linearity_analyse_config.bad_pixel_stat = badfrac;
00896
00897
00898
00899
00900
00901 freespace(ps.flatlist);
00902 ps.nflatlist = 0;
00903
00904
00905
00906 nalloc = 16;
00907 fdata = cpl_malloc(nalloc*sizeof(double *));
00908 dexps = cpl_malloc(nalloc*sizeof(double));
00909 mjds = cpl_malloc(nalloc*sizeof(double));
00910
00911
00912
00913
00914
00915 cpl_msg_info(fctid,
00916 "Beginning linearity work on extension %" CPL_SIZE_FORMAT,
00917 (cpl_size)j);
00918 ps.nfdata = 0;
00919 outdark = NULL;
00920 for (i = 0; i < ps.nddg; i++) {
00921 if (ps.ddg[i].flag == SATURATE_FLAG)
00922 continue;
00923 for (k = 0; k < ps.ddg[i].ndomes; k++) {
00924
00925
00926
00927
00928 if (ps.ddg[i].proc[k] == NULL) {
00929 if (outdark == NULL) {
00930
00931
00932
00933 darks = vircam_fits_load_list(ps.ddg[i].darks,
00934 CPL_TYPE_FLOAT,j);
00935 ndarks = ps.ddg[i].ndarks;
00936 if (darks == NULL) {
00937 cpl_msg_error(fctid,
00938 "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
00939 (cpl_size)j,ps.ddg[i].exptime);
00940 continue;
00941 }
00942
00943
00944
00945
00946
00947 if (ps.ddg[i].ndarks == 1) {
00948 outdark = vircam_fits_duplicate(darks[0]);
00949 } else {
00950 status = VIR_OK;
00951 (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
00952 &outimage,&rejmask,&rejplus,
00953 &drs,&status);
00954 freespace(rejmask);
00955 freespace(rejplus);
00956 freepropertylist(drs);
00957 if (status != VIR_OK) {
00958 cpl_msg_error(fctid,
00959 "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
00960 (cpl_size)j,ps.ddg[i].exptime);
00961 freefitslist(darks,ndarks);
00962 continue;
00963 }
00964 outdark = vircam_fits_wrap(outimage,darks[0],
00965 NULL,NULL);
00966 }
00967 freefitslist(darks,ndarks);
00968 }
00969
00970
00971
00972 frame = cpl_frameset_get_frame(ps.ddg[i].domes,k);
00973 fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
00974 vircam_darkcor(fframe,outdark,1.0,&status);
00975
00976
00977
00978 } else {
00979 fframe = ps.ddg[i].proc[k];
00980 }
00981
00982
00983
00984 d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
00985 if (ps.nfdata >= nalloc) {
00986 nalloc += 16;
00987 fdata = cpl_realloc(fdata,nalloc*sizeof(double *));
00988 dexps = cpl_realloc(dexps,nalloc*sizeof(double));
00989 mjds = cpl_realloc(mjds,nalloc*sizeof(double));
00990 }
00991 (void)vircam_pfits_get_mjd(vircam_fits_get_phu(fframe),&mjd);
00992 mjds[ps.nfdata] = mjd;
00993 dexps[ps.nfdata] = (double)(ps.ddg[i].exptime);
00994 fdata[ps.nfdata] = d;
00995
00996
00997
00998
00999 if (ps.diag1 != NULL) {
01000 cpl_table_set_string(ps.diag1,"filename",
01001 (cpl_size)(ps.nfdata),
01002 vircam_fits_get_filename(fframe));
01003 cpl_table_set_double(ps.diag1,"exptime",
01004 (cpl_size)ps.nfdata,
01005 dexps[ps.nfdata]);
01006 cpl_table_set_double(ps.diag1,"mjd",(cpl_size)(ps.nfdata),
01007 mjd);
01008 for (n = 1; n <= np; n++) {
01009 snprintf(colname,16,"rawflux_%02d",n);
01010 cpl_table_set_double(ps.diag1,colname,
01011 (cpl_size)(ps.nfdata),d[n-1]);
01012 }
01013 }
01014 if (ps.ddg[i].proc[k] != NULL) {
01015 freefits(ps.ddg[i].proc[k]);
01016 } else {
01017 freefits(fframe);
01018 }
01019 ps.nfdata++;
01020 }
01021 freefits(outdark);
01022 }
01023 freefits(outdark);
01024 if (ps.diag1 != NULL)
01025 cpl_table_set_size(ps.diag1,(cpl_size)(ps.nfdata));
01026
01027
01028
01029
01030 if (vircam_linearity_analyse_config.adjust) {
01031
01032
01033
01034 cdata = cpl_malloc(ps.ndomecheck*sizeof(double *));
01035
01036
01037
01038
01039 test = vircam_fits_load(cpl_frameset_get_first(ps.domecheck),
01040 CPL_TYPE_FLOAT,j);
01041 (void)vircam_pfits_get_exptime(vircam_fits_get_phu(test),&expt);
01042 med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
01043 med *= (1.0 + mindit/expt);
01044 adjust = 1;
01045 if (med > sat) {
01046 cpl_msg_info(fctid,"Monitor exposures saturated. No drift adjustment made");
01047 adjust = 0;
01048 }
01049 vircam_fits_delete(test);
01050
01051
01052
01053 if (adjust) {
01054
01055 darks = vircam_fits_load_list(ps.darkcheck,CPL_TYPE_FLOAT,j);
01056 ndarks = ps.ndarkcheck;
01057 if (darks == NULL) {
01058 cpl_msg_error(fctid,
01059 "Error loading check darks extension %" CPL_SIZE_FORMAT,
01060 (cpl_size)j);
01061 continue;
01062 }
01063
01064
01065
01066
01067
01068 if (ndarks == 1) {
01069 outdark = vircam_fits_duplicate(darks[0]);
01070 } else {
01071 status = VIR_OK;
01072 (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
01073 &outimage,&rejmask,&rejplus,
01074 &drs,&status);
01075 freespace(rejmask);
01076 freespace(rejplus);
01077 freepropertylist(drs);
01078 if (status != VIR_OK) {
01079 cpl_msg_error(fctid,
01080 "Combine failure extension %" CPL_SIZE_FORMAT " monitor",
01081 (cpl_size)j);
01082 freefitslist(darks,ndarks);
01083 continue;
01084 }
01085 outdark = vircam_fits_wrap(outimage,darks[0],
01086 NULL,NULL);
01087 }
01088 freefitslist(darks,ndarks);
01089
01090
01091
01092
01093 mjdcheck = cpl_malloc(ps.ndomecheck*sizeof(double));
01094 for (i = 0; i < ps.ndomecheck; i++) {
01095 frame = cpl_frameset_get_frame(ps.domecheck,i);
01096 fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
01097 vircam_darkcor(fframe,outdark,1.0,&status);
01098 d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
01099 cdata[i] = d;
01100 vircam_pfits_get_mjd(vircam_fits_get_phu(fframe),&mjd);
01101 vircam_pfits_get_exptime(vircam_fits_get_phu(fframe),&expt);
01102 mjdcheck[i] = mjd;
01103
01104
01105
01106 if (ps.diag2 != NULL) {
01107 cpl_table_set_string(ps.diag2,"filename",(cpl_size)i,
01108 vircam_fits_get_filename(fframe));
01109 cpl_table_set_double(ps.diag2,"exptime",
01110 (cpl_size)i,(double)expt);
01111 cpl_table_set_double(ps.diag2,"mjd",
01112 (cpl_size)i,mjd);
01113 for (n = 1; n <= np; n++) {
01114 snprintf(colname,16,"rawflux_%02d",n);
01115 cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
01116 d[n-1]);
01117 snprintf(colname,16,"linflux_%02d",n);
01118 cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
01119 d[n-1]);
01120 }
01121 }
01122 freefits(fframe);
01123 }
01124 freefits(outdark);
01125
01126
01127
01128 cf = vircam_linearity_tweakfac(cdata,mjdcheck,ps.ndomecheck,
01129 np,&facrng,&maxdiff);
01130 vircam_linearity_analyse_config.facrng = 100.0*(float)facrng;
01131 vircam_linearity_analyse_config.maxdiff = 100.0*(float)maxdiff;
01132 if (ps.diag2 != NULL) {
01133 for (i = 0; i < ps.ndomecheck; i++)
01134 cpl_table_set_double(ps.diag2,"adjust_fac",
01135 (cpl_size)i,cf[i]);
01136 }
01137
01138
01139
01140
01141 for (i = 0; i < ps.nfdata; i++) {
01142 mjd = mjds[i];
01143 krem = -1;
01144 for (k = 0; k < ps.ndomecheck; k++) {
01145 if (mjd < mjdcheck[k]) {
01146 krem = k;
01147 break;
01148 }
01149 }
01150 if (krem == -1) {
01151 fac = cf[ps.ndomecheck-1];
01152 } else if (krem == 0) {
01153 fac = cf[0];
01154 } else {
01155 fac = 0.5*(cf[krem -1] + cf[krem]);
01156 }
01157 for (k = 0; k < np; k++)
01158 fdata[i][k] /= fac;
01159 if (ps.diag1 != NULL)
01160 cpl_table_set_double(ps.diag1,"adjust_fac",
01161 (cpl_size)i,fac);
01162 }
01163
01164
01165
01166 freespace2(cdata,ps.ndomecheck);
01167 freespace(cf);
01168 freespace(mjdcheck);
01169 }
01170 }
01171
01172
01173
01174 freespace(mjds);
01175 vircam_chan_free(np,&pp);
01176
01177
01178
01179
01180 (void)vircam_genlincur(fdata,ps.nfdata,dexps,(double)mindit,ps.chantab,
01181 vircam_linearity_analyse_config.norder,
01182 &(ps.lchantab),&lindata,&status);
01183 if (ps.diag1 != NULL) {
01184 for (i = 0; i < ps.nfdata; i++) {
01185 for (n = 0; n < np; n++) {
01186 snprintf(colname,16,"linflux_%02d",n+1);
01187 cpl_table_set_double(ps.diag1,colname,
01188 (cpl_size)i,lindata[i*np+n]);
01189 }
01190 }
01191 }
01192 freespace2(fdata,ps.nfdata);
01193 freespace(dexps);
01194 freespace(lindata);
01195 if (status != VIR_OK) {
01196 cpl_msg_error(fctid,
01197 "Linearity curve fit failed extension %" CPL_SIZE_FORMAT,
01198 (cpl_size)j);
01199 dummy = 1;
01200 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
01201 return(-1);
01202 vircam_linearity_analyse_tidy(1);
01203 continue;
01204 }
01205
01206
01207
01208
01209 vircam_linearity_analyse_config.linearity =
01210 (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000");
01211 vircam_linearity_analyse_config.linerror =
01212 (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000_err");
01213
01214
01215
01216 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
01217 return(-1);
01218 }
01219
01220
01221
01222 vircam_linearity_analyse_tidy(2);
01223 return(0);
01224 }
01225
01226
01227
01234
01235
01236 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
01237 cpl_parameterlist *parlist) {
01238 cpl_propertylist *plist,*elist,*pafprop;
01239 cpl_image *outimg;
01240 const char *outtab = "lchantab.fits";
01241 const char *outbpm = "bpm.fits";
01242 const char *outtabpaf = "lchantab";
01243 const char *outbpmpaf = "bpm";
01244 const char *outdiag1 = "ldiag1.fits";
01245 const char *outdiag2 = "ldiag2.fits";
01246 const char *fctid = "vircam_linearity_analyse_save";
01247 const char *recipeid = "vircam_linearity_analyse";
01248 int nx,ny,nord,*bpm,i;
01249
01250
01251
01252 nord = vircam_linearity_analyse_config.norder;
01253 if (isfirst) {
01254
01255
01256
01257 product_frame_chantab = cpl_frame_new();
01258 cpl_frame_set_filename(product_frame_chantab,outtab);
01259 cpl_frame_set_tag(product_frame_chantab,VIRCAM_PRO_CHANTAB);
01260 cpl_frame_set_type(product_frame_chantab,CPL_FRAME_TYPE_TABLE);
01261 cpl_frame_set_group(product_frame_chantab,CPL_FRAME_GROUP_PRODUCT);
01262 cpl_frame_set_level(product_frame_chantab,CPL_FRAME_LEVEL_FINAL);
01263
01264
01265
01266 ps.phupaf = vircam_paf_phu_items(ps.plist);
01267 plist = cpl_propertylist_duplicate(ps.plist);
01268 vircam_dfs_set_product_primary_header(plist,product_frame_chantab,
01269 framelist,parlist,
01270 (char *)recipeid,
01271 "PRO-1.15",ps.inherit,0);
01272
01273
01274
01275
01276 elist = cpl_propertylist_duplicate(ps.elist);
01277 cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
01278 vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
01279 framelist,parlist,(char *)recipeid,
01280 "PRO-1.15",ps.inherit);
01281
01282
01283
01284 cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
01285 vircam_linearity_analyse_config.linearity);
01286 cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
01287 "% non-linearity at 10000 ADU");
01288 cpl_propertylist_update_float(elist,"ESO QC LINERROR",
01289 vircam_linearity_analyse_config.linerror);
01290 cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
01291 "% error non-linearity at 10000 ADU");
01292 cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
01293 vircam_linearity_analyse_config.facrng);
01294 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
01295 "total % range in screen variation");
01296 cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
01297 vircam_linearity_analyse_config.maxdiff);
01298 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
01299 "maximum % step in screen variation");
01300 cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
01301
01302
01303
01304 if (dummy == 1) {
01305 vircam_dummy_property(elist);
01306 if (ps.lchantab == NULL)
01307 ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
01308 }
01309
01310
01311
01312 if (cpl_table_save(ps.lchantab,plist,elist,outtab,CPL_IO_DEFAULT)
01313 != CPL_ERROR_NONE) {
01314 cpl_msg_error(fctid,"Cannot save product table extension");
01315 freepropertylist(plist);
01316 freepropertylist(elist);
01317 return(-1);
01318 }
01319 cpl_frameset_insert(framelist,product_frame_chantab);
01320
01321
01322
01323 pafprop = vircam_paf_req_items(elist);
01324 vircam_merge_propertylists(pafprop,ps.phupaf);
01325 vircam_paf_append(pafprop,plist,"ESO INS FILT1 NAME");
01326 vircam_paf_append(pafprop,elist,"ESO PRO CATG");
01327 vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
01328 if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
01329 "QC file",pafprop) != VIR_OK)
01330 cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
01331 cpl_propertylist_delete(pafprop);
01332
01333
01334
01335 freepropertylist(plist);
01336 freepropertylist(elist);
01337
01338
01339
01340 product_frame_bpm = cpl_frame_new();
01341 cpl_frame_set_filename(product_frame_bpm,outbpm);
01342 cpl_frame_set_tag(product_frame_bpm,VIRCAM_PRO_BPM);
01343 cpl_frame_set_type(product_frame_bpm,CPL_FRAME_TYPE_IMAGE);
01344 cpl_frame_set_group(product_frame_bpm,CPL_FRAME_GROUP_PRODUCT);
01345 cpl_frame_set_level(product_frame_bpm,CPL_FRAME_LEVEL_FINAL);
01346
01347
01348
01349 plist = ps.plist;
01350 vircam_dfs_set_product_primary_header(plist,product_frame_bpm,
01351 framelist,parlist,
01352 (char *)recipeid,"PRO-1.15",
01353 ps.inherit,0);
01354
01355
01356
01357 if (cpl_image_save(NULL,outbpm,CPL_TYPE_UCHAR,plist,
01358 CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
01359 cpl_msg_error(fctid,"Cannot save product PHU");
01360 cpl_frame_delete(product_frame_bpm);
01361 return(-1);
01362 }
01363 cpl_frameset_insert(framelist,product_frame_bpm);
01364
01365
01366
01367
01368 if (ps.diag1 != NULL) {
01369
01370
01371
01372 product_frame_diag1 = cpl_frame_new();
01373 cpl_frame_set_filename(product_frame_diag1,outdiag1);
01374 cpl_frame_set_tag(product_frame_diag1,VIRCAM_PRO_LIN_DIAG1);
01375 cpl_frame_set_type(product_frame_diag1,CPL_FRAME_TYPE_TABLE);
01376 cpl_frame_set_group(product_frame_diag1,CPL_FRAME_GROUP_PRODUCT);
01377 cpl_frame_set_level(product_frame_diag1,CPL_FRAME_LEVEL_FINAL);
01378
01379
01380
01381 plist = cpl_propertylist_duplicate(ps.plist);
01382 vircam_dfs_set_product_primary_header(plist,product_frame_diag1,
01383 framelist,parlist,
01384 (char *)recipeid,
01385 "PRO-1.15",ps.inherit,0);
01386
01387
01388
01389
01390 elist = cpl_propertylist_duplicate(ps.elist);
01391 vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
01392 framelist,parlist,
01393 (char *)recipeid,"PRO-1.15",
01394 ps.inherit);
01395
01396
01397
01398 if (dummy == 1)
01399 vircam_dummy_property(elist);
01400
01401
01402
01403 if (cpl_table_save(ps.diag1,plist,elist,outdiag1,CPL_IO_DEFAULT)
01404 != CPL_ERROR_NONE) {
01405 cpl_msg_error(fctid,"Cannot save product table extension");
01406 freepropertylist(plist);
01407 freepropertylist(elist);
01408 return(-1);
01409 }
01410 cpl_frameset_insert(framelist,product_frame_diag1);
01411 freepropertylist(plist);
01412 freepropertylist(elist);
01413 }
01414
01415
01416
01417 if (ps.diag2 != NULL) {
01418
01419
01420
01421 product_frame_diag2 = cpl_frame_new();
01422 cpl_frame_set_filename(product_frame_diag2,outdiag2);
01423 cpl_frame_set_tag(product_frame_diag2,VIRCAM_PRO_LIN_DIAG2);
01424 cpl_frame_set_type(product_frame_diag2,CPL_FRAME_TYPE_TABLE);
01425 cpl_frame_set_group(product_frame_diag2,CPL_FRAME_GROUP_PRODUCT);
01426 cpl_frame_set_level(product_frame_diag2,CPL_FRAME_LEVEL_FINAL);
01427
01428
01429
01430 plist = cpl_propertylist_duplicate(ps.plist);
01431 vircam_dfs_set_product_primary_header(plist,product_frame_diag2,
01432 framelist,parlist,
01433 (char *)recipeid,
01434 "PRO-1.15",ps.inherit,0);
01435
01436
01437
01438
01439 elist = cpl_propertylist_duplicate(ps.elist);
01440 vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
01441 framelist,parlist,
01442 (char *)recipeid,"PRO-1.15",
01443 ps.inherit);
01444
01445
01446
01447 if (dummy == 1)
01448 vircam_dummy_property(elist);
01449
01450
01451
01452 if (cpl_table_save(ps.diag2,plist,elist,outdiag2,CPL_IO_DEFAULT)
01453 != CPL_ERROR_NONE) {
01454 cpl_msg_error(fctid,"Cannot save product table extension");
01455 freepropertylist(plist);
01456 freepropertylist(elist);
01457 return(-1);
01458 }
01459 cpl_frameset_insert(framelist,product_frame_diag2);
01460 freepropertylist(plist);
01461 freepropertylist(elist);
01462 }
01463
01464
01465
01466 } else {
01467
01468
01469
01470 elist = cpl_propertylist_duplicate(ps.elist);
01471 cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
01472 vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
01473 framelist,parlist,
01474 (char *)recipeid,
01475 "PRO-1.15",ps.inherit);
01476
01477
01478
01479 cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
01480 vircam_linearity_analyse_config.linearity);
01481 cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
01482 "% non-linearity at 10000 ADU");
01483 cpl_propertylist_update_float(elist,"ESO QC LINERROR",
01484 vircam_linearity_analyse_config.linerror);
01485 cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
01486 "% error non-linearity at 10000 ADU");
01487 cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
01488 vircam_linearity_analyse_config.facrng);
01489 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
01490 "total % range in screen variation");
01491 cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
01492 vircam_linearity_analyse_config.maxdiff);
01493 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
01494 "maximum % step in screen variation");
01495 cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
01496
01497
01498
01499 if (dummy == 1) {
01500 vircam_dummy_property(elist);
01501 if (ps.lchantab == NULL)
01502 ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
01503 }
01504
01505
01506
01507 if (cpl_table_save(ps.lchantab,NULL,elist,outtab,CPL_IO_EXTEND)
01508 != CPL_ERROR_NONE) {
01509 cpl_msg_error(fctid,"Cannot save product table extension");
01510 freepropertylist(elist);
01511 return(-1);
01512 }
01513
01514
01515
01516 pafprop = vircam_paf_req_items(elist);
01517 vircam_merge_propertylists(pafprop,ps.phupaf);
01518 vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
01519 vircam_paf_append(pafprop,elist,"ESO PRO CATG");
01520 vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
01521 if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
01522 "QC file",pafprop) != VIR_OK)
01523 cpl_msg_warning(fctid,"Unable to save PAF for BPM");
01524 cpl_propertylist_delete(pafprop);
01525
01526
01527
01528 freepropertylist(elist);
01529
01530
01531
01532 if (ps.diag1 != NULL) {
01533 elist = cpl_propertylist_duplicate(ps.elist);
01534 vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
01535 framelist,parlist,
01536 (char *)recipeid,"PRO-1.15",
01537 ps.inherit);
01538
01539
01540
01541 if (dummy == 1)
01542 vircam_dummy_property(elist);
01543
01544
01545
01546 if (cpl_table_save(ps.diag1,NULL,elist,outdiag1,CPL_IO_EXTEND)
01547 != CPL_ERROR_NONE) {
01548 cpl_msg_error(fctid,"Cannot save product table extension");
01549 freepropertylist(elist);
01550 return(-1);
01551 }
01552 freepropertylist(elist);
01553 }
01554 if (ps.diag2 != NULL) {
01555 elist = cpl_propertylist_duplicate(ps.elist);
01556 vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
01557 framelist,parlist,
01558 (char *)recipeid,"PRO-1.15",
01559 ps.inherit);
01560
01561
01562
01563 if (dummy == 1)
01564 vircam_dummy_property(elist);
01565
01566
01567
01568 if (cpl_table_save(ps.diag2,NULL,elist,outdiag2,CPL_IO_EXTEND)
01569 != CPL_ERROR_NONE) {
01570 cpl_msg_error(fctid,"Cannot save product table extension");
01571 freepropertylist(elist);
01572 return(-1);
01573 }
01574 freepropertylist(elist);
01575 }
01576
01577 }
01578
01579
01580
01581 plist = ps.elist;
01582 nx = ps.nx;
01583 ny = ps.ny;
01584 if (dummy && ps.bpm_array == NULL) {
01585 ps.bpm_array = cpl_array_new((cpl_size)(nx*ny),CPL_TYPE_INT);
01586 bpm = cpl_array_get_data_int(ps.bpm_array);
01587 for (i = 0; i < nx*ny; i++)
01588 bpm[i] = 0;
01589 }
01590 bpm = cpl_array_get_data_int(ps.bpm_array);
01591 vircam_dfs_set_product_exten_header(plist,product_frame_bpm,
01592 framelist,parlist,(char *)recipeid,
01593 "PRO-1.15",ps.inherit);
01594 cpl_propertylist_update_float(plist,"ESO QC BAD_PIXEL_STAT",
01595 vircam_linearity_analyse_config.bad_pixel_stat);
01596 cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_STAT",
01597 "Fraction of pixels that are bad");
01598 cpl_propertylist_update_int(plist,"ESO QC BAD_PIXEL_NUM",
01599 vircam_linearity_analyse_config.bad_pixel_num);
01600 cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_NUM",
01601 "Number of pixels that are bad");
01602 cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps.nuse);
01603 if (dummy)
01604 vircam_dummy_property(plist);
01605 outimg = cpl_image_wrap_int((cpl_size)nx,(cpl_size)ny,bpm);
01606 if (cpl_image_save(outimg,outbpm,CPL_TYPE_UCHAR,plist,
01607 CPL_IO_EXTEND) != CPL_ERROR_NONE) {
01608 cpl_msg_error(fctid,"Cannot save product image extension");
01609 return(-1);
01610 }
01611
01612
01613
01614 pafprop = vircam_paf_req_items(plist);
01615 vircam_merge_propertylists(pafprop,ps.phupaf);
01616 vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
01617 vircam_paf_append(pafprop,ps.plist,"ESO PRO CATG");
01618 vircam_paf_append(pafprop,plist,"ESO PRO DATANCOM");
01619 if (vircam_paf_print((char *)outbpmpaf,"VIRCAM/vircam_linearity_analyse",
01620 "QC file",pafprop) != VIR_OK)
01621 cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
01622 cpl_propertylist_delete(pafprop);
01623
01624
01625
01626 cpl_image_unwrap(outimg);
01627
01628 return(0);
01629 }
01630
01631
01639
01640
01641 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
01642 cpl_parameterlist *parlist) {
01643 int retval;
01644 const char *fctid = "vircam_linearity_analyse_lastbit";
01645
01646
01647
01648 cpl_msg_info(fctid,
01649 "Saving linearity table and bpm for extension %" CPL_SIZE_FORMAT,
01650 (cpl_size)jext);
01651 retval = vircam_linearity_analyse_save(framelist,parlist);
01652 if (retval != 0) {
01653 vircam_linearity_analyse_tidy(2);
01654 return(-1);
01655 }
01656
01657
01658
01659 vircam_linearity_analyse_tidy(1);
01660 return(0);
01661 }
01662
01663
01667
01668
01669 static int vircam_linearity_analyse_domedark_groups(void) {
01670 int i,j,found;
01671 float texp;
01672 cpl_frame *frame;
01673 cpl_propertylist *plist;
01674 const char *fctid = "vircam_linearity_analyse_domedark_groups";
01675
01676
01677
01678 ps.ddg = cpl_calloc(ps.ndomes,sizeof(ddgrp));
01679 ps.nddg = 0;
01680
01681
01682
01683
01684 for (i = 0; i < ps.ndomes; i++) {
01685 frame = cpl_frameset_get_frame(ps.domelist,i);
01686 plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
01687 if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
01688 cpl_msg_warning(fctid,"No exposure time found in %s",
01689 cpl_frame_get_filename(frame));
01690 cpl_propertylist_delete(plist);
01691 continue;
01692 }
01693 cpl_propertylist_delete(plist);
01694
01695
01696
01697
01698
01699 found = 0;
01700 for (j = 0; j < ps.nddg; j++) {
01701 if (ps.ddg[j].exptime == texp) {
01702 found = 1;
01703 break;
01704 }
01705 }
01706 if (found) {
01707 cpl_frameset_insert(ps.ddg[j].domes,cpl_frame_duplicate(frame));
01708 ps.ddg[j].ndomes += 1;
01709 } else {
01710 ps.ddg[ps.nddg].exptime = texp;
01711 ps.ddg[ps.nddg].darks = cpl_frameset_new();
01712 ps.ddg[ps.nddg].domes = cpl_frameset_new();
01713 ps.ddg[ps.nddg].ndarks = 0;
01714 ps.ddg[ps.nddg].ndomes = 1;
01715 ps.ddg[ps.nddg].flag = OK_FLAG;
01716 cpl_frameset_insert(ps.ddg[ps.nddg].domes,
01717 cpl_frame_duplicate(frame));
01718 ps.nddg += 1;
01719 }
01720 }
01721
01722
01723
01724 for (i = 0; i < ps.ndarks; i++) {
01725 frame = cpl_frameset_get_frame(ps.darklist,i);
01726 plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
01727 if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
01728 cpl_msg_warning(fctid,"No exposure time found in %s",
01729 cpl_frame_get_filename(frame));
01730 cpl_propertylist_delete(plist);
01731 continue;
01732 }
01733 cpl_propertylist_delete(plist);
01734
01735
01736
01737
01738
01739 found = 0;
01740 for (j = 0; j < ps.nddg; j++) {
01741 if (ps.ddg[j].exptime == texp) {
01742 found = 1;
01743 break;
01744 }
01745 }
01746 if (found) {
01747 cpl_frameset_insert(ps.ddg[j].darks,cpl_frame_duplicate(frame));
01748 ps.ddg[j].ndarks += 1;
01749 }
01750 }
01751
01752
01753
01754
01755 i = 0;
01756 while (i < ps.nddg) {
01757 if (ps.ddg[i].ndarks == 0) {
01758 cpl_msg_warning(fctid,
01759 "No dark frames exist for exposure %g\nThrowing these away",
01760 ps.ddg[i].exptime);
01761 freeframeset(ps.ddg[i].darks);
01762 freeframeset(ps.ddg[i].domes);
01763 for (j = i+1; j < ps.nddg; j++)
01764 ps.ddg[j-1] = ps.ddg[j];
01765 ps.nddg -= 1;
01766 } else
01767 i++;
01768 }
01769
01770
01771
01772 for (i = 0; i < ps.nddg; i++) {
01773 ps.ddg[i].proc = cpl_malloc(ps.ddg[i].ndomes*sizeof(vir_fits *));
01774 for (j = 0; j < ps.ddg[i].ndomes; j++)
01775 ps.ddg[i].proc[j] = NULL;
01776 }
01777
01778
01779
01780
01781 if (ps.nddg > 0) {
01782 ps.ddg = cpl_realloc(ps.ddg,ps.nddg*sizeof(ddgrp));
01783 return(0);
01784 } else {
01785 cpl_msg_error(fctid,"There are no darks defined for input domes");
01786 return(-1);
01787 }
01788 }
01789
01790
01799
01800
01801 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
01802 parquet *p, int np) {
01803 int i,ist,ifn,jst,jfn,n,jind2,iind2,jj,nx,ii;
01804 parquet *pp;
01805 double *d;
01806 float *tmp,*data;
01807
01808
01809
01810 d = cpl_malloc(np*sizeof(double));
01811
01812
01813
01814 nx = (int)cpl_image_get_size_x(vircam_fits_get_image(fframe));
01815 data = cpl_image_get_data_float(vircam_fits_get_image(fframe));
01816
01817
01818
01819 tmp = cpl_malloc(SUBSET*SUBSET*sizeof(float));
01820
01821
01822
01823 for (i = 0; i < np; i++) {
01824 pp = p + i;
01825
01826
01827
01828 ist = ((pp->delta_i)/2 - SUBSET2);
01829 ifn = ist + SUBSET - 1;
01830 jst = ((pp->delta_j)/2 - SUBSET2);
01831 jfn = jst + SUBSET - 1;
01832
01833
01834
01835 n = 0;
01836 for (jj = jst; jj <= jfn; jj++) {
01837 jind2 = (jj + pp->iymin - 1)*nx;
01838 for (ii = ist; ii <= ifn; ii++) {
01839 iind2 = jind2 + ii + pp->ixmin - 1;
01840 if (bpm[iind2] == 0)
01841 tmp[n++] = data[iind2];
01842 }
01843 }
01844 d[i] = (double)vircam_med(tmp,NULL,(long)n);
01845 }
01846
01847
01848
01849 freespace(tmp);
01850 return(d);
01851 }
01852
01853
01865
01866
01867 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
01868 int nchan, double *facrng,
01869 double *maxdiff) {
01870 int i,ist,ifn,j;
01871 double *factors,sum,midval,minfac,maxfac;
01872
01873
01874
01875 factors = cpl_malloc(nim*sizeof(double));
01876
01877
01878
01879 vircam_mjdsort(fdata,mjd,nim);
01880
01881
01882
01883 if (nim % 2 == 0) {
01884 ist = nim/2 - 1;
01885 ifn = ist + 1;
01886 } else {
01887 ist = nim/2;
01888 ifn = ist;
01889 }
01890
01891
01892
01893 for (i = 0; i < nchan; i++) {
01894
01895
01896
01897 midval = 0.5*(fdata[ist][i] + fdata[ifn][i]);
01898
01899
01900
01901 for (j = 0; j < nim; j++)
01902 fdata[j][i] /= midval;
01903 }
01904
01905
01906
01907
01908 *maxdiff = 0.0;
01909 maxfac = 0.0;
01910 minfac = 0.0;
01911 for (j = 0; j < nim; j++) {
01912 sum = 0.0;
01913 for (i = 0; i < nchan; i++)
01914 sum += fdata[j][i];
01915 factors[j] = sum/(double)nchan;
01916 if (j == 0) {
01917 maxfac = factors[j];
01918 minfac = factors[j];
01919 } else {
01920 minfac = min(minfac,factors[j]);
01921 maxfac = max(maxfac,factors[j]);
01922 *maxdiff = max(*maxdiff,fabs(factors[j]-factors[j-1]));
01923 }
01924 }
01925 *facrng = maxfac - minfac;
01926
01927
01928
01929 return(factors);
01930 }
01931
01932
01940
01941
01942 static void vircam_mjdsort(double **fdata, double *mjd, int n) {
01943 int iii,ii,i,ifin,j;
01944 double tmpmjd,*tmpdata;
01945
01946
01947 iii = 2;
01948 while (iii < n)
01949 iii *= 2;
01950 iii = min(n,(3*iii)/4 - 1);
01951
01952 while (iii > 1) {
01953 iii /= 2;
01954 ifin = n - iii;
01955 for (ii = 0; ii < ifin; ii++) {
01956 i = ii;
01957 j = i + iii;
01958 if (mjd[i] > mjd[j]) {
01959 tmpmjd = mjd[j];
01960 tmpdata = fdata[j];
01961 while (1) {
01962 mjd[j] = mjd[i];
01963 fdata[j] = fdata[i];
01964 j = i;
01965 i = i - iii;
01966 if (i < 0 || mjd[0] <= tmpmjd)
01967 break;
01968 }
01969 mjd[j] = tmpmjd;
01970 fdata[j] = tmpdata;
01971 }
01972 }
01973 }
01974 }
01975
01976
01983
01984
01985 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows) {
01986 int i;
01987 char colname[16];
01988 cpl_table *t;
01989
01990
01991
01992 t = cpl_table_new(nrows);
01993
01994
01995
01996 cpl_table_new_column(t,"filename",CPL_TYPE_STRING);
01997 cpl_table_new_column(t,"exptime",CPL_TYPE_DOUBLE);
01998 cpl_table_set_column_unit(t,"exptime","seconds");
01999 cpl_table_new_column(t,"mjd",CPL_TYPE_DOUBLE);
02000 cpl_table_set_column_unit(t,"mjd","days");
02001
02002
02003
02004
02005 for (i = 1; i <= np; i++) {
02006 (void)snprintf(colname,16,"rawflux_%02d",i);
02007 cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
02008 cpl_table_set_column_unit(t,colname,"ADU");
02009 (void)snprintf(colname,16,"linflux_%02d",i);
02010 cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
02011 cpl_table_set_column_unit(t,colname,"ADU");
02012 }
02013
02014
02015
02016 cpl_table_new_column(t,"adjust_fac",CPL_TYPE_DOUBLE);
02017
02018
02019
02020 return(t);
02021 }
02022
02023
02027
02028
02029 static void vircam_linearity_analyse_init(void) {
02030 ps.labels = NULL;
02031 ps.domelist = NULL;
02032 ps.darklist = NULL;
02033 ps.domecheck = NULL;
02034 ps.darkcheck = NULL;
02035 ps.ndomes = 0;
02036 ps.ndarks = 0;
02037 ps.ndomecheck = 0;
02038 ps.ndarkcheck = 0;
02039 ps.chanfrm = NULL;
02040 ps.chantab = NULL;
02041 ps.lchantab = NULL;
02042 ps.flatlist = NULL;
02043 ps.bpm_array = NULL;
02044 ps.ddg = NULL;
02045 ps.plist = NULL;
02046 ps.elist = NULL;
02047 ps.phupaf = NULL;
02048 ps.diag1 = NULL;
02049 ps.diag2 = NULL;
02050 ps.inherit = NULL;
02051 }
02052
02053
02057
02058
02059 static void vircam_linearity_analyse_tidy(int level) {
02060 int i;
02061
02062 freetfits(ps.chantab);
02063 freearray(ps.bpm_array);
02064 freefitslist(ps.flatlist,ps.nflatlist);
02065 freetable(ps.lchantab);
02066 freepropertylist(ps.plist);
02067 freepropertylist(ps.elist);
02068 freetable(ps.diag1);
02069 freetable(ps.diag2);
02070 if (level == 1)
02071 return;
02072
02073 freespace(ps.labels);
02074 freeframeset(ps.domelist);
02075 freeframeset(ps.darklist);
02076 freeframeset(ps.domecheck);
02077 freeframeset(ps.darkcheck);
02078 freeframe(ps.chanfrm);
02079 if (ps.ddg != NULL) {
02080 for (i = 0; i < ps.nddg; i++) {
02081 freeframeset(ps.ddg[i].darks);
02082 freeframeset(ps.ddg[i].domes);
02083 freefitslist(ps.ddg[i].proc,ps.ddg[i].ndomes);
02084 }
02085 freespace(ps.ddg);
02086 }
02087 freepropertylist(ps.phupaf);
02088 }
02089
02092
02093
02094
02095
02096
02097
02098
02099
02100
02101
02102
02103
02104
02105
02106
02107
02108
02109
02110
02111
02112
02113
02114
02115
02116
02117
02118
02119
02120
02121
02122
02123
02124
02125
02126
02127
02128
02129
02130
02131
02132
02133
02134
02135
02136
02137
02138
02139
02140
02141
02142
02143
02144
02145
02146
02147
02148
02149
02150
02151
02152
02153
02154
02155
02156
02157
02158
02159
02160
02161
02162
02163
02164
02165
02166
02167
02168
02169
02170
02171
02172
02173
02174
02175
02176
02177
02178
02179
02180
02181
02182
02183
02184
02185
02186
02187
02188
02189
02190
02191
02192
02193
02194
02195
02196
02197
02198
02199
02200
02201
02202
02203
02204
02205
02206
02207
02208
02209
02210
02211
02212
02213
02214
02215
02216
02217
02218
02219
02220
02221
02222
02223
02224
02225
02226
02227
02228
02229
02230
02231
02232
02233
02234
02235
02236
02237
02238
02239
02240
02241
02242
02243
02244
02245
02246
02247
02248
02249
02250
02251
02252
02253
02254
02255
02256
02257
02258
02259
02260
02261
02262
02263
02264
02265
02266
02267
02268
02269
02270
02271
02272
02273
02274
02275
02276
02277
02278
02279
02280
02281
02282
02283
02284
02285
02286
02287
02288
02289
02290
02291
02292
02293
02294
02295
02296
02297
02298
02299
02300
02301
02302
02303
02304
02305
02306
02307
02308