VIRCAM Pipeline  1.3.4
vircam_reset_combine.c
1 /* $Id: vircam_reset_combine.c,v 1.60 2012-01-16 12:32:18 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2012-01-16 12:32:18 $
24  * $Revision: 1.60 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <stdio.h>
35 #include <cpl.h>
36 #include <math.h>
37 
38 #include "vircam_utils.h"
39 #include "vircam_mask.h"
40 #include "vircam_dfs.h"
41 #include "vircam_mods.h"
42 #include "vircam_stats.h"
43 #include "vircam_fits.h"
44 #include "vircam_pfits.h"
45 #include "vircam_channel.h"
46 #include "vircam_paf.h"
47 #include "vircam_wcsutils.h"
48 
49 /* Define values for the bit mask that flags dummy results */
50 
51 #define MEANRESET 1
52 #define DIFFIMG 2
53 #define STATS_TAB 4
54 
55 /* Function prototypes */
56 
57 static int vircam_reset_combine_create(cpl_plugin *) ;
58 static int vircam_reset_combine_exec(cpl_plugin *) ;
59 static int vircam_reset_combine_destroy(cpl_plugin *) ;
60 static int vircam_reset_combine(cpl_parameterlist *, cpl_frameset *) ;
61 static int vircam_reset_combine_save(cpl_frameset *framelist,
62  cpl_parameterlist *parlist);
63 static void vircam_reset_combine_dummy_products(void);
64 static void vircam_reset_combine_normal(int jext);
65 static int vircam_reset_combine_lastbit(int jext, cpl_frameset *framelist,
66  cpl_parameterlist *parlist);
67 static void vircam_reset_combine_init(void);
68 static void vircam_reset_combine_tidy(int level);
69 
70 /* Static global variables */
71 
72 static struct {
73 
74  /* Input */
75 
76  int combtype;
77  int scaletype;
78  int xrej;
79  float thresh;
80  int ncells;
81  int extenum;
82 
83  /* Output */
84 
85  float resetmed;
86  float resetrms;
87  float resetdiff_med;
88  float resetdiff_rms;
89 
90 } vircam_reset_combine_config ;
91 
92 static struct {
93  cpl_size *labels;
94  cpl_frameset *resetlist;
95  vir_fits **resets;
96  int nresets;
97  vir_fits **good;
98  int ngood;
99  cpl_frame *master_reset;
100  vir_mask *master_mask;
101  cpl_frame *chantab;
102  cpl_image *outimage;
103  cpl_propertylist *drs;
104  unsigned char *rejmask;
105  unsigned char *rejplus;
106  vir_fits *mrimage;
107  cpl_image *diffimg;
108  cpl_table *diffimstats;
109  cpl_propertylist *phupaf;
110 } ps;
111 
112 static int isfirst;
113 static cpl_frame *product_frame_mean_reset = NULL;
114 static cpl_frame *product_frame_diffimg = NULL;
115 static cpl_frame *product_frame_diffimg_stats = NULL;
116 static int we_expect;
117 static int we_get;
118 
119 static char vircam_reset_combine_description[] =
120 "vircam_reset_combine -- VIRCAM reset combine recipe.\n\n"
121 "Combine a list of reset frames into a mean reset frame. Optionally compare \n"
122 "the output frame to a master reset frame\n\n"
123 "The program requires the following files in the SOF:\n\n"
124 " Tag Description\n"
125 " -----------------------------------------------------------------------\n"
126 " %-21s A list of raw reset images\n"
127 " %-21s Optional reference reset frame\n"
128 " %-21s Optional master bad pixel map or\n"
129 " %-21s Optional master confidence map\n"
130 " %-21s Optional channel table or\n"
131 " %-21s Optional initial channel table\n"
132 "\n"
133 "If no master reset frame is made available, then no comparison will be done\n"
134 "This means there will be no output difference image. If a master reset is\n"
135 "available, but no channel table is, then a difference image will be formed\n"
136 "but no stats will be written."
137 "\n";
138 
253 /* Function code */
254 
255 /*---------------------------------------------------------------------------*/
263 /*---------------------------------------------------------------------------*/
264 
265 int cpl_plugin_get_info(cpl_pluginlist *list) {
266  cpl_recipe *recipe = cpl_calloc(1,sizeof(*recipe));
267  cpl_plugin *plugin = &recipe->interface;
268  char alldesc[SZ_ALLDESC];
269  (void)snprintf(alldesc,SZ_ALLDESC,vircam_reset_combine_description,
270  VIRCAM_RESET_RAW,VIRCAM_REF_RESET,VIRCAM_CAL_BPM,
271  VIRCAM_CAL_CONF,VIRCAM_CAL_CHANTAB,VIRCAM_CAL_CHANTAB_INIT);
272 
273  cpl_plugin_init(plugin,
274  CPL_PLUGIN_API,
275  VIRCAM_BINARY_VERSION,
276  CPL_PLUGIN_TYPE_RECIPE,
277  "vircam_reset_combine",
278  "VIRCAM reset combination recipe",
279  alldesc,
280  "Jim Lewis",
281  "jrl@ast.cam.ac.uk",
283  vircam_reset_combine_create,
284  vircam_reset_combine_exec,
285  vircam_reset_combine_destroy);
286 
287  cpl_pluginlist_append(list,plugin);
288 
289  return(0);
290 }
291 
292 /*---------------------------------------------------------------------------*/
301 /*---------------------------------------------------------------------------*/
302 
303 static int vircam_reset_combine_create(cpl_plugin *plugin) {
304  cpl_recipe *recipe;
305  cpl_parameter *p;
306 
307  /* Get the recipe out of the plugin */
308 
309  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
310  recipe = (cpl_recipe *)plugin;
311  else
312  return(-1);
313 
314  /* Create the parameters list in the cpl_recipe object */
315 
316  recipe->parameters = cpl_parameterlist_new();
317 
318  /* Fill in the parameters. First the combination type */
319 
320  p = cpl_parameter_new_range("vircam.vircam_reset_combine.combtype",
321  CPL_TYPE_INT,
322  "1 == Median,\n 2 == Mean",
323  "vircam.vircam_reset_combine",
324  1,1,2);
325  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"combtype");
326  cpl_parameterlist_append(recipe->parameters,p);
327 
328  /* The requested scaling */
329 
330  p = cpl_parameter_new_range("vircam.vircam_reset_combine.scaletype",
331  CPL_TYPE_INT,
332  "0 == none,\n 1 == additive offset,\n 2 == multiplicative offset,\n 3 == exposure time scaling + additive offset",
333  "vircam.vircam_reset_combine",
334  1,0,3);
335  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"scaletype");
336  cpl_parameterlist_append(recipe->parameters,p);
337 
338  /* Extra rejection cycle */
339 
340  p = cpl_parameter_new_value("vircam.vircam_reset_combine.xrej",
341  CPL_TYPE_BOOL,
342  "True if using extra rejection cycle",
343  "vircam.vircam_reset_combine",
344  TRUE);
345  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"xrej");
346  cpl_parameterlist_append(recipe->parameters,p);
347 
348  /* Rejection threshold */
349 
350  p = cpl_parameter_new_value("vircam.vircam_reset_combine.thresh",
351  CPL_TYPE_DOUBLE,
352  "Rejection threshold in sigma above background",
353  "vircam.vircam_reset_combine",5.0);
354  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"thresh");
355  cpl_parameterlist_append(recipe->parameters,p);
356 
357  /* How many cells to divide each data channel */
358 
359  p = cpl_parameter_new_enum("vircam.vircam_reset_combine.ncells",
360  CPL_TYPE_INT,
361  "Number of cells for data channel stats",
362  "vircam.vircam_reset_combine",8,7,1,2,4,8,
363  16,32,64);
364  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ncells");
365  cpl_parameterlist_append(recipe->parameters,p);
366 
367  /* Extension number of input frames to use */
368 
369  p = cpl_parameter_new_range("vircam.vircam_reset_combine.extenum",
370  CPL_TYPE_INT,
371  "Extension number to be done, 0 == all",
372  "vircam.vircam_reset_combine",
373  1,0,16);
374  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
375  cpl_parameterlist_append(recipe->parameters,p);
376 
377  /* Get out of here */
378 
379  return(0);
380 }
381 
382 
383 /*---------------------------------------------------------------------------*/
389 /*---------------------------------------------------------------------------*/
390 
391 static int vircam_reset_combine_exec(cpl_plugin *plugin) {
392  cpl_recipe *recipe;
393 
394  /* Get the recipe out of the plugin */
395 
396  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
397  recipe = (cpl_recipe *)plugin;
398  else
399  return(-1);
400 
401  return(vircam_reset_combine(recipe->parameters,recipe->frames));
402 }
403 
404 /*---------------------------------------------------------------------------*/
410 /*---------------------------------------------------------------------------*/
411 
412 static int vircam_reset_combine_destroy(cpl_plugin *plugin) {
413  cpl_recipe *recipe ;
414 
415  /* Get the recipe out of the plugin */
416 
417  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
418  recipe = (cpl_recipe *)plugin;
419  else
420  return(-1);
421 
422  cpl_parameterlist_delete(recipe->parameters);
423  return(0);
424 }
425 
426 /*---------------------------------------------------------------------------*/
433 /*---------------------------------------------------------------------------*/
434 
435 static int vircam_reset_combine(cpl_parameterlist *parlist,
436  cpl_frameset *framelist) {
437  const char *fctid="vircam_reset_combine";
438  int j,jst,jfn,retval,status,i,live,nx,ny;
439  cpl_size nlab;
440  cpl_parameter *p;
441  vir_fits *ff;
442 
443  /* Check validity of input frameset */
444 
445  if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
446  cpl_msg_error(fctid,"Input framelist NULL or has no input data");
447  return(-1);
448  }
449 
450  /* Check the files in the frameset */
451 
452  if (vircam_frameset_fexists(framelist) != VIR_OK) {
453  cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
454  return(-1);
455  }
456 
457  /* Initialise a few things */
458 
459  vircam_reset_combine_init();
460  we_expect |= MEANRESET;
461 
462  /* Get the parameters */
463 
464  p = cpl_parameterlist_find(parlist,"vircam.vircam_reset_combine.combtype");
465  vircam_reset_combine_config.combtype = cpl_parameter_get_int(p);
466  p = cpl_parameterlist_find(parlist,"vircam.vircam_reset_combine.scaletype");
467  vircam_reset_combine_config.scaletype = cpl_parameter_get_int(p);
468  p = cpl_parameterlist_find(parlist,"vircam.vircam_reset_combine.xrej");
469  vircam_reset_combine_config.xrej = cpl_parameter_get_bool(p);
470  p = cpl_parameterlist_find(parlist,"vircam.vircam_reset_combine.thresh");
471  vircam_reset_combine_config.thresh = (float)cpl_parameter_get_double(p);
472  p = cpl_parameterlist_find(parlist,"vircam.vircam_reset_combine.ncells");
473  vircam_reset_combine_config.ncells = cpl_parameter_get_int(p);
474  p = cpl_parameterlist_find(parlist,"vircam.vircam_reset_combine.extenum");
475  vircam_reset_combine_config.extenum = cpl_parameter_get_int(p);
476 
477  /* Sort out raw from calib frames */
478 
479  if (vircam_dfs_set_groups(framelist) != VIR_OK) {
480  cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
481  return(-1);
482  }
483 
484  /* Get the reset frames */
485 
486  if ((ps.labels = cpl_frameset_labelise(framelist,vircam_compare_tags,
487  &nlab)) == NULL) {
488  cpl_msg_error(fctid,"Cannot labelise the input frames");
489  return(-1);
490  }
491  if ((ps.resetlist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
492  VIRCAM_RESET_RAW)) == NULL) {
493  cpl_msg_error(fctid,"Cannot find reset frames in input frameset");
494  return(-1);
495  }
496  ps.nresets = cpl_frameset_get_size(ps.resetlist);
497 
498  /* Check to see if there is a master reset frame */
499 
500  if ((ps.master_reset = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
501  VIRCAM_REF_RESET)) == NULL)
502  cpl_msg_info(fctid,"No master reset found -- no difference image will be formed");
503  else
504  we_expect |= DIFFIMG;
505 
506  /* Check to see if there is a master bad pixel map. If there isn't one
507  then look for a confidence map */
508 
509  ps.master_mask = vircam_mask_define(framelist,ps.labels,nlab);
510 
511  /* Check to see if there is a channel table */
512 
513  if ((ps.chantab = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
514  VIRCAM_CAL_CHANTAB)) == NULL) {
515  if ((ps.chantab = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
516  VIRCAM_CAL_CHANTAB_INIT)) == NULL)
517  cpl_msg_info(fctid,"No channel table found -- no difference image stats will be done");
518  } else if (we_expect & DIFFIMG)
519  we_expect |= STATS_TAB;
520 
521  /* Now, how many image extensions do we want to do? If the extension
522  number is zero, then we loop for all possible extensions. If it
523  isn't then we just do the extension specified */
524 
525  vircam_exten_range(vircam_reset_combine_config.extenum,
526  (const cpl_frame *)cpl_frameset_get_frame(ps.resetlist,0),
527  &jst,&jfn);
528  if (jst == -1 || jfn == -1) {
529  cpl_msg_error(fctid,"Unable to continue");
530  vircam_reset_combine_tidy(2);
531  return(-1);
532  }
533 
534  /* Get some space for the good frames */
535 
536  ps.good = cpl_malloc(ps.nresets*sizeof(vir_fits *));
537 
538  /* Now loop for all the extension... */
539 
540  for (j = jst; j <= jfn; j++) {
541  status = VIR_OK;
542  we_get = 0;
543  isfirst = (j == jst);
544 
545  /* Load up the images. If they won't load the signal a major error */
546 
547  ps.resets = vircam_fits_load_list(ps.resetlist,CPL_TYPE_FLOAT,j);
548  if (ps.resets == NULL) {
549  cpl_msg_info(fctid,
550  "Extension %" CPL_SIZE_FORMAT " resets wouldn't load",
551  (cpl_size)j);
552  retval = vircam_reset_combine_lastbit(j,framelist,parlist);
553  if (retval != 0)
554  return(-1);
555  continue;
556  }
557 
558  /* Are any of these reset images good? */
559 
560  ps.ngood = 0;
561  for (i = 0; i < ps.nresets; i++) {
562  ff = ps.resets[i];
564  if (! live) {
565  cpl_msg_info(fctid,"Detector flagged dead %s",
567  vircam_fits_set_error(ff,VIR_FATAL);
568  } else {
569  ps.good[ps.ngood] = ff;
570  ps.ngood += 1;
571  }
572  }
573 
574  /* If there are no good images, then signal that wee need to
575  create some dummy products and move on */
576 
577  if (ps.ngood == 0) {
578  cpl_msg_info(fctid,"All images flagged bad for this extension");
579  retval = vircam_reset_combine_lastbit(j,framelist,parlist);
580  if (retval != 0)
581  return(-1);
582  continue;
583  }
584 
585  /* Load the mask */
586 
587  nx = (int)cpl_image_get_size_x(vircam_fits_get_image(ps.good[0]));
588  ny = (int)cpl_image_get_size_y(vircam_fits_get_image(ps.good[0]));
589  if (vircam_mask_load(ps.master_mask,j,nx,ny) == VIR_FATAL) {
590  cpl_msg_info(fctid,
591  "Unable to load mask image %s[%" CPL_SIZE_FORMAT "]",
592  vircam_mask_get_filename(ps.master_mask),
593  (cpl_size)j);
594  cpl_msg_info(fctid,"Forcing all pixels to be good from now on");
595  vircam_mask_force(ps.master_mask,nx,ny);
596  }
597 
598  /* Call the combine module. If it fails then signal that
599  all products will be dummies */
600 
601  cpl_msg_info(fctid,"Doing combination for extension %" CPL_SIZE_FORMAT,
602  (cpl_size)j);
603  (void)vircam_imcombine(ps.good,ps.ngood,
604  vircam_reset_combine_config.combtype,
605  vircam_reset_combine_config.scaletype,
606  vircam_reset_combine_config.xrej,
607  vircam_reset_combine_config.thresh,
608  &(ps.outimage),&(ps.rejmask),
609  &(ps.rejplus),&(ps.drs),&status);
610  if (status == VIR_OK) {
611  we_get |= MEANRESET;
612  vircam_reset_combine_normal(j);
613  }
614 
615  /* Create any dummies and save products */
616 
617  retval = vircam_reset_combine_lastbit(j,framelist,parlist);
618  if (retval != 0)
619  return(-1);
620  }
621  vircam_reset_combine_tidy(2);
622  return(0);
623 }
624 
625 /*---------------------------------------------------------------------------*/
632 /*---------------------------------------------------------------------------*/
633 
634 static int vircam_reset_combine_save(cpl_frameset *framelist,
635  cpl_parameterlist *parlist) {
636  cpl_propertylist *plist,*elist,*p,*pafprop;
637  int status;
638  const char *fctid = "vircam_reset_combine_save";
639  const char *outfile = "resetcomb.fits";
640  const char *outdiff = "resetdiff.fits";
641  const char *outdimst = "resetdifftab.fits";
642  const char *outfilepaf = "resetcomb";
643  const char *outdiffpaf = "resetdiff";
644  const char *recipeid = "vircam_reset_combine";
645 
646  /* If we need to make a PHU then do that now based on the first frame
647  in the input frame list */
648 
649  if (isfirst) {
650 
651  /* Create a new product frame object and define some tags */
652 
653  product_frame_mean_reset = cpl_frame_new();
654  cpl_frame_set_filename(product_frame_mean_reset,outfile);
655  cpl_frame_set_tag(product_frame_mean_reset,VIRCAM_PRO_RESET);
656  cpl_frame_set_type(product_frame_mean_reset,CPL_FRAME_TYPE_IMAGE);
657  cpl_frame_set_group(product_frame_mean_reset,CPL_FRAME_GROUP_PRODUCT);
658  cpl_frame_set_level(product_frame_mean_reset,CPL_FRAME_LEVEL_FINAL);
659 
660  /* Set up product phu */
661 
662  plist = vircam_fits_get_phu(ps.resets[0]);
663  ps.phupaf = vircam_paf_phu_items(plist);
664  if (ps.master_reset != NULL) {
665  cpl_propertylist_update_string(ps.phupaf,"REF_RESET",
666  cpl_frame_get_filename(ps.master_reset));
667  cpl_propertylist_set_comment(ps.phupaf,"REF_RESET",
668  "Reference reset used");
669  }
670  vircam_dfs_set_product_primary_header(plist,product_frame_mean_reset,
671  framelist,parlist,
672  (char *)recipeid,
673  "PRO-1.15",NULL,0);
674 
675  /* 'Save' the PHU image */
676 
677  if (cpl_image_save(NULL,outfile,CPL_TYPE_UCHAR,plist,
678  CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
679  cpl_msg_error(fctid,"Cannot save product PHU");
680  cpl_frame_delete(product_frame_mean_reset);
681  return(-1);
682  }
683  cpl_frameset_insert(framelist,product_frame_mean_reset);
684 
685  /* Create a new product frame object for the difference image */
686 
687  if (we_expect & DIFFIMG) {
688  product_frame_diffimg = cpl_frame_new();
689  cpl_frame_set_filename(product_frame_diffimg,outdiff);
690  cpl_frame_set_tag(product_frame_diffimg,
691  VIRCAM_PRO_DIFFIMG_RESET);
692  cpl_frame_set_type(product_frame_diffimg,CPL_FRAME_TYPE_IMAGE);
693  cpl_frame_set_group(product_frame_diffimg,CPL_FRAME_GROUP_PRODUCT);
694  cpl_frame_set_level(product_frame_diffimg,CPL_FRAME_LEVEL_FINAL);
695 
696  /* Set up product phu */
697 
698  plist = vircam_fits_get_phu(ps.resets[0]);
699  vircam_dfs_set_product_primary_header(plist,product_frame_diffimg,
700  framelist,parlist,
701  (char *)recipeid,
702  "PRO-1.15",NULL,0);
703  /* 'Save' the PHU image */
704 
705  if (cpl_image_save(NULL,outdiff,CPL_TYPE_UCHAR,plist,
706  CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
707  cpl_msg_error(fctid,"Cannot save product PHU");
708  cpl_frame_delete(product_frame_diffimg);
709  return(-1);
710  }
711  cpl_frameset_insert(framelist,product_frame_diffimg);
712  }
713 
714  /* Create a new product frame object for the difference image stats
715  table */
716 
717  if (we_expect & STATS_TAB) {
718  product_frame_diffimg_stats = cpl_frame_new();
719  cpl_frame_set_filename(product_frame_diffimg_stats,outdimst);
720  cpl_frame_set_tag(product_frame_diffimg_stats,
721  VIRCAM_PRO_DIFFIMG_RESET_STATS);
722  cpl_frame_set_type(product_frame_diffimg_stats,
723  CPL_FRAME_TYPE_TABLE);
724  cpl_frame_set_group(product_frame_diffimg_stats,
725  CPL_FRAME_GROUP_PRODUCT);
726  cpl_frame_set_level(product_frame_diffimg_stats,
727  CPL_FRAME_LEVEL_FINAL);
728 
729  /* Set up product phu */
730 
731  plist = vircam_fits_get_phu(ps.resets[0]);
733  product_frame_diffimg_stats,
734  framelist,parlist,
735  (char *)recipeid,
736  "PRO-1.15",NULL,0);
737 
738  /* Fiddle with the extension header */
739 
740  elist = vircam_fits_get_ehu(ps.resets[0]);
741  p = cpl_propertylist_duplicate(elist);
742  vircam_merge_propertylists(p,ps.drs);
743  if (! (we_get & STATS_TAB))
745  vircam_dfs_set_product_exten_header(p,product_frame_diffimg_stats,
746  framelist,parlist,
747  (char *)recipeid,
748  "PRO-1.15",NULL);
749  status = VIR_OK;
750  vircam_removewcs(p,&status);
751  if (cpl_table_save(ps.diffimstats,plist,p,outdimst,
752  CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
753  cpl_msg_error(fctid,"Cannot save product table extension");
754  cpl_propertylist_delete(p);
755  return(-1);
756  }
757  cpl_propertylist_delete(p);
758  cpl_frameset_insert(framelist,product_frame_diffimg_stats);
759  }
760  }
761 
762  /* Get the extension property list */
763 
764  plist = vircam_fits_get_ehu(ps.resets[0]);
765  cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps.ngood);
766 
767  /* Fiddle with the header now */
768 
769  vircam_merge_propertylists(plist,ps.drs);
770  p = cpl_propertylist_duplicate(plist);
771  if (! (we_get & MEANRESET))
773  vircam_dfs_set_product_exten_header(p,product_frame_mean_reset,framelist,
774  parlist,(char *)recipeid,"PRO-1.15",
775  NULL);
776 
777  /* Now save the reset image extension */
778 
779  cpl_propertylist_update_float(p,"ESO QC RESETMED",
780  vircam_reset_combine_config.resetmed);
781  cpl_propertylist_set_comment(p,"ESO QC RESETMED",
782  "Median of mean reset frame");
783  cpl_propertylist_update_float(p,"ESO QC RESETRMS",
784  vircam_reset_combine_config.resetrms);
785  cpl_propertylist_set_comment(p,"ESO QC RESETRMS",
786  "RMS of mean reset frame");
787  if (cpl_image_save(ps.outimage,outfile,CPL_TYPE_FLOAT,p,
788  CPL_IO_EXTEND) != CPL_ERROR_NONE) {
789  cpl_msg_error(fctid,"Cannot save product image extension");
790  cpl_propertylist_delete(p);
791  return(-1);
792  }
793 
794  /* Write out PAF for mean image */
795 
796  pafprop = vircam_paf_req_items(p);
797  vircam_merge_propertylists(pafprop,ps.phupaf);
798  vircam_paf_append(pafprop,p,"ESO PRO CATG");
799  vircam_paf_append(pafprop,p,"ESO PRO DATANCOM");
800  if (vircam_paf_print((char *)outfilepaf,"VIRCAM/vircam_reset_combine",
801  "QC file",pafprop) != VIR_OK)
802  cpl_msg_warning(fctid,"Unable to save PAF for mean reset");
803  cpl_propertylist_delete(pafprop);
804  cpl_propertylist_delete(p);
805 
806  /* Now save the reset difference image extension */
807 
808  if (we_expect & DIFFIMG) {
809  p = cpl_propertylist_duplicate(plist);
810  if (! (we_get & DIFFIMG))
812  cpl_propertylist_update_float(p,"ESO QC RESETDIFF_MED",
813  vircam_reset_combine_config.resetdiff_med);
814  cpl_propertylist_set_comment(p,"ESO QC RESETDIFF_MED",
815  "Median value of difference image");
816  cpl_propertylist_update_float(p,"ESO QC RESETDIFF_RMS",
817  vircam_reset_combine_config.resetdiff_rms);
818  cpl_propertylist_set_comment(p,"ESO QC RESETDIFF_RMS",
819  "RMS value of difference image");
820  vircam_dfs_set_product_exten_header(p,product_frame_diffimg,
821  framelist,parlist,(char *)recipeid,
822  "PRO-1.15",NULL);
823  if (cpl_image_save(ps.diffimg,outdiff,CPL_TYPE_FLOAT,p,
824  CPL_IO_EXTEND) != CPL_ERROR_NONE) {
825  cpl_propertylist_delete(p);
826  cpl_msg_error(fctid,"Cannot save product image extension");
827  return(-1);
828  }
829  /* Write out PAF for difference image */
830 
831  pafprop = vircam_paf_req_items(p);
832  vircam_merge_propertylists(pafprop,ps.phupaf);
833  vircam_paf_append(pafprop,p,"ESO PRO CATG");
834  if (vircam_paf_print((char *)outdiffpaf,"VIRCAM/vircam_reset_combine",
835  "QC file",pafprop) != VIR_OK)
836  cpl_msg_warning(fctid,"Unable to save PAF for difference image");
837  cpl_propertylist_delete(pafprop);
838  cpl_propertylist_delete(p);
839  }
840 
841  /* Now any further difference image stats tables */
842 
843  if (! isfirst && (we_expect & STATS_TAB)) {
844  p = cpl_propertylist_duplicate(plist);
845  if (! (we_get & STATS_TAB))
847  vircam_dfs_set_product_exten_header(p,product_frame_diffimg_stats,
848  framelist,parlist,(char *)recipeid,
849  "PRO-1.15",NULL);
850  status = VIR_OK;
851  vircam_removewcs(p,&status);
852  if (cpl_table_save(ps.diffimstats,NULL,p,outdimst,CPL_IO_EXTEND)
853  != CPL_ERROR_NONE) {
854  cpl_msg_error(fctid,"Cannot save product table extension");
855  cpl_propertylist_delete(p);
856  return(-1);
857  }
858  cpl_propertylist_delete(p);
859  }
860 
861  return(0);
862 }
863 
864 /*---------------------------------------------------------------------------*/
868 /*---------------------------------------------------------------------------*/
869 
870 static void vircam_reset_combine_dummy_products(void) {
871 
872  /* See if you even need to be here */
873 
874  if (we_get == we_expect)
875  return;
876 
877  /* First an output combined reset frame */
878 
879  if (! (we_get & MEANRESET)) {
880  ps.outimage = vircam_dummy_image(ps.resets[0]);
881 
882  /* Set up the QC parameters */
883 
884  vircam_reset_combine_config.resetmed = 0.0;
885  vircam_reset_combine_config.resetrms = 0.0;
886  }
887 
888  /* Do the difference image */
889 
890  if ((we_expect & DIFFIMG) && ! (we_get & DIFFIMG)) {
891  vircam_reset_combine_config.resetdiff_med = 0.0;
892  vircam_reset_combine_config.resetdiff_rms = 0.0;
893 
894  /* Is a difference image required? If so then let's have it... */
895 
896  ps.diffimg = vircam_dummy_image(ps.resets[0]);
897  }
898 
899  /* If a difference image stats table is required, then do that now */
900 
901  if ((we_expect & STATS_TAB) && ! (we_get & STATS_TAB))
902  ps.diffimstats = vircam_create_diffimg_stats(0);
903 
904 
905  return;
906 }
907 
908 /*---------------------------------------------------------------------------*/
913 /*---------------------------------------------------------------------------*/
914 
915 static void vircam_reset_combine_normal(int jext) {
916  int nx,ny,ncells;
917  long npi;
918  unsigned char *bpm;
919  float med,sig,*idata,grms,gdiff;
920  const char *fctid="vircam_reset_combine_normal";
921  cpl_table *ctable;
922  cpl_propertylist *p;
923 
924  /* Load up the bad pixel mask */
925 
926  nx = (int)cpl_image_get_size_x(ps.outimage);
927  ny = (int)cpl_image_get_size_y(ps.outimage);
928  npi = nx*ny;
929  bpm = vircam_mask_get_data(ps.master_mask);
930 
931  /* Work out the RMS of the mean reset frame */
932 
933  idata = cpl_image_get_data(ps.outimage);
934  vircam_medmad(idata,bpm,npi,&med,&sig);
935  sig *= 1.48;
936  vircam_reset_combine_config.resetmed = med;
937  vircam_reset_combine_config.resetrms = sig;
938 
939  /* Load up the master reset */
940 
941  if (ps.master_reset != NULL) {
942  ps.mrimage = vircam_fits_load(ps.master_reset,CPL_TYPE_FLOAT,jext);
943  if (ps.mrimage == NULL)
944  cpl_msg_info(fctid,
945  "Master reset extension %" CPL_SIZE_FORMAT " won't load",
946  (cpl_size)jext);
947  else if (vircam_is_dummy(vircam_fits_get_ehu(ps.mrimage))) {
948  cpl_msg_info(fctid,
949  "Master reset extension %" CPL_SIZE_FORMAT " is a dummy!",
950  (cpl_size)jext);
951  freefits(ps.mrimage);
952  }
953  } else
954  ps.mrimage = NULL;
955 
956  /* Load up the channel table */
957 
958  if (ps.chantab != NULL) {
959  ctable = cpl_table_load(cpl_frame_get_filename(ps.chantab),jext,0);
960  if (ctable == NULL) {
961  cpl_error_reset();
962  cpl_msg_info(fctid,
963  "Channel table extension %" CPL_SIZE_FORMAT " won't load",
964  (cpl_size)jext);
965  } else if (vircam_chantab_verify(ctable) != VIR_OK) {
966  cpl_msg_info(fctid,
967  "Channel table extension %" CPL_SIZE_FORMAT " has errors",
968  (cpl_size)jext);
969  freetable(ctable);
970  } else {
971  p = cpl_propertylist_load(cpl_frame_get_filename(ps.chantab),
972  (cpl_size)jext);
973  if (vircam_is_dummy(p)) {
974  cpl_msg_info(fctid,
975  "Channel table extensions %" CPL_SIZE_FORMAT " is a dummy",
976  (cpl_size)jext);
977  freetable(ctable);
978  }
979  freepropertylist(p);
980  }
981  } else
982  ctable = NULL;
983 
984  /* Form the difference image. NB: the difference image routine
985  copes if the input mean image and or the channel tables are
986  null. Thus if either or both are null because of a failure
987  to load then the routine will do as much as it can and return
988  allowing you to fill in the rest with dummy products */
989 
990  vircam_reset_combine_config.resetdiff_med = 0.0;
991  vircam_reset_combine_config.resetdiff_rms = 0.0;
992  ncells = vircam_reset_combine_config.ncells;
994  ps.outimage,bpm,ctable,ncells,1,
995  &gdiff,&grms,&(ps.diffimg),
996  &(ps.diffimstats));
997  vircam_mask_clear(ps.master_mask);
998  vircam_reset_combine_config.resetdiff_med = gdiff;
999  vircam_reset_combine_config.resetdiff_rms = grms;
1000  freetable(ctable);
1001  if (ps.diffimg != NULL)
1002  we_get |= DIFFIMG;
1003  if (ps.diffimstats != NULL)
1004  we_get |= STATS_TAB;
1005  return;
1006 }
1007 
1008 /*---------------------------------------------------------------------------*/
1016 /*---------------------------------------------------------------------------*/
1017 
1018 static int vircam_reset_combine_lastbit(int jext, cpl_frameset *framelist,
1019  cpl_parameterlist *parlist) {
1020  int retval;
1021  const char *fctid="vircam_reset_combine_lastbit";
1022 
1023  /* Make whatever dummy products you need */
1024 
1025  vircam_reset_combine_dummy_products();
1026 
1027  /* Save everything */
1028 
1029  cpl_msg_info(fctid,
1030  "Saving products for extension %" CPL_SIZE_FORMAT,
1031  (cpl_size)jext);
1032  retval = vircam_reset_combine_save(framelist,parlist);
1033  if (retval != 0) {
1034  vircam_reset_combine_tidy(2);
1035  return(-1);
1036  }
1037 
1038  /* Free some stuff up */
1039 
1040  vircam_reset_combine_tidy(1);
1041  return(0);
1042 }
1043 
1044 /*---------------------------------------------------------------------------*/
1048 /*---------------------------------------------------------------------------*/
1049 
1050 static void vircam_reset_combine_init(void) {
1051  ps.labels = NULL;
1052  ps.resetlist = NULL;
1053  ps.resets = NULL;
1054  ps.nresets = 0;
1055  ps.good = NULL;
1056  ps.master_reset = NULL;
1057  ps.master_mask = NULL;
1058  ps.chantab = NULL;
1059  ps.outimage = NULL;
1060  ps.drs = NULL;
1061  ps.rejmask = NULL;
1062  ps.rejplus = NULL;
1063  ps.mrimage = NULL;
1064  ps.diffimg = NULL;
1065  ps.diffimstats = NULL;
1066  ps.phupaf = NULL;
1067 }
1068 
1069 /*---------------------------------------------------------------------------*/
1073 /*---------------------------------------------------------------------------*/
1074 
1075 static void vircam_reset_combine_tidy(int level) {
1076  freeimage(ps.outimage);
1077  freefitslist(ps.resets,ps.nresets);
1078  freespace(ps.rejmask);
1079  freespace(ps.rejplus);
1080  freepropertylist(ps.drs);
1081  freefits(ps.mrimage);
1082  freeimage(ps.diffimg);
1083  freetable(ps.diffimstats);
1084  if (level == 1)
1085  return;
1086  freespace(ps.labels);
1087  freeframeset(ps.resetlist);
1088  freeframe(ps.master_reset);
1089  freemask(ps.master_mask);
1090  freeframe(ps.chantab);
1091  freespace(ps.good);
1092  freepropertylist(ps.phupaf);
1093 }
1094 
1097 /*
1098 
1099 $Log: not supported by cvs2svn $
1100 Revision 1.59 2012/01/15 17:40:09 jim
1101 Minor modifications to take into accout the changes in cpl API for v6
1102 
1103 Revision 1.58 2010/03/21 06:48:21 jim
1104 Fixed bug where DATANCOM wasn't being updated in all products
1105 
1106 Revision 1.57 2010/03/09 14:27:40 jim
1107 Now updates ESO PRO DATANCOM to reflect the number of images used
1108 
1109 Revision 1.56 2010/02/05 09:42:22 jim
1110 Fixed call to non-existent cpl routine
1111 
1112 Revision 1.55 2010/01/31 18:54:01 jim
1113 Reference reset included in paf
1114 
1115 Revision 1.54 2009/09/09 09:50:21 jim
1116 Modified to try and get headers right
1117 
1118 Revision 1.53 2008/12/05 13:28:32 jim
1119 Fixed save routine so that the correct version of PRO CATG is written to the
1120 paf file
1121 
1122 Revision 1.52 2008/10/01 04:59:13 jim
1123 Added call to vircam_frameset_fexists to check input frameset
1124 
1125 Revision 1.51 2008/09/30 11:33:23 jim
1126 Added PRO CATG to pafs
1127 
1128 Revision 1.50 2007/10/19 09:25:09 jim
1129 Fixed problems with missing includes
1130 
1131 Revision 1.49 2007/10/15 12:53:26 jim
1132 Modified for compatibiliity with cpl_4.0
1133 
1134 Revision 1.48 2007/07/18 15:35:42 jim
1135 Added better error handling for missing or corrupt mask extensions
1136 
1137 Revision 1.47 2007/07/09 13:21:56 jim
1138 Modified to use new version of vircam_exten_range
1139 
1140 Revision 1.46 2007/04/04 10:36:18 jim
1141 Modified to use new dfs tags
1142 
1143 Revision 1.45 2007/03/29 12:19:39 jim
1144 Little changes to improve documentation
1145 
1146 Revision 1.44 2007/03/02 12:37:16 jim
1147 Removed WCS stuff from table headers
1148 
1149 Revision 1.43 2007/03/01 12:41:49 jim
1150 Modified slightly after code checking
1151 
1152 Revision 1.42 2007/02/25 06:27:41 jim
1153 plugged a few memory leaks
1154 
1155 Revision 1.41 2007/02/19 10:03:02 jim
1156 Fixed small memory leak
1157 
1158 Revision 1.40 2007/02/15 11:54:09 jim
1159 Modified to make a distinction between initial channel table and one that
1160 has the proper linearity information
1161 
1162 Revision 1.39 2007/02/15 06:59:38 jim
1163 Added ability to write QC paf files
1164 
1165 Revision 1.38 2007/02/06 13:11:12 jim
1166 Fixed entry for PRO dictionary in cpl_dfs_set_product_header
1167 
1168 Revision 1.37 2007/02/05 14:14:05 jim
1169 Input master frame is now tagged as REFERENCE. QC removed from stats table
1170 headers
1171 
1172 Revision 1.36 2007/01/09 11:39:02 jim
1173 Moved free for ps.good in tidy routine to the correct place
1174 
1175 Revision 1.35 2007/01/08 19:09:11 jim
1176 Fixed memory leak
1177 
1178 Revision 1.34 2006/12/13 13:19:52 jim
1179 Fixed problem with bad sigma estimate
1180 
1181 Revision 1.33 2006/12/08 11:39:27 jim
1182 Fixed bug where we_expect didn't check to see if the difference image was
1183 being produced before deciding whether or not a table would be produced.
1184 
1185 Revision 1.32 2006/11/27 12:15:08 jim
1186 changed calls to cpl_propertylist_append to cpl_propertylist_update
1187 
1188 Revision 1.31 2006/09/29 11:19:31 jim
1189 changed aliases on parameter names
1190 
1191 Revision 1.30 2006/09/09 16:49:40 jim
1192 Header comment update
1193 
1194 Revision 1.29 2006/08/27 20:30:02 jim
1195 Major mods to structure of the main processing routine to deal with missing
1196 and dummy frames. Deals better with lower level failures too
1197 
1198 Revision 1.28 2006/06/15 09:58:58 jim
1199 Minor changes to docs
1200 
1201 Revision 1.27 2006/06/09 11:26:25 jim
1202 Small changes to keep lint happy
1203 
1204 Revision 1.26 2006/06/06 13:01:40 jim
1205 Fixed so that the QC parameters go into the correct headers
1206 
1207 Revision 1.25 2006/05/17 14:43:58 jim
1208 Fixed problem in save routine which messed up the PRO CATG keywords
1209 
1210 Revision 1.24 2006/05/16 13:58:47 jim
1211 Fixed memory leaks that occur from not closing images at the end of
1212 the image extension loop
1213 
1214 Revision 1.23 2006/05/09 09:27:06 jim
1215 removed unecessary call to cpl_propertylist_delete
1216 
1217 Revision 1.22 2006/05/04 11:53:15 jim
1218 Fixed the way the _save routine works to be more consistent with the
1219 standard CPL way of doing things
1220 
1221 Revision 1.21 2006/04/27 09:46:01 jim
1222 Modified DFS frame types to conform to new dictionary
1223 
1224 Revision 1.20 2006/04/25 13:45:57 jim
1225 Fixed to adhere to new calling sequence for vircam_dfs routines
1226 
1227 Revision 1.19 2006/03/23 21:18:46 jim
1228 Minor changes mainly to comment headers
1229 
1230 Revision 1.18 2006/03/22 12:13:52 jim
1231 Modified to use new vircam_mask capability
1232 
1233 Revision 1.17 2006/03/15 10:43:40 jim
1234 Fixed a few things
1235 
1236 Revision 1.16 2006/03/08 14:32:35 jim
1237 Lots of little mods
1238 
1239 Revision 1.15 2006/03/03 14:29:06 jim
1240 Now calls routines with vir_fits.
1241 
1242 Revision 1.13 2006/02/22 10:01:38 jim
1243 Modified to use new version of vircam_imcombine
1244 
1245 Revision 1.12 2006/02/18 11:50:43 jim
1246 Modified the way the dfs product keywords are written using the vircam
1247 routines, rather than the cpl routine that doesn't understand image
1248 extensions
1249 
1250 Revision 1.11 2006/01/23 10:37:21 jim
1251 Now allows either a BPM or a CPM to be used as a mask
1252 
1253 Revision 1.10 2005/12/14 22:19:12 jim
1254 fixed docs
1255 
1256 Revision 1.9 2005/12/09 09:47:58 jim
1257 Many changes to add more documentation
1258 
1259 Revision 1.8 2005/12/02 10:45:38 jim
1260 The tags used in the sof are now written to the description string in the
1261 constructor. This is so that if they change in the vircam_dfs.h file, they
1262 aren't then hardcopied into each of the recipes...
1263 
1264 Revision 1.7 2005/12/01 16:25:48 jim
1265 Made the routine a bit more forgiving if certain master calibration files
1266 were missing. Now does as much as it can with the info it has
1267 
1268 Revision 1.6 2005/11/25 09:56:14 jim
1269 Tidied up some more documentation
1270 
1271 Revision 1.5 2005/11/23 14:57:40 jim
1272 A bit of tidying in response to splint messages
1273 
1274 Revision 1.4 2005/11/08 12:47:44 jim
1275 Made garbage collection a little better
1276 
1277 Revision 1.3 2005/11/07 13:13:43 jim
1278 Added some docs and calls to vircam_getnpts
1279 
1280 Revision 1.2 2005/11/03 15:16:28 jim
1281 Lots of changes mainly to strengthen error reporting
1282 
1283 Revision 1.1 2005/09/29 08:58:25 jim
1284 new routine
1285 
1286 
1287 
1288 */
1289 
1290 
const char * vircam_get_license(void)
Definition: vircam_utils.c:92
void vircam_mask_force(vir_mask *m, int nx, int ny)
Definition: vircam_mask.c:385
int vircam_compare_tags(const cpl_frame *frame1, const cpl_frame *frame2)
Definition: vircam_utils.c:141
int vircam_chantab_verify(cpl_table *intab)
int vircam_is_dummy(cpl_propertylist *p)
void vircam_merge_propertylists(cpl_propertylist *p1, cpl_propertylist *p2)
vir_fits ** vircam_fits_load_list(cpl_frameset *f, cpl_type type, int exten)
Definition: vircam_fits.c:231
cpl_frame * vircam_frameset_subgroup_1(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Definition: vircam_utils.c:247
void vircam_mask_clear(vir_mask *m)
Definition: vircam_mask.c:349
cpl_image * vircam_dummy_image(vir_fits *model)
int vircam_mask_load(vir_mask *m, int nexten, int nx, int ny)
Definition: vircam_mask.c:210
char * vircam_fits_get_fullname(vir_fits *p)
Definition: vircam_fits.c:560
unsigned char * vircam_mask_get_data(vir_mask *m)
Definition: vircam_mask.c:535
cpl_image * vircam_fits_get_image(vir_fits *p)
Definition: vircam_fits.c:349
cpl_table * vircam_create_diffimg_stats(int nrows)
cpl_frameset * vircam_frameset_subgroup(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Definition: vircam_utils.c:193
void vircam_dfs_set_product_exten_header(cpl_propertylist *plist, cpl_frame *frame, cpl_frameset *frameset, cpl_parameterlist *parlist, char *recipeid, const char *dict, cpl_frame *inherit)
Definition: vircam_dfs.c:273
void vircam_dfs_set_product_primary_header(cpl_propertylist *plist, cpl_frame *frame, cpl_frameset *frameset, cpl_parameterlist *parlist, char *recipeid, const char *dict, cpl_frame *inherit, int synch)
Definition: vircam_dfs.c:203
int vircam_fits_set_error(vir_fits *p, int status)
Definition: vircam_fits.c:627
void vircam_medmad(float *data, unsigned char *bpm, long np, float *med, float *mad)
Definition: vircam_stats.c:625
void vircam_dummy_property(cpl_propertylist *p)
cpl_propertylist * vircam_fits_get_phu(vir_fits *p)
Definition: vircam_fits.c:416
vir_fits * vircam_fits_load(cpl_frame *frame, cpl_type type, int nexten)
Definition: vircam_fits.c:80
cpl_propertylist * vircam_fits_get_ehu(vir_fits *p)
Definition: vircam_fits.c:457
int vircam_frameset_fexists(cpl_frameset *frameset)
Definition: vircam_utils.c:285
const char * vircam_mask_get_filename(vir_mask *m)
Definition: vircam_mask.c:438
vir_mask * vircam_mask_define(cpl_frameset *framelist, cpl_size *labels, cpl_size nlab)
Definition: vircam_mask.c:86
int vircam_pfits_get_detlive(const cpl_propertylist *plist, int *detlive)
Get the value of DET_LIVE.
Definition: vircam_pfits.c:624
int vircam_imcombine(vir_fits **fset, int nfits, int combtype, int scaletype, int xrej, float thresh, cpl_image **outimage, unsigned char **rejmask, unsigned char **rejplus, cpl_propertylist **drs, int *status)
Stack images into a mean or median image with rejection.
void vircam_exten_range(int inexten, const cpl_frame *fr, int *out1, int *out2)
Definition: vircam_utils.c:353
void vircam_difference_image(cpl_image *master, cpl_image *prog, unsigned char *bpm, cpl_table *chantab, int ncells, int oper, float *global_diff, float *global_rms, cpl_image **diffim, cpl_table **diffimstats)
Definition: vircam_utils.c:950
int vircam_removewcs(cpl_propertylist *p, int *status)
int vircam_dfs_set_groups(cpl_frameset *set)
Definition: vircam_dfs.c:87