UVES Pipeline Reference Manual  5.4.0
uves_response_impl.c
1 /* *
2  * This file is part of the ESO UVES Pipeline *
3  * Copyright (C) 2004,2005 European Southern Observatory *
4  * *
5  * This library is free software; you can redistribute it and/or modify *
6  * it under the terms of the GNU General Public License as published by *
7  * the Free Software Foundation; either version 2 of the License, or *
8  * (at your option) any later version. *
9  * *
10  * This program is distributed in the hope that it will be useful, *
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13  * GNU General Public License for more details. *
14  * *
15  * You should have received a copy of the GNU General Public License *
16  * along with this program; if not, write to the Free Software *
17  * Foundation, 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA *
18  * */
19 
20 /*
21  * $Author: amodigli $
22  * $Date: 2013-08-08 13:36:46 $
23  * $Revision: 1.85 $
24  * $Name: not supported by cvs2svn $
25  *
26  */
27 #ifdef HAVE_CONFIG_H
28 # include <config.h>
29 #endif
30 
31 /*----------------------------------------------------------------------------*/
39 /*----------------------------------------------------------------------------*/
40 
41 /*-----------------------------------------------------------------------------
42  Includes
43  -----------------------------------------------------------------------------*/
44 
45 /* Definitions */
46 #include <uves.h>
47 
48 /* Macro steps */
49 #include <uves_reduce.h>
50 #include <uves_reduce_utils.h>
51 #include <uves_response_efficiency.h>
52 #include <uves_response_utils.h>
53 
54 /* Utility functions */
55 #include <uves_extract.h>
56 #include <uves_plot.h>
57 #include <uves_dfs.h>
58 #include <uves_pfits.h>
59 #include <uves_parameters.h>
60 #include <uves_utils.h>
61 #include <uves_utils_wrappers.h>
62 #include <uves_utils_cpl.h>
63 #include <uves_qclog.h>
64 #include <uves_recipe.h>
65 #include <uves_error.h>
66 #include <uves_msg.h>
67 #include <uves_merge.h>
68 
69 /* Library */
70 #include <cpl.h>
71 #include <stdbool.h>
72 #include <string.h>
73 
74 /*-----------------------------------------------------------------------------
75  Functions prototypes
76  -----------------------------------------------------------------------------*/
77 static void uves_efficiency_qclog(cpl_table* table,
78  uves_propertylist* raw_header,
79  enum uves_chip chip,
80  cpl_table* qclog,
81  const char *ref_obj_name);
82 
83 static int
84 uves_response_define_parameters(cpl_parameterlist *parameters);
85 
86 /*-----------------------------------------------------------------------------
87  Recipe standard code
88  -----------------------------------------------------------------------------*/
89 #define cpl_plugin_get_info uves_response_get_info
90 UVES_RECIPE_DEFINE(
91  UVES_RESPONSE_ID, UVES_RESPONSE_DOM, uves_response_define_parameters,
92  "Jonas M. Larsen", "cpl@eso.org",
93  "Determines response function and quantum efficiency",
94 "This recipe reduces a standard star frame (STANDARD_xxx or STANDARD_xxx,\n"
95 "where xxx = BLUE, RED) using a combination (depending on recipe parameters\n"
96 "and provided input frames) of the steps:\n"
97 " - bias subtraction,\n"
98 " - dark subtraction,\n"
99 " - background subtraction,\n"
100 " - extraction/cosmic ray removal,\n"
101 " - flat-field correction,\n"
102 " - wavelength rebinning,\n"
103 " - sky subtraction,\n"
104 " - order merging.\n"
105 "\n"
106 " Expected input for this recipe is an raw std star frame, STANDARD_xxx or \n"
107 "order table(s) for each chip, ORDER_TABLE_xxxx (where xxxx=BLUE, REDL, REDU),\n"
108 "line table(s) for each chip, LINE_TABLE_xxxx, a master bias frame,\n"
109 "MASTER_BIAS_xxxx, a master flat, MASTER_FLAT_xxxx, a reference standard star\n"
110 "flux table, FLUX_STD_TABLE, a table describing the atmospheric extintion,\n"
111 "EXTCOEFF_TABLE. \n"
112 
113 "Two reductions are performed, the first using optimal extraction (used to\n"
114 "compute the instrument response function), the second using linear extraction\n"
115 "(used to get the Quantum Detection Efficiency)\n"
116 "\n"
117 "For each chip (xxxx = BLUE, REDL, REDU) the recipe produces\n"
118 " INSTR_RESPONSE_xxxx Response curve\n"
119 " WCALIB_FF_RESPONSE_xxxx Response curve in (lambda,order) space before\n"
120 " correcting for exposure time, gain, binning and\n"
121 " atmospheric absorption\n"
122 " RED_STD_xxxx Reduced spectrum\n"
123 " EFFICIENCY_TABLE_xxxx Efficiency table\n"
124  " BKG_STD_xxxx The subtracted background\n");
125 
127 /*----------------------------------------------------------------------------*/
133 /*----------------------------------------------------------------------------*/
134 static int
135 uves_response_define_parameters(cpl_parameterlist *parameters)
136 {
137  /*****************
138  * General *
139  *****************/
140 
141  if (uves_define_global_parameters(parameters) != CPL_ERROR_NONE)
142  {
143  return -1;
144  }
145 
146  /*******************
147  * Reduce *
148  ******************/
149  /* Get reduce parameters for the top level and also for 'efficiency' substep */
150  if (uves_propagate_parameters_step(
151  UVES_REDUCE_ID, parameters, make_str(UVES_RESPONSE_ID), NULL) != 0)
152  {
153  return -1;
154  }
155 
156 
157  check(uves_define_efficiency_parameters(parameters),
158  "Defining efficiency parameters");
159 
160 
161  cleanup:
162  return (cpl_error_get_code() != CPL_ERROR_NONE);
163 }
164 
165 /*----------------------------------------------------------------------------*/
219 /*----------------------------------------------------------------------------*/
220 
221 static cpl_error_code
222 uves_response_process_chip(const cpl_image *raw_image,
223  const uves_propertylist *raw_header,
224  const uves_propertylist *rotated_header,
225  const cpl_image *master_bias,
226  const uves_propertylist *mbias_header,
227  const cpl_image *master_dark,
228  const uves_propertylist *mdark_header,
229  const cpl_image *master_flat,
230  const uves_propertylist *mflat_header,
231  const cpl_table *ordertable,
232  const polynomial *order_locations,
233  const cpl_table *linetable[3],
234  const uves_propertylist *linetable_header[3],
235  const polynomial *dispersion_relation[3],
236  const cpl_table *flux_table,
237  const cpl_table *atm_extinction,
238  enum uves_chip chip,
239  /* General */
240  bool debug_mode,
241  /* Backsub */
242  /* Flat fielding */
243  /* Extraction */
244  /* Rebinning */
245  const cpl_parameterlist *parameters,
246  bool calc_response,
247  /* Identification */
248  double PACCURACY,
249  /* Output */
250  char **ref_obj_id,
251  cpl_image **reduced_spectrum,
252  uves_propertylist **reduced_header,
253  cpl_image **background,
254  cpl_image **response_orders,
255  uves_propertylist **response_header_2d,
256  cpl_image **response_curve,
257  uves_propertylist **response_header,
258  cpl_table **efficiency,
259  cpl_table** blaze_efficiency,
260  cpl_table** info_tbl,
261  double *extraction_slit)
262 {
263  cpl_image *rebinned_spectrum = NULL;
264  cpl_image *rebinned_noise = NULL;
265  uves_propertylist *rebinned_header = NULL;
266  cpl_image *reduced_rebinned = NULL;
267  cpl_image *reduced_rebinned_noise = NULL;
268 
269  cpl_image *reduced_noise = NULL;
270 
271  cpl_table *cosmic_mask = NULL; /* Cosmic ray table (not a product of this recipe) */
272  cpl_table *order_trace = NULL; /* Order trace table (not a product of this recipe) */
273  cpl_image *merged_spectrum = NULL; /* Not sky-subtracted (if simple extraction) */
274  cpl_image *wave_map = NULL;
275  cpl_image *merged_sky = NULL;
276  cpl_image *merged_noise = NULL;
277  cpl_image *reduced_scaled = NULL; /* Merged, sky-subtracted and normalized
278  (exposure time, gain...) */
279 
280  cpl_table *catalogue_flux = NULL; /* Std star catalogue flux */
281 
282  /* Do the science reduction. Produces wave.cal. spectra. */
283  uves_msg("Reducing standard star");
284 
285  check( uves_reduce(raw_image,
286  raw_header,
287  rotated_header,
288  master_bias,
289  mbias_header,
290  master_dark,
291  mdark_header,
292  master_flat,
293  mflat_header,
294  ordertable,
295  order_locations,
296  linetable,
297  linetable_header,
298  dispersion_relation,
299  chip,
300  debug_mode,
301  parameters,
302  make_str(UVES_RESPONSE_ID),
303  "",
304  /* Output */
305  NULL,
306  NULL,
307  NULL, /* 2d products */
308  &cosmic_mask,
309  &wave_map,
310  background,
311  NULL,
312  NULL, /* Variance of flat-fielded */
313  NULL,
314  NULL, /* Don't need these
315  intermediate products */
316  &merged_sky,
317  &rebinned_spectrum,
318  &rebinned_noise,
319  &rebinned_header,
320  &merged_spectrum,
321  &merged_noise,
322  reduced_header,
323  &reduced_rebinned,
324  &reduced_rebinned_noise,
325  reduced_spectrum,
326  &reduced_noise,
327  info_tbl,
328  extraction_slit,
329  &order_trace),
330  "Could not reduce frame");
331 
332 
333  check( uves_plot_image_rows(*reduced_spectrum, 1, 1, 1,
334  "Wavelength (arbitrary units)", NULL,
335  "Reduced spectrum (%s chip)",
337  "Plotting failed");
338 
339  if (calc_response)
340  {
341  /* Calculate 2d response curve (but don't scale to unit exposure time, ...) */
342  uves_msg("Filtering rebinned spectrum");
343  check( uves_filter_image_median(&reduced_rebinned, 10, 0, false),
344  "Could not smooth spectrum");
345  check( uves_filter_image_average(reduced_rebinned, 10, 0),
346  "Could not smooth spectrum");
347 
348  uves_msg("Calculating 2d response curve");
349 
350  check( *response_orders = uves_calculate_response(reduced_rebinned,
351  rebinned_header,
352  flux_table,
353  raw_header,
354  PACCURACY,
355  true,
356  /* std_flux / flux */
357  ref_obj_id),
358  "Could not calculate response curve");
359 
360  check( *response_header_2d = uves_propertylist_duplicate(rebinned_header),
361  "Error creating FITS header for 2d response curve");
362 
363  check( uves_pfits_set_bunit(*response_header_2d, "FLUX_CAT / FLUX_STD"),
364  "Error writing BUNIT keyword");
365 
366  /*
367  * Calculate 1d response curve
368  */
369 
370  uves_msg("Normalizing reduced spectrum");
371 
372  {
373  int n_traces = cpl_image_get_size_y(*reduced_spectrum);
374  assure( n_traces == 1, CPL_ERROR_ILLEGAL_INPUT,
375  "2d extraction/reduction not supported");
376 
377  check( reduced_scaled = uves_normalize_spectrum(*reduced_spectrum,
378  reduced_noise,
379  *reduced_header,
380  raw_header,
381  n_traces,
382  chip,
383  atm_extinction,
384  true, /* Yes, divide by binning */
385  NULL), /* Noise spectrum */
386  "Error normalizing reduced spectrum");
387  }
388 
389  uves_msg("Filtering reduced spectrum");
390  check( uves_filter_image_median(&reduced_scaled, 10, 0, false),
391  "Could not smooth spectrum");
392  check( uves_filter_image_average(reduced_scaled, 10, 0),
393  "Could not smooth spectrum");
394 
395  uves_msg("Calculating response curve from scaled spectrum");
396 
397  cpl_free(*ref_obj_id); *ref_obj_id = NULL;
398  check( *response_curve = uves_calculate_response(reduced_scaled,
399  *reduced_header,
400  flux_table,
401  raw_header,
402  PACCURACY,
403  true, /* catalogue_flux / flux */
404  ref_obj_id),
405  "Could not calculate response curve");
406 
407  check( *response_header = uves_propertylist_duplicate(*reduced_header),
408  "Error creating FITS header for response curve");
409 
410  check( uves_pfits_set_bunit(*response_header, "FLUX_CAT / FLUX_STD"),
411  "Error writing BUNIT keyword");
412 
413  if (debug_mode)
414  {
415  check( uves_save_image_local("Pre-smoothed response curve", "raw_response",
416  *response_curve, chip, -1, -1, *response_header, true),
417  "Error saving image");
418  }
419 
420  check( uves_plot_image_rows(*response_curve, 1, 1, 1, "Wavelength (arbitrary units)",
421  NULL,
422  "Raw response (%s chip)", uves_chip_tostring_upper(chip)),
423  "Plotting failed");
424 
425 
426  /* Rebin the response curve to 50 wlu:
427  1) smooth it using a radius of 25 wlu,
428  2) then extract every n'th pixel where n = 50/step */
429  uves_msg("Rebinning response curve to step size = 50 w.l.u.");
430  {
431  double dlambda, lambda_start;
432  int n, bin, newbin;
433 
434  check( lambda_start = uves_pfits_get_crval1(*response_header),
435  "Error reading start wavelength from header");
436  check( dlambda = uves_pfits_get_cdelt1(*response_header),
437  "Error reading wavelength step from header");
438 
439  n = uves_round_double(50.0/dlambda);
440 
441  assure( n >= 1, CPL_ERROR_ILLEGAL_OUTPUT,
442  "Cannot rebin to 50 w.l.u. Current step is only %f w.l.u.", dlambda);
443 
444  /* Filter radius = 25 wlu, 0 (It's a 1d image) */
445  check( uves_filter_image_average(*response_curve, n/2, 0),
446  "Error filtering response curve");
447 
448  newbin = 1;
449  for (bin = 1+n/2; bin <= cpl_image_get_size_x(*response_curve); bin += n)
450  {
451  int pis_rejected;
452 
453  /* Write to the same image buffer */
454  cpl_image_set(*response_curve,
455  newbin, 1,
456  cpl_image_get(*response_curve, bin, 1, &pis_rejected)
457  );
458  newbin++;
459  }
460 
461 
462  /* Extract image, update start+step wavelengths */
463  uves_crop_image(response_curve, 1, 1, newbin-1, 1);
464 
465  lambda_start = lambda_start + dlambda * ((1+n/2) - 1); /* Center of first bin */
466  dlambda = n * dlambda;
467 
468  check( uves_pfits_set_crval1(*response_header, lambda_start),
469  "Error updating start wavelength");
470  check( uves_pfits_set_cdelt1(*response_header, dlambda),
471  "Error updating wavelength step");
472 
473  }
474 
475  check( uves_plot_image_rows(*response_curve, 1, 1, 1, "Wavelength (arbitrary units)",
476  NULL,
477  "Response curve (%s chip)",
479  "Plotting failed");
480 
481  /* Calculate efficiency table */
482  uves_msg("Calculating efficiency curve");
483 
484  check( uves_response_efficiency(raw_image,
485  raw_header,
486  rotated_header,
487  master_bias,
488  mbias_header,
489  master_dark,
490  mdark_header,
491  ordertable,
492  order_locations,
493  linetable,
494  linetable_header,
495  dispersion_relation,
496  flux_table,
497  atm_extinction,
498  chip,
499  debug_mode,
500  parameters,
501  PACCURACY,
502  efficiency,
503  blaze_efficiency),
504  "Efficiency calculation failed");
505 
506  check( uves_plot_table(*efficiency, "Wave", "Eff",
507  "Detection Quantum Efficiency (%s chip)",
508  uves_chip_tostring_upper(chip)),
509  "Plotting failed");
510 
511  /* Save blaze function efficiency (efficiency at center of order) */
512  if (debug_mode) check( uves_save_table_local("Blaze efficiency table",
513  "blaze_efficiency",
514  *blaze_efficiency, chip, -1, -1, rotated_header, NULL),
515  "Error saving blaze efficiency table");
516  }
517  else
518  {
519  uves_msg("Skipping response/efficiency computation");
520  }
521 
522  cleanup:
523  uves_free_propertylist(&rebinned_header);
524  uves_free_image(&rebinned_noise);
525  uves_free_image(&rebinned_spectrum);
526  uves_free_table(&cosmic_mask);
527  uves_free_table(&order_trace);
528  uves_free_image(&merged_spectrum);
529  uves_free_image(&merged_noise);
530  uves_free_image(&merged_sky);
531  uves_free_image(&reduced_rebinned);
532  uves_free_image(&reduced_rebinned_noise);
533  uves_free_image(&reduced_noise);
534  uves_free_image(&reduced_scaled);
535  uves_free_table(&catalogue_flux);
536 
537  if (cpl_error_get_code() != CPL_ERROR_NONE)
538  {
539  /* Output */
540  uves_free_image(reduced_spectrum);
541  uves_free_image(background);
542  uves_free_image(response_orders);
543  uves_free_image(response_curve);
544  uves_free_propertylist(reduced_header);
545  uves_free_propertylist(response_header);
546  uves_free_propertylist(response_header_2d);
547  uves_free_table(efficiency);
548  }
549 
550  return cpl_error_get_code();
551 }
552 
553 /*----------------------------------------------------------------------------*/
560 /*----------------------------------------------------------------------------*/
561 static void
562 UVES_CONCAT2X(UVES_RESPONSE_ID,exe)(cpl_frameset *frames,
563  const cpl_parameterlist *parameters,
564  const char *starttime)
565 {
566  /*
567  * Variables containg the values of recipe parameters
568  */
569 
570  /* General */
571  bool debug_mode;
572 
573  /* Background subtraction */
574  /* Implicitly passed */
575 
576  /* Flat-fielding */
577 
578  /* Extraction */
579  /* Implicitly passed */
580 
581  /* Rebinning */
582  /* Implicitly passed */
583 
584  /* Efficiency */
585  double PACCURACY;
586 
587  /* CPL objects */
588  /* Input, raw */
589  cpl_image *raw_image[2] = {NULL, NULL};
590  uves_propertylist *raw_header[2] = {NULL, NULL};
591  uves_propertylist *rotated_header[2] = {NULL, NULL};
592 
593  /* Input, calib */
594  cpl_image *master_bias = NULL;
595  uves_propertylist *master_bias_header = NULL;
596 
597  cpl_image *master_dark = NULL;
598  uves_propertylist *master_dark_header = NULL;
599 
600  cpl_image *master_flat = NULL;
601  uves_propertylist *master_flat_header = NULL;
602 
603  cpl_table *ordertable = NULL;
604  uves_propertylist *ordertable_header= NULL;
605  polynomial *order_locations = NULL;
606  cpl_table *traces = NULL;
607 
608  cpl_table *flux_table = NULL;
609 
610  cpl_table *atm_extinction = NULL;
611 
612  /* Line tables for sky, object, sky (UVES specific) */
613  const cpl_table *linetable[3] = {NULL, NULL, NULL};
614  const uves_propertylist *linetable_header[3] = {NULL, NULL, NULL};
615  const polynomial *dispersion_relation[3] = {NULL, NULL, NULL};
616 
617  /* Output */
618  cpl_image *background = NULL;
619  cpl_image *reduced_spectrum = NULL;
620  uves_propertylist *spectrum_header = NULL;
621  cpl_image *response_orders = NULL;
622  uves_propertylist *response_header_2d = NULL;
623  cpl_image *response_curve = NULL;
624  uves_propertylist *response_header = NULL;
625  cpl_table *efficiency = NULL;
626  cpl_table *blaze_efficiency = NULL;
627  uves_propertylist *efficiency_header = NULL;
628  cpl_table* info_tbl=NULL;
629 
630  /* Local variables */
631  cpl_table *qclog[2] = {NULL, NULL};
632  cpl_table *qclog_optext[2] = {NULL, NULL};
633  cpl_table *catalogue_flux = NULL;
634  const char *raw_filename = ""; /* Static */
635  const char *flux_table_filename = ""; /* Static */
636  const char *atm_ext_filename = ""; /* Static */
637  char *product_filename = NULL; /* Dynamically allocated */
638  char *prod_catg = NULL;
639  char *ref_obj_name = NULL; /* Reference object id */
640  bool calc_response = false; /* Calculate instr response? */
641  double extraction_slit;
642 
643  bool blue = false;
644  enum uves_chip chip;
645  int binx = 0;
646  int biny = 0;
647  const char* PROCESS_CHIP=NULL;
648 
649  const char *ordertable_filename = "";
650  const char *linetable_filename = "";
651  const char *master_bias_filename = "";
652  const char *master_dark_filename = "";
653  const char *master_flat_filename = "";
654  const char *chip_name = "";
655  int tracerow=0; /* Index of table row */
656 
657  int raw_index = 0;
658 
659 
660  double trace_offset=0;
661  int trace_number=0;
662  int trace_enabled=0;
663  int window=0; /* window number */
664  merge_method m_method;
665 
666  /* Read recipe parameters */
667  {
668  /* General */
669  check( uves_get_parameter(parameters, NULL, "uves", "debug",
670  CPL_TYPE_BOOL , &debug_mode ), "Could not read parameter");
671 
672  check( uves_get_parameter(parameters, NULL, "uves", "process_chip", CPL_TYPE_STRING, &PROCESS_CHIP),
673  "Could not read parameter");
674  uves_string_toupper((char*)PROCESS_CHIP);
675 
676  /* Background subtraction, Flat-fielding, Rebinning */
677  /* The input parameter list is passed */
678 
679  /* For both response curve and efficiency step:
680  Allow only extraction methods average/linear/optimal */
681  {
682  extract_method em;
683 
684  /* Validate uves_response.reduce.extract.method */
686  parameters, NULL,
687  make_str(UVES_RESPONSE_ID) "." UVES_REDUCE_ID "." UVES_EXTRACT_ID),
688  "Could not read extraction method");
689 
690  assure( em == EXTRACT_LINEAR || em == EXTRACT_AVERAGE || em == EXTRACT_OPTIMAL,
691  CPL_ERROR_UNSUPPORTED_MODE,
692  "Use linear/average/optimal extraction method to calculate response curve");
693 
694  /* Validate uves_response.efficiency.reduce.extract.method */
696  parameters, NULL,
697  make_str(UVES_RESPONSE_ID) ".efficiency." UVES_REDUCE_ID "." UVES_EXTRACT_ID),
698  "Could not read extraction method");
699 
700  assure( em == EXTRACT_LINEAR || em == EXTRACT_AVERAGE || em == EXTRACT_OPTIMAL,
701  CPL_ERROR_UNSUPPORTED_MODE,
702  "Use linear/average/optimal extraction "
703  "method to calculate quantum efficiency");
704  }
705 
706  /* Identification */
707  check( uves_get_parameter(parameters, NULL,
708  make_str(UVES_RESPONSE_ID) ".efficiency", "paccuracy",
709  CPL_TYPE_DOUBLE, &PACCURACY),
710  "Could not read parameter");
711  }
712 
713  /* Load raw image and header, and identify input frame as red or blue */
714  check( uves_load_standard(frames,
715  &raw_filename, raw_image, raw_header, rotated_header,
716  &blue),
717  "Error loading raw frame");
718 
719  /* Load flux table */
720  check( uves_load_flux_table(frames, &flux_table_filename, &flux_table),
721  "Error loading standard flux table");
722 
723  uves_msg_low("Using standard star flux table in '%s'", flux_table_filename);
724 
725  /* Before doing the reduction, find out if the standard star is in the table.
726  * If not, still do the science reduction, but skip the instr.response part
727  */
728  catalogue_flux = uves_align(raw_header[0], flux_table, PACCURACY, &ref_obj_name);
729 
730  calc_response = true;
731  if (cpl_error_get_code() == CPL_ERROR_INCOMPATIBLE_INPUT)
732  {
734 
735  uves_msg_warning("No catalogue object found within %.2f arcsecs. "
736  "Instrument response curve will not be computed",
737  PACCURACY);
738 
739  calc_response = false;
740  }
741 
742  /* Load atmospheric extinction table */
743  check( uves_load_atmo_ext(frames, &atm_ext_filename, &atm_extinction),
744  "Error loading extinction coefficients");
745 
746  uves_msg_low("Using atmospheric extinction table in '%s'", atm_ext_filename);
747 
748  //cpl_parameterlist_dump(parameters,stdout);
749  //uves_msg("recipe=%s",UVES_RESPONSE_ID);
750  check( m_method = uves_get_merge_method(parameters, "uves_cal_response", "reduce"),
751  "Could not get merging method");
752  /* Adjust parameters according to binning
753  */
754  check (binx = uves_pfits_get_binx(raw_header[0]),
755  "Could not read x binning factor from input header");
756  check (biny = uves_pfits_get_biny(raw_header[0]),
757  "Could not read y binning factor from input header");
758 
759  /* Loop over one or two chips, over traces and
760  over extraction windows */
761  for (chip = uves_chip_get_first(blue);
762  chip != UVES_CHIP_INVALID;
763  chip = uves_chip_get_next(chip))
764  {
765 
766 
767  if(strcmp(PROCESS_CHIP,"REDU") == 0) {
768  chip = uves_chip_get_next(chip);
769  }
770 
771 
772  /* const char *drs_filename = ""; not used */
773  raw_index = uves_chip_get_index(chip);
774  uves_msg("Processing %s chip in '%s'",
775  uves_chip_tostring_upper(chip), raw_filename);
776 
777  check_nomsg( chip_name = uves_pfits_get_chipid(raw_header[raw_index], chip));
778 
779  uves_msg_debug("Binning = %dx%d", binx, biny);
780 
781  /* Load master bias, set pointer to NULL if not present */
782  uves_free_image(&master_bias);
783  uves_free_propertylist(&master_bias_header);
784  if (cpl_frameset_find(frames, UVES_MASTER_BIAS(chip)) != NULL)
785  {
786  check( uves_load_mbias(frames, chip_name,
787  &master_bias_filename,
788  &master_bias,
789  &master_bias_header, chip),
790  "Error loading master bias");
791 
792  uves_msg_low("Using master bias in '%s'", master_bias_filename);
793  }
794  else
795  {
796  uves_msg_low("No master bias in SOF. Bias subtraction not done");
797  }
798 
799 
800  /* Load master dark, set pointer to NULL if not present */
801  uves_free_image(&master_dark);
802  uves_free_propertylist(&master_dark_header);
803  if (cpl_frameset_find(frames, UVES_MASTER_DARK(chip)) != NULL)
804  {
805  check( uves_load_mdark(frames, chip_name,
806  &master_dark_filename,
807  &master_dark,
808  &master_dark_header, chip),
809  "Error loading master dark");
810 
811  uves_msg_low("Using master dark in '%s'", master_dark_filename);
812  }
813  else
814  {
815  uves_msg_low("No master dark in SOF. Dark subtraction not done");
816  }
817 
818  /* Load master flat */
819  uves_free_image(&master_flat);
820  uves_free_propertylist(&master_flat_header);
821  check( uves_load_mflat_const(frames, chip_name,
822  &master_flat_filename,
823  &master_flat,
824  &master_flat_header,
825  chip, NULL), "Error loading master flat");
826 
827  uves_msg_low("Using master flat in '%s'", master_flat_filename);
828 
829 
830  /* Load the order table for this chip */
831  uves_free_table (&ordertable);
832  uves_free_propertylist(&ordertable_header);
833  uves_polynomial_delete(&order_locations);
834  uves_free_table (&traces);
835 
836  check( uves_load_ordertable(frames,
837  false, /* FLAMES? */
838  chip_name,
839  &ordertable_filename,
840  &ordertable,
841  &ordertable_header,
842  NULL,
843  &order_locations, &traces,
844  NULL, NULL,
845  NULL, NULL, /* fibre_pos,fibre_mask */
846  chip, false),
847  "Could not load order table");
848  uves_msg_low("Using order table in '%s'", ordertable_filename);
849 
850 
851  /* Loop over all traces (1 trace for UVES) */
852  for(tracerow = 0; tracerow < cpl_table_get_nrow(traces); tracerow++)
853  {
854 
855 
856  trace_offset = cpl_table_get_double(traces, "Offset" , tracerow, NULL);
857  trace_number = cpl_table_get_int (traces, "TraceID" , tracerow, NULL);
858  trace_enabled = cpl_table_get_int (traces, "Tracemask" , tracerow, NULL);
859 
860  if (trace_enabled != 0)
861  {
862 
863  if (cpl_table_get_nrow(traces) > 1) {
864  uves_msg("Processing trace %d", trace_number);
865  }
866 
867  /* This is UVES specific. Load linetable for the
868  two sky windows (number 1, 3) and for the object
869  window (number 2) */
870 
871  for (window = 1; window <= 3; window ++)
872  {
873  uves_free_table_const ( &(linetable[window-1]) );
874  uves_free_propertylist_const( &(linetable_header[window-1]) );
875  uves_polynomial_delete_const( &(dispersion_relation[window-1]) );
876  check( uves_load_linetable_const(
877  frames,
878  false, /* FLAMES? */
879  chip_name,
880  order_locations,
881  cpl_table_get_column_min(ordertable, "Order"),
882  cpl_table_get_column_max(ordertable, "Order"),
883  &linetable_filename,
884  &(linetable [window-1]),
885  &(linetable_header [window-1]),
886  &(dispersion_relation[window-1]),
887  NULL,
888  chip,
889  trace_number,
890  window),
891  "Could not load line table, window #%d", window);
892  }
893 
894  uves_msg_low("Using line tables in '%s'", linetable_filename);
895  /* end, UVES specific */
896 
897  /* Do the reduction + response calculation */
898  cpl_free(ref_obj_name); ref_obj_name = NULL;
899  uves_free_image(&reduced_spectrum);
900  uves_free_image(&background);
901  uves_free_image(&response_orders);
902  uves_free_image(&response_curve);
903  uves_free_propertylist(&response_header);
904  uves_free_propertylist(&spectrum_header);
905  uves_free_propertylist(&response_header_2d);
906  uves_free_table(&efficiency);
907  uves_free_table(&blaze_efficiency);
908  uves_free_table(&info_tbl);
909 
910  check( uves_response_process_chip(
911  raw_image[raw_index], /* Raw */
912  raw_header[raw_index],
913  rotated_header[raw_index],
914  master_bias, /* Calibration */
915  master_bias_header,
916  master_dark,
917  master_dark_header,
918  master_flat,
919  master_flat_header,
920  ordertable,
921  order_locations,
922  linetable,
923  linetable_header,
924  dispersion_relation,
925  flux_table,
926  atm_extinction,
927  chip, /* Parameters */
928  debug_mode,
929  parameters,
930  calc_response,
931  PACCURACY, /* Identification */
932  &ref_obj_name,
933  &reduced_spectrum,
934  &spectrum_header,
935  &background,
936  &response_orders,
937  &response_header_2d,
938  &response_curve,
939  &response_header,
940  &efficiency,
941  &blaze_efficiency,
942  &info_tbl,
943  &extraction_slit),
944  "Response computation failed");
945 
946  uves_msg("Saving products...");
947 
948  /* Calculate QC (two tables) */
949 
950  if (calc_response)
951  {
952  uves_qclog_delete(&qclog[0]);
953  qclog[0] = uves_qclog_init(raw_header[raw_index], chip);
954 
955  check( uves_efficiency_qclog(blaze_efficiency,
956  raw_header[raw_index],
957  chip,
958  qclog[0],
959  ref_obj_name),
960  "Error generating efficiency QC log");
961  }
962 
963  uves_qclog_delete(&qclog_optext[0]);
964  qclog_optext[0] = cpl_table_new(0);
965  /* Do not:
966  qclog_optext[0] = uves_qclog_init(raw_header[raw_index], chip);
967  because we don't want QC.DID for this
968  */
969  cpl_table_new_column(qclog_optext[0],"key_name", CPL_TYPE_STRING);
970  cpl_table_new_column(qclog_optext[0],"key_type", CPL_TYPE_STRING);
971  cpl_table_new_column(qclog_optext[0],"key_value",CPL_TYPE_STRING);
972  cpl_table_new_column(qclog_optext[0],"key_help", CPL_TYPE_STRING);
973 
974  check( uves_qclog_add_sci(qclog_optext[0],
975  raw_header[raw_index],
976  raw_image[raw_index],
977  extraction_slit,
978  info_tbl),
979  "Error generating extraction QC log");
980 
981  if (calc_response)
982  {
983  /* Save response curve */
984  cpl_free(product_filename);
985  check( product_filename = uves_response_curve_filename(chip),
986  "Error getting filename");
987  check( uves_frameset_insert(
988  frames,
989  response_curve,
990  CPL_FRAME_GROUP_PRODUCT,
991  CPL_FRAME_TYPE_IMAGE,
992  CPL_FRAME_LEVEL_INTERMEDIATE,
993  product_filename,
994  UVES_INSTR_RESPONSE(chip),
995  raw_header[raw_index],
996  response_header,
997  NULL,
998  parameters,
999  make_str(UVES_RESPONSE_ID),
1000  PACKAGE "/" PACKAGE_VERSION,
1001  qclog_optext,
1002  starttime, false,
1003  UVES_ALL_STATS),
1004  "Could not add response curve '%s' (%s) to frameset",
1005  product_filename, UVES_INSTR_RESPONSE(chip));
1006 
1007  uves_msg("Response curve '%s' (%s) added to frameset",
1008  product_filename, UVES_INSTR_RESPONSE(chip));
1009 
1010  /* Save response curve (2d) */
1011  cpl_free(product_filename);
1012  check( product_filename =
1013  uves_response_curve_2d_filename(chip),
1014  "Error getting filename");
1015  check( uves_frameset_insert(
1016  frames,
1017  response_orders,
1018  CPL_FRAME_GROUP_PRODUCT,
1019  CPL_FRAME_TYPE_IMAGE,
1020  CPL_FRAME_LEVEL_INTERMEDIATE,
1021  product_filename,
1022  UVES_WCALIB_FF_RESPONSE(chip),
1023  raw_header[raw_index],
1024  response_header_2d,
1025  NULL,
1026  parameters,
1027  make_str(UVES_RESPONSE_ID),
1028  PACKAGE "/" PACKAGE_VERSION,
1029  qclog_optext,
1030  starttime, false,
1031  UVES_ALL_STATS),
1032  "Could not add response curve (2d) "
1033  "'%s' (%s) to frameset",
1034  product_filename, UVES_WCALIB_FF_RESPONSE(chip));
1035 
1036  uves_msg("Response curve (2d) '%s' (%s) added to frameset",
1037  product_filename, UVES_WCALIB_FF_RESPONSE(chip));
1038  }
1039 
1040  /* Save reduced spectrum */
1041  cpl_free(product_filename);
1042  if (m_method == MERGE_NOAPPEND) {
1043  check( product_filename = uves_response_red_noappend_standard_filename(chip),
1044  "Error getting filename");
1045  prod_catg=UVES_RED_NOAPPEND_STD(chip);
1046  } else {
1047 
1048  check( product_filename = uves_response_red_standard_filename(chip),
1049  "Error getting filename");
1050  prod_catg=UVES_RED_STD(chip);
1051 
1052  }
1053  check( uves_frameset_insert(frames,
1054  reduced_spectrum,
1055  CPL_FRAME_GROUP_PRODUCT,
1056  CPL_FRAME_TYPE_IMAGE,
1057  CPL_FRAME_LEVEL_INTERMEDIATE,
1058  product_filename,
1059  prod_catg,
1060  raw_header[raw_index],
1061  spectrum_header,
1062  NULL,
1063  parameters,
1064  make_str(UVES_RESPONSE_ID),
1065  PACKAGE "/" PACKAGE_VERSION,
1066  qclog_optext,
1067  starttime, false,
1068  UVES_ALL_STATS),
1069  "Could not add reduced spectrum '%s' (%s) to frameset",
1070  product_filename, UVES_RED_STD(chip));
1071 
1072  uves_msg("Reduced spectrum '%s' (%s) added to frameset",
1073  product_filename, UVES_RED_STD(chip));
1074 
1075  if (calc_response)
1076  {
1077  /* Save efficiency table */
1078  uves_free_propertylist(&efficiency_header);
1079  efficiency_header = uves_propertylist_new();
1080 
1081  cpl_free(product_filename);
1082  check( product_filename =
1083  uves_response_efficiency_filename(chip),
1084  "Error getting filename");
1085 
1086  check( uves_frameset_insert(
1087  frames,
1088  efficiency,
1089  CPL_FRAME_GROUP_PRODUCT,
1090  CPL_FRAME_TYPE_TABLE,
1091  CPL_FRAME_LEVEL_INTERMEDIATE,
1092  product_filename,
1093  UVES_EFFICIENCY_TABLE(chip),
1094  raw_header[raw_index],
1095  efficiency_header,
1096  NULL,
1097  parameters,
1098  make_str(UVES_RESPONSE_ID),
1099  PACKAGE "/" PACKAGE_VERSION,
1100  qclog,
1101  starttime, true, 0),
1102  "Could not add background image '%s' (%s) to frameset",
1103  product_filename, UVES_EFFICIENCY_TABLE(chip));
1104 
1105  uves_msg("Efficiency table '%s' (%s) added to frameset",
1106  product_filename, UVES_EFFICIENCY_TABLE(chip));
1107  } /* end if calc_response */
1108 
1109  /* Save background image */
1110  cpl_free(product_filename);
1111  check( product_filename =
1112  uves_response_bkg_standard_filename(chip),
1113  "Error getting filename");
1114  check( uves_frameset_insert(frames,
1115  background,
1116  CPL_FRAME_GROUP_PRODUCT,
1117  CPL_FRAME_TYPE_IMAGE,
1118  CPL_FRAME_LEVEL_INTERMEDIATE,
1119  product_filename,
1120  UVES_BKG_STD(chip),
1121  raw_header[raw_index],
1122  rotated_header[raw_index],
1123  NULL,
1124  parameters,
1125  make_str(UVES_RESPONSE_ID),
1126  PACKAGE "/" PACKAGE_VERSION, NULL,
1127  starttime, false,
1128  CPL_STATS_MIN | CPL_STATS_MAX),
1129  "Could not add background image '%s' (%s) to frameset",
1130  product_filename, UVES_BKG_STD(chip));
1131 
1132  uves_msg("Background image '%s' (%s) added to frameset",
1133  product_filename, UVES_BKG_STD(chip));
1134 
1135 
1136  /* Save info table */
1137  cpl_free(product_filename);
1138  check( product_filename =
1139  uves_order_extract_qc_standard_filename(chip),
1140  "Error getting filename");
1141  check( uves_frameset_insert(frames,
1142  info_tbl,
1143  CPL_FRAME_GROUP_PRODUCT,
1144  CPL_FRAME_TYPE_TABLE,
1145  CPL_FRAME_LEVEL_INTERMEDIATE,
1146  product_filename,
1147  UVES_ORDER_EXTRACT_QC(chip),
1148  raw_header[raw_index],
1149  rotated_header[raw_index],
1150  NULL,
1151  parameters,
1152  make_str(UVES_RESPONSE_ID),
1153  PACKAGE "/" PACKAGE_VERSION, NULL,
1154  starttime, true,
1155  0),
1156  "Could not add extraction quality table '%s' (%s) to frameset",
1157  product_filename, UVES_ORDER_EXTRACT_QC(chip));
1158 
1159  uves_msg("Extraction quality table '%s' (%s) added to frameset",
1160  product_filename, UVES_ORDER_EXTRACT_QC(chip));
1161 
1162 
1163 
1164  }/* ... if trace is enabled */
1165  else
1166  {
1167  uves_msg_low("Skipping trace number %d", trace_number);
1168  }
1169 
1170  }/* For each trace */
1171 
1172 
1173  if(strcmp(PROCESS_CHIP,"REDL") == 0) {
1174  chip = uves_chip_get_next(chip);
1175  }
1176 
1177 
1178 
1179 
1180  } /* For each chip */
1181 
1182  cleanup:
1183  /* Input */
1184  uves_free_image(&raw_image[0]);
1185  uves_free_image(&raw_image[1]);
1186  uves_free_propertylist(&raw_header[0]);
1187  uves_free_propertylist(&raw_header[1]);
1188  uves_free_propertylist(&rotated_header[0]);
1189  uves_free_propertylist(&rotated_header[1]);
1190 
1191  /* Input, calib */
1192  uves_free_image(&master_bias);
1193  uves_free_propertylist(&master_bias_header);
1194 
1195  uves_free_image(&master_dark);
1196  uves_free_propertylist(&master_dark_header);
1197 
1198  uves_free_image(&master_flat);
1199  uves_free_propertylist(&master_flat_header);
1200 
1201  uves_free_table(&ordertable);
1202  uves_free_propertylist(&ordertable_header);
1203  uves_polynomial_delete(&order_locations);
1204  uves_free_table(&traces);
1205 
1206  uves_free_table_const( &(linetable[0]) );
1207  uves_free_table_const( &(linetable[1]) );
1208  uves_free_table_const( &(linetable[2]) );
1209  uves_free_propertylist_const( &(linetable_header[0]) );
1210  uves_free_propertylist_const( &(linetable_header[1]) );
1211  uves_free_propertylist_const( &(linetable_header[2]) );
1212  uves_polynomial_delete_const( &(dispersion_relation[0]) );
1213  uves_polynomial_delete_const( &(dispersion_relation[1]) );
1214  uves_polynomial_delete_const( &(dispersion_relation[2]) );
1215  uves_free_table( &flux_table );
1216  uves_free_table( &atm_extinction );
1217 
1218  /* Output */
1219  uves_qclog_delete(&qclog[0]);
1220  uves_qclog_delete(&qclog_optext[0]);
1221  uves_free_image(&background);
1222  uves_free_image(&reduced_spectrum);
1223  uves_free_propertylist(&spectrum_header);
1224  uves_free_propertylist(&response_header_2d);
1225  uves_free_propertylist(&response_header);
1226  uves_free_propertylist(&efficiency_header);
1227  uves_free_image(&response_orders);
1228  uves_free_image(&response_curve);
1229  uves_free_table(&efficiency);
1230  uves_free_table(&blaze_efficiency);
1231  uves_free_table(&info_tbl);
1232 
1233  /* Local */
1234  uves_free_table(&catalogue_flux);
1235  cpl_free(product_filename);
1236  cpl_free(ref_obj_name);
1237 
1238  return;
1239 }
1240 
1250 static void uves_efficiency_qclog(cpl_table* table,
1251  uves_propertylist* raw_header,
1252  enum uves_chip chip,
1253  cpl_table* qclog,
1254  const char *ref_obj_name)
1255 {
1256  int i=0;
1257  bool new_format;
1258 
1259  check( new_format = uves_format_is_new(raw_header),
1260  "Error determining FITS header format");
1261 
1263  "QC TEST1 ID",
1264  "Efficiency-Test-Results",
1265  "Name of QC test",
1266  "%s"));
1267 
1269  uves_remove_string_prefix(UVES_INSPATH,"ESO "),
1270  uves_pfits_get_inspath(raw_header),
1271  "Optical path used.",
1272  "%s"));
1273 
1275  uves_remove_string_prefix(UVES_INSMODE,"ESO "),
1276  uves_pfits_get_insmode(raw_header),
1277  "Instrument mode used.",
1278  "%s"));
1279 
1281  uves_remove_string_prefix(UVES_GRATNAME(chip),"ESO "),
1282  uves_pfits_get_gratname(raw_header,chip),
1283  "Cross disperser ID",
1284  "%s"));
1285 
1287  chip, qclog));
1288 
1289  check_nomsg(
1290  uves_qclog_add_string(qclog,
1291  uves_remove_string_prefix(UVES_CHIP_NAME(chip),"ESO "),
1292  /* UVES_QC_CHIP_VAL(chip), */
1293  uves_pfits_get_chip_name(raw_header, chip),
1294  "Detector chip name",
1295  "%s"));
1296 
1297  check_nomsg(
1298  uves_qclog_add_double(qclog,
1299  uves_remove_string_prefix(UVES_GRATWLEN(chip),"ESO "),
1300  uves_pfits_get_gratwlen(raw_header,chip),
1301  "Grating central wavelength [nm] (hs).",
1302  "%.1f"));
1303 
1304  check_nomsg(
1305  uves_qclog_add_double(qclog,
1306  uves_remove_string_prefix(UVES_CONAD(new_format, chip),"ESO "),
1307  uves_pfits_get_conad(raw_header, chip),
1308  "Conversion from ADUs to electrons",
1309  "%8.2f"));
1310 
1311 
1312  check_nomsg(
1313  uves_qclog_add_double(qclog,
1314  uves_remove_string_prefix(UVES_QC_UIT(new_format, chip), "ESO "),
1315  uves_pfits_get_uit(raw_header),
1316  "user defined subintegration time",
1317  "%8.0f"));
1318 
1319  check_nomsg(
1320  uves_qclog_add_double(qclog,
1321  "AIRMASS",
1322  (uves_pfits_get_airmass_start(raw_header) +
1323  uves_pfits_get_airmass_end(raw_header))/2.0,
1324  "Averaged airmass",
1325  "%8.4f"));
1326 
1327  check_nomsg(
1328  uves_qclog_add_string(qclog,
1329  uves_remove_string_prefix(UVES_TARG_NAME, "ESO "),
1330  ref_obj_name,
1331  "OB target name",
1332  "%s"));
1333 
1334 
1335  for(i = 0; i < cpl_table_get_nrow(table); i++)
1336  {
1337  char key_name[25];
1338 
1339  int order = cpl_table_get_int(table, "Order", i, NULL);
1340 
1341  sprintf(key_name,"QC BLAZEFF%d", order);
1342 
1344  key_name,
1345  cpl_table_get_double(table, "Eff", i, NULL),
1346  "Blaze Efficiency",
1347  "%13.6f"));
1348  /*
1349  uves_msg("QC-LOG: Wlen =%g Eff=%g",
1350  cpl_table_get_double(table,"Wave",i,NULL),
1351  cpl_table_get_double(table,"Eff",i,NULL));
1352  */
1353 
1354  sprintf(key_name,"QC BLAZWLEN%d", order);
1356  key_name,
1357  cpl_table_get_double(table, "Wave", i, NULL)/10.,
1358  "Blaze wavelength", /* nm */
1359  "%13.6f"));
1360  }
1361 
1362  /* Are these QC parameters needed anywhere? */
1363 #if 0
1364  for(i = 0; i < cpl_table_get_nrow(info_tbl); i++)
1365  {
1366  char key_name[25];
1367 
1368  int order = cpl_table_get_int(info_tbl, "Order", i, NULL);
1369 
1370  sprintf(key_name,"QC ORDER NUM%d", order);
1372  key_name,
1373  order,
1374  "Order Number",
1375  "%d"));
1376  /*
1377  uves_msg("QC-LOG: Order =%d S/N=%g",
1378  cpl_table_get_int(info_tbl,"Order",i,NULL),
1379  cpl_table_get_double(info_tbl,"S/N",i,NULL));
1380  */
1381 
1382  sprintf(key_name,"QC OBJ SN%d", order);
1384  key_name,
1385  cpl_table_get_double(info_tbl,"S/N",i,NULL),
1386  "Order S/N",
1387  "%f13.6"));
1388  }
1389 #endif
1390 
1391  cleanup:
1392  return;
1393 }
1394