UVES Pipeline Reference Manual  5.4.0
uves_redchain_impl.c
1 /* *
2  * This file is part of the ESO UVES Pipeline *
3  * Copyright (C) 2004,2005 European Southern Observatory *
4  * *
5  * This library is free software; you can redistribute it and/or modify *
6  * it under the terms of the GNU General Public License as published by *
7  * the Free Software Foundation; either version 2 of the License, or *
8  * (at your option) any later version. *
9  * *
10  * This program is distributed in the hope that it will be useful, *
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13  * GNU General Public License for more details. *
14  * *
15  * You should have received a copy of the GNU General Public License *
16  * along with this program; if not, write to the Free Software *
17  * Foundation, 51 Franklin St, Fifth Floor, Boston MA 02110-1301 USA *
18  */
19 
20 /*
21  * $Author: amodigli $
22  * $Date: 2010-09-24 09:32:07 $
23  * $Revision: 1.47 $
24  * $Name: not supported by cvs2svn $
25  * $Log: not supported by cvs2svn $
26  * Revision 1.45 2010/06/09 07:17:14 amodigli
27  * renamed parameter definition function used by response step in redchain recipe to make them more explicit
28  *
29  * Revision 1.44 2010/06/08 16:30:18 amodigli
30  * Fixed problems in parameter definition for reduction chain recipe
31  *
32  * Revision 1.43 2010/04/28 08:48:23 amodigli
33  * use uves_define_efficiency_parameters()
34  *
35  * Revision 1.42 2008/09/29 06:59:16 amodigli
36  * add #include <string.h>
37  *
38  * Revision 1.41 2008/03/28 08:54:27 amodigli
39  * IRPLIB_CONCAT2X-->UVES_CONCAT2X
40  *
41  * Revision 1.40 2008/02/15 12:43:49 amodigli
42  * allow lower/upper chip for parameter process_chip
43  *
44  * Revision 1.39 2007/11/13 16:19:55 amodigli
45  * adding DATAMD5 to calib products
46  *
47  * Revision 1.38 2007/10/05 16:01:45 amodigli
48  * using proces_chip parameter to process or not a given RED chip
49  *
50  * Revision 1.37 2007/08/21 13:08:26 jmlarsen
51  * Removed irplib_access module, largely deprecated by CPL-4
52  *
53  * Revision 1.36 2007/06/28 09:19:24 jmlarsen
54  * Do not compute master dark if provided
55  *
56  * Revision 1.35 2007/06/11 13:28:26 jmlarsen
57  * Changed recipe contact address to cpl at eso.org
58  *
59  * Revision 1.34 2007/06/08 13:06:16 jmlarsen
60  * Send bug reports to Andrea
61  *
62  * Revision 1.33 2007/06/06 08:17:33 amodigli
63  * replace tab with 4 spaces
64  *
65  * Revision 1.32 2007/05/22 09:11:06 jmlarsen
66  * Reverse logic bugfix
67  *
68  * Revision 1.31 2007/05/22 08:13:19 amodigli
69  * allow to process only blu or red data
70  *
71  * Revision 1.30 2007/05/14 08:09:48 amodigli
72  * updated input frames and tag description in recipe man page
73  *
74  * Revision 1.29 2007/05/03 16:03:58 jmlarsen
75  * uves_obs_redchain: Implemented option to skip final science reduction
76  *
77  * Revision 1.28 2007/02/09 13:40:26 jmlarsen
78  * Use defines for recipe id
79  *
80  * Revision 1.27 2007/02/09 08:58:51 jmlarsen
81  * Use define's rather than hard-coded recipe names
82  *
83  * Revision 1.26 2006/11/24 16:21:39 jmlarsen
84  * Added FIB_LINE_TABLE_x
85  *
86  * Revision 1.25 2006/11/15 15:02:15 jmlarsen
87  * Implemented const safe workarounds for CPL functions
88  *
89  * Revision 1.23 2006/11/15 14:04:08 jmlarsen
90  * Removed non-const version of parameterlist_get_first/last/next which is
91  * already in CPL, added const-safe wrapper, unwrapper and deallocator functions
92  *
93  * Revision 1.22 2006/11/13 12:47:08 jmlarsen
94  * Changed syntax of UVES_ARC_LAMP to support FLAMES
95  *
96  * Revision 1.21 2006/11/06 15:19:41 jmlarsen
97  * Removed unused include directives
98  *
99  * Revision 1.20 2006/10/24 14:06:27 jmlarsen
100  * Added flames=false where relevant
101  *
102  * Revision 1.19 2006/10/19 13:53:25 jmlarsen
103  * Changed guess line table tag to LINE_GUESS_TAB
104  *
105  * Revision 1.18 2006/10/17 12:33:02 jmlarsen
106  * Added semicolon at UVES_RECIPE_DEFINE invocation
107  *
108  * Revision 1.17 2006/10/09 13:01:13 jmlarsen
109  * Use macro to define recipe interface functions
110  *
111  * Revision 1.16 2006/10/02 08:37:49 jmlarsen
112  * Added REF_TFLAT
113  *
114  * Revision 1.15 2006/09/27 15:08:45 jmlarsen
115  * Fixed doc. bug
116  *
117  * Revision 1.14 2006/09/14 08:46:51 jmlarsen
118  * Added support for TFLAT, SCREEN_FLAT frames
119  *
120  * Revision 1.13 2006/08/17 13:56:53 jmlarsen
121  * Reduced max line length
122  *
123  * Revision 1.12 2006/08/16 11:58:02 jmlarsen
124  * Fixed trivial but harmful buffer overrun
125  *
126  * Revision 1.11 2006/08/11 14:56:05 amodigli
127  * removed Doxygen warnings
128  *
129  * Revision 1.10 2006/07/14 12:30:34 jmlarsen
130  * Compute PRO CATG depending on DO CATG
131  *
132  * Revision 1.9 2006/07/03 13:27:10 jmlarsen
133  * Minor doc update
134  *
135  * Revision 1.8 2006/07/03 13:16:47 amodigli
136  * updated description
137  *
138  * Revision 1.7 2006/06/16 08:25:45 jmlarsen
139  * Manually propagate ESO.DET. keywords from 1st/2nd input header
140  *
141  * Revision 1.6 2006/06/13 12:00:54 jmlarsen
142  * Support ORDER_GUESS_TAB
143  *
144  * Revision 1.5 2006/04/24 09:23:33 jmlarsen
145  * Recognize PDARK, DFLAT, IFLAT as products
146  *
147  * Revision 1.4 2006/04/06 12:56:50 jmlarsen
148  * Added support for PDARK, IFLAT, DLFAT frames
149  *
150  * Revision 1.3 2006/04/06 11:48:17 jmlarsen
151  * Support for SCI_POINT_-, SCI_EXTND_- and SCI_SLICER-frames
152  *
153  * Revision 1.2 2006/04/06 08:47:16 jmlarsen
154  * Support DFLAT, IFLAT, PDARK
155  *
156  * Revision 1.1 2006/02/03 07:46:30 jmlarsen
157  * Moved recipe implementations to ./uves directory
158  *
159  * Revision 1.14 2006/01/19 08:47:24 jmlarsen
160  * Inserted missing doxygen end tag
161  *
162  * Revision 1.13 2005/12/20 08:11:44 jmlarsen
163  * Added CVS entry
164  *
165  */
166 
167 #ifdef HAVE_CONFIG_H
168 # include <config.h>
169 #endif
170 
171 
172 /*----------------------------------------------------------------------------*/
178 /*----------------------------------------------------------------------------*/
179 
180 /*-----------------------------------------------------------------------------
181  Includes
182  -----------------------------------------------------------------------------*/
183 
184 #include <uves.h>
185 
186 #include <uves_parameters.h>
187 #include <uves_utils.h>
188 #include <uves_utils_wrappers.h>
189 #include <uves_dfs.h>
190 #include <uves_recipe.h>
191 #include <uves_error.h>
192 #include <uves_msg.h>
193 
194 /* Library */
195 #include <cpl.h>
196 #include <string.h>
197 /*-----------------------------------------------------------------------------
198  Local constants
199  -----------------------------------------------------------------------------*/
200 static const bool flames = false; /* This recipe is only for UVES */
201 
202 /*-----------------------------------------------------------------------------
203  Functions prototypes
204  -----------------------------------------------------------------------------*/
205 static bool frame_is_needed(bool blue, const cpl_frame *f);
206 static cpl_error_code execute_recipe(const char *recipe_id,
207  cpl_frameset *frames, const cpl_parameterlist *parameters,
208  const char *products[], int n_products, bool reclassify);
209 static bool is_missing(const cpl_frameset *frames, const char *frame1, const char *frame2);
210 static void remove_input_frame(cpl_frameset *frames, const char *tag);
211 
212 static int uves_redchain_define_parameters(cpl_parameterlist *parameters);
213 
214 /*-----------------------------------------------------------------------------
215  Recipe standard code
216  -----------------------------------------------------------------------------*/
217 #define cpl_plugin_get_info uves_redchain_get_info
218 UVES_RECIPE_DEFINE(
219  UVES_REDCHAIN_ID, UVES_REDCHAIN_DOM, uves_redchain_define_parameters,
220  "Jonas M. Larsen", "cpl@eso.org",
221  "Runs the full UVES reduction chain",
222  "This recipe does a complete science reduction. It runs all necessary\n"
223  "calibration recipes depending on the availability of raw/processed\n"
224  "calibration frames.\n"
225  "Input frames are all UVES raw and reference frames:\n"
226  "formatchecks, ARC_LAMP_FORM_xxxx, xxxx=BLUE or RED,\n"
227  "order definition frames, ORDER_FLAT_xxx,\n"
228  "biases, BIAS_xxx,\n"
229  "darks, DARK_xxx,\n"
230  "flats, FLAT_xxx,\n"
231  "arc lamps, ARC_LAMP_xxx,\n"
232  "standard stars, STANDARD_xxx\n"
233  "a wavelength catalogue table,LINE_REFER_TABLE, \n"
234  "and optionally a wavelength table of bright lines,LINE_INTMON_TABLE, \n"
235  "used only for computing Quality Control parameters.\n"
236  "a reference standard star flux table, FLUX_STD_TABLE, \n"
237  "a table describing the atmospheric extintion,EXTCOEFF_TABLE.\n"
238  "optionally, science frames, SCIENCE_xxx, or UVES_SCI_POINT_xxx, \n"
239  "or UVES_SCI_EXTND_xxx, or UVES_SCI_SLICER_xxx.\n"
240  "For further details on the data reduction and the input frame types\n"
241  "refer to the man page of the individual recipes.\n");
242 
245 /*-----------------------------------------------------------------------------
246  Functions code
247  -----------------------------------------------------------------------------*/
248 /*----------------------------------------------------------------------------*/
254 /*----------------------------------------------------------------------------*/
255 static int
256 uves_redchain_define_parameters(cpl_parameterlist *parameters)
257 {
258  const char *recipe_id = make_str(UVES_REDCHAIN_ID);
259  const char *subcontext = NULL;
260 
261  uves_par_new_value("scired",
262  CPL_TYPE_BOOL,
263  "Whether or not to do science reduction. "
264  "If false, only master calibration frames "
265  "are created. If false, either zero or all "
266  "necessary calibration frames must be provided "
267  "for each arm",
268  true);
269 
270  /*****************
271  * General *
272  *****************/
273  if (uves_define_global_parameters(parameters) != CPL_ERROR_NONE)
274  {
275  return -1;
276  }
277 
278  /******************
279  * Master bias *
280  ******************/
281  if (uves_propagate_parameters(
282  make_str(UVES_MBIAS_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
283  {
284  return -1;
285  }
286 
287 
288  /******************
289  * Master dark *
290  ******************/
291  if (uves_propagate_parameters(
292  make_str(UVES_MDARK_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
293  {
294  return -1;
295  }
296 
297  /******************
298  * Physical model *
299  ******************/
300  if (uves_propagate_parameters(
301  make_str(UVES_PHYSMOD_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
302  {
303  return -1;
304  }
305 
306  /******************
307  * Order position *
308  ******************/
309  if (uves_propagate_parameters(
310  make_str(UVES_ORDERPOS_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
311  {
312  return -1;
313  }
314 
315  /******************
316  * Master flat *
317  ******************/
318  if (uves_propagate_parameters(
319  make_str(UVES_MFLAT_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
320  {
321  return -1;
322  }
323 
324 
325  /******************
326  * Wave.cal. *
327  ******************/
328  if (uves_propagate_parameters(
329  make_str(UVES_WAVECAL_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
330  {
331  return -1;
332  }
333 
334  /******************
335  * Response *
336  ******************/
337 /*
338  if (uves_propagate_parameters(
339  make_str(UVES_RESPONSE_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
340  {
341  return -1;
342  }
343 */
344 
345  uves_define_background_for_response_chain_parameters(parameters);
346  uves_define_extract_for_response_chain_parameters(parameters);
347  uves_define_reduce_for_response_chain_parameters(parameters);
348  uves_define_rebin_for_response_chain_parameters(parameters);
349  uves_define_efficiency_for_response_chain_parameters(parameters);
350 
351 
352 
353  /******************
354  * Scired *
355  ******************/
356  if (uves_propagate_parameters(
357  make_str(UVES_SCIRED_ID), parameters, make_str(UVES_REDCHAIN_ID), NULL) != 0)
358  {
359  return -1;
360  }
361 
362  return (cpl_error_get_code() != CPL_ERROR_NONE);
363 }
364 
365 /*----------------------------------------------------------------------------*/
374 /*----------------------------------------------------------------------------*/
375 static void
376 UVES_CONCAT2X(UVES_REDCHAIN_ID,exe)(cpl_frameset *frames,
377  const cpl_parameterlist *parameters,
378  const char *starttime)
379 {
380  cpl_frameset *blue_frames = NULL;
381  cpl_frameset *red_frames = NULL;
382  cpl_frameset *common_frames = NULL;
383 
384 
385  bool blue;
386  bool do_science;
387 
388  bool run_mbias[2]; /* index 0 (==false): red arm */
389  bool run_mdark[2]; /* index 1 (==true ): blue arm */
390  bool run_mflat[2];
391  bool run_physmod[2];
392  bool run_orderpos[2];
393  bool run_wavecal[2];
394  bool run_response[2];
395  bool run_scired[2];
396  bool nraw_arm[2]; /* Do we have frames used exclusively
397  for this arm? */
398  const char* PROCESS_CHIP=NULL;
399 
400  /* Exceptionally, this parameter is not used because this
401  recipe does not create any products on its own. Suppress
402  warning about unused variable */
403  starttime = starttime;
404 
405  check( uves_get_parameter(parameters, NULL, make_str(UVES_REDCHAIN_ID), "scired",
406  CPL_TYPE_BOOL, &do_science), "Could not read parameter");
407 
408  /* Check for at least one science frame */
409  assure(!do_science ||
410  cpl_frameset_find(frames, UVES_SCIENCE(true )) != NULL ||
411  cpl_frameset_find(frames, UVES_SCIENCE(false)) != NULL ||
412  cpl_frameset_find(frames, UVES_SCI_EXTND(true )) != NULL ||
413  cpl_frameset_find(frames, UVES_SCI_EXTND(false)) != NULL ||
414  cpl_frameset_find(frames, UVES_SCI_POINT(true )) != NULL ||
415  cpl_frameset_find(frames, UVES_SCI_POINT(false)) != NULL ||
416  cpl_frameset_find(frames, UVES_SCI_SLICER(true )) != NULL ||
417  cpl_frameset_find(frames, UVES_SCI_SLICER(false)) != NULL,
418  CPL_ERROR_DATA_NOT_FOUND, "No %s, %s, %s, %s, %s, %s, %s or %s in frame set",
419  UVES_SCIENCE(true),
420  UVES_SCIENCE(false),
421  UVES_SCI_EXTND(true),
422  UVES_SCI_EXTND(false),
423  UVES_SCI_POINT(true),
424  UVES_SCI_POINT(false),
425  UVES_SCI_SLICER(true),
426  UVES_SCI_SLICER(false));
427 
428  blue_frames = cpl_frameset_new();
429  red_frames = cpl_frameset_new();
430  common_frames = cpl_frameset_new();
431 
432  check( uves_get_parameter(parameters, NULL, "uves", "process_chip", CPL_TYPE_STRING, &PROCESS_CHIP),
433  "Could not read parameter");
434  uves_string_toupper((char*)PROCESS_CHIP);
435 
436  /* Split in blue/red frames */
437  {
438  cpl_frame *f = NULL;
439 
440  for (f = cpl_frameset_get_first(frames);
441  f != NULL;
442  f = cpl_frameset_get_next(frames))
443  {
444 
445 
446  if (frame_is_needed(true, f)) /* Used in blue arm? */
447  {
448  uves_msg_debug("Found blue frame: '%s'", cpl_frame_get_tag(f));
449  check( cpl_frameset_insert(blue_frames, cpl_frame_duplicate(f)),
450  "Error extracting frame '%s' from frame set",
451  cpl_frame_get_tag(f));
452  }
453  if (frame_is_needed(false, f)) /* Used in red arm? */
454  {
455  uves_msg_debug("Found red frame: '%s'", cpl_frame_get_tag(f));
456  check( cpl_frameset_insert(red_frames, cpl_frame_duplicate(f)),
457  "Error extracting frame '%s' from frame set",
458  cpl_frame_get_tag(f));
459  }
460 
461  if (frame_is_needed(true, f) &&
462  frame_is_needed(false, f)) /* Used in both arms? */
463  {
464  uves_msg_debug("Found common frame: '%s'", cpl_frame_get_tag(f));
465  check( cpl_frameset_insert(common_frames, cpl_frame_duplicate(f)),
466  "Error extracting frame '%s' from frame set",
467  cpl_frame_get_tag(f));
468  }
469 
470  }
471 
472  /* Remove all frames from input frame set */
473  while ((f = cpl_frameset_get_first(frames)) != NULL)
474  {
475  cpl_frameset_erase_frame(frames, f);
476  }
477  }
478 
479  /* Algorithm:
480  (with purpose of failing early if we have to fail.)
481 
482  1) Find out which recipes to run
483  2) Check for necessary input frames
484  3) Execute
485  */
486 
487  blue = true;
488  do {
489  enum uves_chip chip1 = (blue) ? UVES_CHIP_BLUE : UVES_CHIP_REDL;
490  enum uves_chip chip2 = (blue) ? UVES_CHIP_BLUE : UVES_CHIP_REDU;
491 
492  cpl_frameset *fms = (blue) ? blue_frames : red_frames;
493 
494  nraw_arm[blue] =
495  cpl_frameset_get_size(fms) >
496  cpl_frameset_get_size(common_frames);
497 
498  uves_msg_debug("nraw_arm=%d (%s arm)", nraw_arm[blue], blue ? "blue" : "red");
499 
500  run_scired[blue] = do_science &&
501  !(is_missing(fms, UVES_SCIENCE(blue), NULL) &&
502  is_missing(fms, UVES_SCI_EXTND(blue), NULL) &&
503  is_missing(fms, UVES_SCI_POINT(blue), NULL) &&
504  is_missing(fms, UVES_SCI_SLICER(blue), NULL));
505 
506  /* If calibrations must be produced for this arm */
507  if (run_scired[blue]
508  ||
509  (!do_science && nraw_arm[blue])) {
510 
511  /* Require master bias */
512  run_mbias[blue] = is_missing(fms,
513  UVES_MASTER_BIAS(chip1),
514  UVES_MASTER_BIAS(chip2)
515  );
516 
517  /* Run master dark, only if raw frames are available */
518  run_mdark[blue] =
519  is_missing(fms,
520  UVES_MASTER_DARK(chip1),
521  UVES_MASTER_DARK(chip2))
522  &&
523  is_missing(fms,
524  UVES_MASTER_PDARK(chip1),
525  UVES_MASTER_PDARK(chip2))
526  &&(
527  !is_missing(fms, UVES_DARK(blue), NULL) ||
528  !is_missing(fms, UVES_PDARK(blue), NULL));
529 
530  /* Run orderpos if either order table is missing,
531  or raw frame available */
532  run_orderpos[blue] = is_missing(fms,
533  UVES_ORDER_TABLE(flames, chip1),
534  UVES_ORDER_TABLE(flames, chip2)
535  ) ||
536  !is_missing(fms, UVES_ORDER_FLAT(flames, blue), NULL);
537 
538  /* Run master flat recipe if master flat frame is missing */
539  run_mflat[blue] =
540  is_missing(fms,
541  UVES_MASTER_FLAT(chip1),
542  UVES_MASTER_FLAT(chip2))
543  &&
544  is_missing(fms,
545  UVES_MASTER_DFLAT(chip1),
546  UVES_MASTER_DFLAT(chip2))
547  &&
548  is_missing(fms,
549  UVES_MASTER_IFLAT(chip1),
550  UVES_MASTER_IFLAT(chip2))
551  &&
552  is_missing(fms,
553  UVES_MASTER_SCREEN_FLAT(chip1),
554  UVES_MASTER_SCREEN_FLAT(chip2))
555  &&
556  is_missing(fms,
557  UVES_REF_TFLAT(chip1),
558  UVES_REF_TFLAT(chip2));
559 
560 
561  /* Line tables are used as both input and output
562  for wavecal recipe. A provided line table is
563  interpreted as an input table if an arc lamp
564  frame is also available, otherwise as output.
565  Line tables produce by the physmod recipe are
566  input tables. The logic is
567 
568  if !linetable
569  physmod=yes
570  wavecal=yes
571  if linetable
572  physmod=no
573  if !arclamp
574  wavecal=no // line table is final
575  if arclamp
576  wavecal=yes // line table is guess
577  */
578 
579  /* Run physical model if there's no
580  line table */
581  run_physmod[blue] = is_missing(fms,
582  UVES_LINE_TABLE(flames, chip1),
583  UVES_LINE_TABLE(flames, chip2))
584  &&
585  is_missing(fms,
586  UVES_GUESS_LINE_TABLE(flames, chip1),
587  UVES_GUESS_LINE_TABLE(flames, chip2))
588  &&
589  (
590  is_missing(fms,
591  UVES_LINE_TABLE_MIDAS(chip1, 1),
592  UVES_LINE_TABLE_MIDAS(chip2, 1)) ||
593  is_missing(fms,
594  UVES_LINE_TABLE_MIDAS(chip1, 2),
595  UVES_LINE_TABLE_MIDAS(chip2, 2)) ||
596  is_missing(fms,
597  UVES_LINE_TABLE_MIDAS(chip1, 3),
598  UVES_LINE_TABLE_MIDAS(chip2, 3))
599  );
600 
601  /* Run wavecal if no line table,
602  or if there's an arc lamp frame
603  */
604  run_wavecal[blue] =
605  run_physmod[blue]
606  ||
607  (
608  is_missing(fms,
609  UVES_LINE_TABLE(flames, chip1),
610  UVES_LINE_TABLE(flames, chip2))
611  &&
612  (
613  is_missing(fms,
614  UVES_LINE_TABLE_MIDAS(chip1, 1),
615  UVES_LINE_TABLE_MIDAS(chip2, 1)) ||
616  is_missing(fms,
617  UVES_LINE_TABLE_MIDAS(chip1, 2),
618  UVES_LINE_TABLE_MIDAS(chip2, 2)) ||
619  is_missing(fms,
620  UVES_LINE_TABLE_MIDAS(chip1, 3),
621  UVES_LINE_TABLE_MIDAS(chip2, 3))
622  )
623  )
624  ||
625  (
626  !is_missing(fms,
627  UVES_ARC_LAMP(flames, blue), NULL) ||
628  !is_missing(fms,
629  UVES_ECH_ARC_LAMP(blue), NULL)
630  );
631 
632  /* Run response only if there's a standard star.
633  Otherwise no response correction is done */
634  run_response[blue] = !is_missing(fms,
635  UVES_STD_STAR(blue), NULL);
636 
637 
638  uves_msg("Reduction strategy for %s arm:", (blue) ? "BLUE" : "RED");
639  uves_msg("Run %-13s: %s", make_str(UVES_MBIAS_ID) , (run_mbias[blue] ) ? "Yes" : "No");
640  uves_msg("Run %-13s: %s", make_str(UVES_MDARK_ID) , (run_mdark[blue] ) ? "Yes" : "No");
641  uves_msg("Run %-13s: %s", make_str(UVES_PHYSMOD_ID) , (run_physmod[blue] ) ? "Yes" : "No");
642  uves_msg("Run %-13s: %s", make_str(UVES_ORDERPOS_ID), (run_orderpos[blue]) ? "Yes" : "No");
643  uves_msg("Run %-13s: %s", make_str(UVES_MFLAT_ID) , (run_mflat[blue] ) ? "Yes" : "No");
644  uves_msg("Run %-13s: %s", make_str(UVES_WAVECAL_ID) , (run_wavecal[blue] ) ? "Yes" : "No");
645  uves_msg("Run %-13s: %s", make_str(UVES_RESPONSE_ID), (run_response[blue]) ? "Yes" : "No");
646  uves_msg("Run %-13s: %s", make_str(UVES_SCIRED_ID) , (run_scired[blue] ) ? "Yes" : "No");
647 
648  } /* if reduce this arm */
649  else {
650  uves_msg("Skipping %s arm",
651  (blue) ? "BLUE" : "RED");
652 
653  run_mbias[blue] = false;
654  run_mdark[blue] = false;
655  run_mflat[blue] = false;
656  run_physmod[blue] = false;
657  run_orderpos[blue] = false;
658  run_wavecal[blue] = false;
659  run_response[blue] = false;
660  }
661 
662  blue = !blue;
663  }
664  while (!blue);
665 
666  /* As a service to the user, assure that required
667  raw frames and catalogue calibration frames
668  exist *before* doing the reduction */
669 
670  blue = true;
671  do
672  {
673  cpl_frameset *fms = (blue) ? blue_frames : red_frames;
674 
675  assure( !run_mbias[blue] || !is_missing(fms, UVES_BIAS(blue), NULL),
676  CPL_ERROR_DATA_NOT_FOUND,
677  "One or more '%s' frames needed for recipe '%s'",
678  UVES_BIAS(blue), make_str(UVES_MBIAS_ID));
679 
680  assure( !run_mdark[blue] ||
681  !is_missing(fms, UVES_DARK(blue), NULL) ||
682  !is_missing(fms, UVES_PDARK(blue), NULL),
683  CPL_ERROR_DATA_NOT_FOUND,
684  "One or more '%s' or '%s' frames needed for recipe '%s'",
685  UVES_DARK(blue), UVES_PDARK(blue), make_str(UVES_MDARK_ID));
686 
687  assure( !run_physmod[blue] || !is_missing(fms, UVES_FORMATCHECK(flames, blue), NULL),
688  CPL_ERROR_DATA_NOT_FOUND, "Frame '%s' needed for recipe '%s'",
689  UVES_FORMATCHECK(flames, blue), make_str(UVES_PHYSMOD_ID));
690 
691  assure( !run_orderpos[blue] || !is_missing(fms, UVES_ORDER_FLAT(flames, blue), NULL),
692  CPL_ERROR_DATA_NOT_FOUND, "Frame '%s' needed for recipe '%s'",
693  UVES_ORDER_FLAT(flames, blue), make_str(UVES_ORDERPOS_ID));
694 
695  assure( !run_mflat[blue] ||
696  !is_missing(fms, UVES_FLAT(blue), NULL) ||
697  !is_missing(fms, UVES_IFLAT(blue), NULL) ||
698  !is_missing(fms, UVES_SCREEN_FLAT(blue), NULL) ||
699  !is_missing(fms, UVES_DFLAT(blue), NULL) ||
700  !is_missing(fms, UVES_TFLAT(blue), NULL),
701  CPL_ERROR_DATA_NOT_FOUND,
702  "One or more '%s', '%s', '%s', '%s' or '%s' frames needed for recipe '%s'",
703  UVES_FLAT(blue),
704  UVES_IFLAT(blue),
705  UVES_SCREEN_FLAT(blue),
706  UVES_DFLAT(blue),
707  UVES_TFLAT(blue),
708  make_str(UVES_MFLAT_ID));
709 
710  assure( !run_wavecal[blue] || (
711  !is_missing(fms, UVES_ARC_LAMP(flames, blue), NULL) ||
712  !is_missing(fms, UVES_ECH_ARC_LAMP(blue), NULL)),
713  CPL_ERROR_DATA_NOT_FOUND, "Frame '%s' or '%s' needed for recipe '%s'",
714  UVES_ARC_LAMP(flames, blue), UVES_ECH_ARC_LAMP(blue), make_str(UVES_WAVECAL_ID));
715  assure( !run_wavecal[blue] || !is_missing(fms, UVES_LINE_REFER_TABLE, NULL),
716  CPL_ERROR_DATA_NOT_FOUND, "Frame '%s' needed for recipe '%s'",
717  UVES_LINE_REFER_TABLE, make_str(UVES_WAVECAL_ID));
718 
719  assure( !run_response[blue] || !is_missing(fms, UVES_STD_STAR(blue), NULL),
720  CPL_ERROR_DATA_NOT_FOUND, "Frame '%s' needed for recipe '%s'",
721  UVES_STD_STAR(blue), make_str(UVES_RESPONSE_ID));
722  assure( !run_response[blue] || !is_missing(fms, UVES_FLUX_STD_TABLE, NULL),
723  CPL_ERROR_DATA_NOT_FOUND, "Frame '%s' needed for recipe '%s'",
724  UVES_FLUX_STD_TABLE, make_str(UVES_RESPONSE_ID));
725  assure( !run_response[blue] || !is_missing(fms, UVES_EXTCOEFF_TABLE, NULL),
726  CPL_ERROR_DATA_NOT_FOUND, "Frame '%s' needed for recipe '%s'",
727  UVES_EXTCOEFF_TABLE, make_str(UVES_RESPONSE_ID));
728 
729  blue = !blue;
730  }
731  while (!blue);
732 
733  /* We now know which recipes to run and
734  * that required input frames exist. Execute
735  * chain; re-classify PRODUCT->CALIB under way
736  */
737 
738  blue = true;
739  do
740  {
741  enum uves_chip chip1 = (blue) ? UVES_CHIP_BLUE : UVES_CHIP_REDL;
742  enum uves_chip chip2 = (blue) ? UVES_CHIP_BLUE : UVES_CHIP_REDU;
743 
744  cpl_frameset *fms = (blue) ? blue_frames : red_frames;
745 
746  if (run_mbias[blue])
747  {
748  const char *products[2];
749 
750  int nprod = sizeof(products) / sizeof (char *);
751 
752  products[0] = UVES_MASTER_BIAS(chip1);
753  products[1] = UVES_MASTER_BIAS(chip2);
754 
755  if (blue) nprod /= 2;
756 
757  check( execute_recipe(make_str(UVES_MBIAS_ID), fms, parameters, products, nprod, true),
758  "Recipe execution failed");
759  }
760 
761  check( remove_input_frame(fms, UVES_BIAS(blue)), "Error removing input frames");
762 
763  if (run_mdark[blue])
764  {
765  const char *products[4];
766 
767  int nprod = sizeof(products) / sizeof (char *);
768 
769  products[0] = UVES_MASTER_DARK(chip1);
770  products[1] = UVES_MASTER_PDARK(chip1);
771  products[2] = UVES_MASTER_DARK(chip2);
772  products[3] = UVES_MASTER_PDARK(chip2);
773 
774  if (blue) nprod /= 2;
775 
776  check( execute_recipe(
777  make_str(UVES_MDARK_ID), fms, parameters, products, nprod, true),
778  "Recipe execution failed");
779  }
780 
781  check( remove_input_frame(fms, UVES_DARK(blue)), "Error removing input frames");
782  check( remove_input_frame(fms, UVES_PDARK(blue)), "Error removing input frames");
783 
784  if (run_physmod[blue])
785  {
786  const char *products[4];
787  int nprod = sizeof(products) / sizeof (char *);
788 
789  products[0] = UVES_GUESS_LINE_TABLE (flames, chip1);
790  products[1] = UVES_GUESS_ORDER_TABLE(flames, chip1);
791  products[2] = UVES_GUESS_LINE_TABLE (flames, chip2);
792  products[3] = UVES_GUESS_ORDER_TABLE(flames, chip2);
793 
794  if (blue) nprod /= 2;
795 
796  check( execute_recipe(
797  make_str(UVES_PHYSMOD_ID),
798  fms, parameters, products, nprod, true),
799  "Recipe execution failed");
800  }
801 
802  check( remove_input_frame(fms, UVES_FORMATCHECK(flames, blue)),
803  "Error removing input frames");
804 
805  if (run_orderpos[blue])
806  {
807  const char *products[2];
808  int nprod = sizeof(products) / sizeof (char *);
809 
810  products[0] = UVES_ORDER_TABLE(flames, chip1);
811  products[1] = UVES_ORDER_TABLE(flames, chip2);
812 
813  if (blue) nprod /= 2;
814 
815  check( execute_recipe(
816  make_str(UVES_ORDERPOS_ID),
817  fms, parameters, products, nprod, true),
818  "Recipe execution failed");
819  }
820 
821  check( remove_input_frame(fms, UVES_ORDER_FLAT(flames, blue)),
822  "Error removing input frames");
823 
824  if (run_mflat[blue])
825  {
826  const char *products[10];
827 
828  int nprod = sizeof(products) / sizeof (char *);
829 
830  products[0] = UVES_MASTER_FLAT(chip1);
831  products[1] = UVES_MASTER_DFLAT(chip1);
832  products[2] = UVES_MASTER_IFLAT(chip1);
833  products[3] = UVES_MASTER_TFLAT(chip1);
834  products[4] = UVES_MASTER_SCREEN_FLAT(chip1);
835  products[5] = UVES_MASTER_FLAT(chip2);
836  products[6] = UVES_MASTER_DFLAT(chip2);
837  products[7] = UVES_MASTER_IFLAT(chip2);
838  products[8] = UVES_MASTER_TFLAT(chip2);
839  products[9] = UVES_MASTER_SCREEN_FLAT(chip2);
840 
841  if (blue) nprod /= 2;
842 
843  check( execute_recipe(make_str(UVES_MFLAT_ID),
844  fms, parameters, products, nprod, true),
845  "Recipe execution failed");
846  }
847 
848  check( remove_input_frame(fms, UVES_FLAT(blue)), "Error removing input frames");
849  check( remove_input_frame(fms, UVES_IFLAT(blue)), "Error removing input frames");
850  check( remove_input_frame(fms, UVES_DFLAT(blue)), "Error removing input frames");
851  check( remove_input_frame(fms, UVES_TFLAT(blue)), "Error removing input frames");
852  check( remove_input_frame(fms, UVES_SCREEN_FLAT(blue)), "Error removing input frames");
853 
854  if (run_wavecal[blue])
855  {
856  const char *products[2];
857 
858  int nprod = sizeof(products) / sizeof (char *);
859 
860  products[0] = UVES_LINE_TABLE(flames, chip1);
861  products[1] = UVES_LINE_TABLE(flames, chip2);
862 
863  if (blue) nprod /= 2;
864 
865  check( execute_recipe(make_str(UVES_WAVECAL_ID),
866  fms, parameters, products, nprod, true),
867  "Recipe execution failed");
868  }
869 
870  check( remove_input_frame(fms, UVES_ARC_LAMP(flames, blue)),
871  "Error removing input frames");
872  check( remove_input_frame(fms, UVES_ECH_ARC_LAMP(blue)),
873  "Error removing input frames");
874  check( remove_input_frame(fms, UVES_LINE_REFER_TABLE),
875  "Error removing input frames");
876 
877  if (run_response[blue])
878  {
879  const char *products[2];
880 
881  int nprod = sizeof(products) / sizeof (char *);
882 
883  products[0] = UVES_INSTR_RESPONSE(chip1);
884  products[1] = UVES_INSTR_RESPONSE(chip2);
885 
886  if (blue) nprod /= 2;
887 
888  check( execute_recipe(make_str(UVES_RESPONSE_ID),
889  fms, parameters, products, nprod, true),
890  "Recipe execution failed");
891  }
892 
893  check( remove_input_frame(fms, UVES_STD_STAR(blue)), "Error removing input frames");
894  check( remove_input_frame(fms, UVES_FLUX_STD_TABLE), "Error removing input frames");
895 
896  if (run_scired[blue])
897  {
898  const char *products[2];
899 
900  int nprod = sizeof(products) / sizeof (char *);
901 
902  products[0] = blue ? "RED_SCIENCE_BLUE" : "RED_SCIENCE_REDL";
903  products[1] = blue ? "RED_SCIENCE_BLUE" : "RED_SCIENCE_REDU";
904 
905  if (blue) nprod /= 2;
906 
907  check( execute_recipe(make_str(UVES_SCIRED_ID),
908  fms, parameters, products, nprod, false),
909  "Recipe execution failed");
910  }
911 
912  check( remove_input_frame(fms, UVES_SCIENCE(blue)) , "Error removing input frames");
913  check( remove_input_frame(fms, UVES_SCI_EXTND(blue)) , "Error removing input frames");
914  check( remove_input_frame(fms, UVES_SCI_POINT(blue)) , "Error removing input frames");
915  check( remove_input_frame(fms, UVES_SCI_SLICER(blue)), "Error removing input frames");
916 
917  /* Insert all product frames into recipe frame set */
918  {
919  cpl_frame *f = NULL;
920 
921  for (f = cpl_frameset_get_first(fms);
922  f != NULL;
923  f = cpl_frameset_get_next(fms))
924  {
925  if (cpl_frame_get_group(f) == CPL_FRAME_GROUP_PRODUCT)
926  {
927  check( cpl_frameset_insert(frames, cpl_frame_duplicate(f)),
928  "Error inserting product '%s' into frame set",
929  cpl_frame_get_tag(f));
930  }
931  }
932  }
933 
934  blue = !blue;
935  }
936  while(!blue); /* For each arm */
937 
938  cleanup:
939  uves_free_frameset(&blue_frames);
940  uves_free_frameset(&red_frames);
941  uves_free_frameset(&common_frames);
942 
943  return;
944 }
945 
946 /* Returns true, iff frame is used for blue/red arm
947  (note that some frames like UVES_FLUX_STD_TABLE are
948  used for both arms) */
949 static bool
950 frame_is_needed(bool blue, const cpl_frame *f)
951 {
952  const char *tag = cpl_frame_get_tag(f);
953 
954  bool result = (strcmp(tag, UVES_ORDER_FLAT (flames, blue)) == 0 ||
955  strcmp(tag, UVES_BIAS (blue)) == 0 ||
956  strcmp(tag, UVES_DARK (blue)) == 0 ||
957  strcmp(tag, UVES_PDARK (blue)) == 0 ||
958  strcmp(tag, UVES_FLAT (blue)) == 0 ||
959  strcmp(tag, UVES_IFLAT (blue)) == 0 ||
960  strcmp(tag, UVES_DFLAT (blue)) == 0 ||
961  strcmp(tag, UVES_TFLAT (blue)) == 0 ||
962  strcmp(tag, UVES_SCREEN_FLAT(blue)) == 0 ||
963  strcmp(tag, UVES_STD_STAR (blue)) == 0 ||
964  strcmp(tag, UVES_FORMATCHECK(flames, blue)) == 0 ||
965  strcmp(tag, UVES_STD_STAR (blue)) == 0 ||
966  strcmp(tag, UVES_SCIENCE (blue)) == 0 ||
967  strcmp(tag, UVES_SCI_EXTND (blue)) == 0 ||
968  strcmp(tag, UVES_SCI_POINT (blue)) == 0 ||
969  strcmp(tag, UVES_SCI_SLICER (blue)) == 0 ||
970  strcmp(tag, UVES_ARC_LAMP (flames, blue)) == 0 ||
971  strcmp(tag, UVES_ECH_ARC_LAMP(blue)) == 0);
972 
973  enum uves_chip chip;
974 
975  /* Loop through all blue or red chips (1 or 2) */
976  for (chip = uves_chip_get_first(blue);
977  chip != UVES_CHIP_INVALID;
978  chip = uves_chip_get_next(chip))
979  {
980  result = result || (strcmp(tag, UVES_DRS_SETUP(flames, chip)) == 0 ||
981  strcmp(tag, UVES_ORDER_TABLE(flames, chip)) == 0 ||
982  strcmp(tag, UVES_GUESS_ORDER_TABLE(flames, chip)) == 0 ||
983  strcmp(tag, UVES_MASTER_BIAS(chip)) == 0 ||
984  strcmp(tag, UVES_MASTER_DARK(chip)) == 0 ||
985  strcmp(tag, UVES_MASTER_PDARK(chip)) == 0 ||
986  strcmp(tag, UVES_MASTER_FLAT(chip)) == 0 ||
987  strcmp(tag, UVES_MASTER_DFLAT(chip)) == 0 ||
988  strcmp(tag, UVES_MASTER_IFLAT(chip)) == 0 ||
989  strcmp(tag, UVES_MASTER_TFLAT(chip)) == 0 ||
990  strcmp(tag, UVES_REF_TFLAT(chip)) == 0 ||
991  strcmp(tag, UVES_MASTER_SCREEN_FLAT(chip)) == 0 ||
992  strcmp(tag, UVES_LINE_TABLE (flames, chip)) == 0 ||
993  strcmp(tag, UVES_GUESS_LINE_TABLE(flames, chip)) == 0 ||
994  strcmp(tag, UVES_LINE_TABLE_MIDAS(chip, 1)) == 0 ||
995  strcmp(tag, UVES_LINE_TABLE_MIDAS(chip, 2)) == 0 ||
996  strcmp(tag, UVES_LINE_TABLE_MIDAS(chip, 3)) == 0 ||
997  strcmp(tag, UVES_LINE_REFER_TABLE ) == 0 ||
998  strcmp(tag, UVES_FLUX_STD_TABLE ) == 0 ||
999  strcmp(tag, UVES_EXTCOEFF_TABLE ) == 0);
1000  }
1001  return result;
1002 }
1003 
1004 /* Execute a recipe and re-classify its products as calibration frames */
1005 static cpl_error_code
1006 execute_recipe(const char *recipe_id,
1007  cpl_frameset *frames, const cpl_parameterlist *parameters,
1008  const char *products[],
1009  int n_products,
1010  bool reclassify) /* Re-classify products? */
1011 {
1012  int i;
1013  cpl_frame *f = NULL;
1014 
1015  /* Remove (from frame set) any product
1016  frames already present */
1017  for (i = 0; i < n_products; i++)
1018  {
1019  if ((f = cpl_frameset_find(frames, products[i])) != NULL)
1020  {
1021  if (cpl_frame_get_group(f) == CPL_FRAME_GROUP_PRODUCT)
1022  {
1023  cpl_msg_warning(__func__, "Ignoring %s frame in '%s'. "
1024  "A new %s frame will now be calculated",
1025  products[i], cpl_frame_get_filename(f),
1026  products[i]);
1027 
1028  cpl_frameset_erase_frame(frames, f);
1029  }
1030  }
1031  }
1032 
1033  /* Execute */
1034  check( uves_invoke_recipe(recipe_id, parameters, frames, make_str(UVES_REDCHAIN_ID), NULL),
1035  "Recipe '%s' failed", recipe_id);
1036 
1037  check(cpl_dfs_update_product_header(frames),"Error updating pipe products' header");
1038  if (reclassify)
1039  {
1040  /* Now re-classify PRODUCT->CALIB to be used in the remaining
1041  reduction chain. Before doing that, we have to remove any
1042  calibration frame with same tag as a product (such as line tables),
1043  in order not to confuse the re-classified products with the
1044  previous calibration frames */
1045 
1046  for (i = 0; i < n_products; i++)
1047  {
1048  if ((f = cpl_frameset_find(frames, products[i])) != NULL &&
1049  cpl_frame_get_group(f) != CPL_FRAME_GROUP_PRODUCT)
1050  {
1051  uves_msg("Removing %s frame in '%s' from frameset. "
1052  "It is not tagged as a product",
1053  products[i], cpl_frame_get_filename(f));
1054 
1055  cpl_frameset_erase_frame(frames, f);
1056  }
1057  }
1058 
1059  /*
1060  * Re-classify products
1061  */
1062  for (i = 0; i < n_products; i++)
1063  {
1064  cpl_frame *found = NULL;
1065  for (f = cpl_frameset_get_first(frames);
1066  f != NULL;
1067  f = cpl_frameset_get_next(frames))
1068  {
1069  if (cpl_frame_get_group(f) == CPL_FRAME_GROUP_PRODUCT)
1070  {
1071  if (strcmp(cpl_frame_get_tag(f), products[i]) == 0)
1072  {
1073  found = f;
1074  }
1075  }
1076  }
1077 
1078  if (found != NULL)
1079  {
1080  /* Re-classify the product as calibration frames */
1081  uves_msg("Re-classifying %s product in '%s' PRODUCT->CALIB",
1082  products[i], cpl_frame_get_filename(found));
1083 
1084  cpl_frame_set_group(found, CPL_FRAME_GROUP_CALIB);
1085  }
1086  }
1087 
1088  /*
1089  * Remove other products that
1090  * are not used later (e.g. BKG_FLAT_xxxx)
1091  */
1092  for (f = cpl_frameset_get_first(frames);
1093  f != NULL;
1094  f = cpl_frameset_get_next(frames))
1095  {
1096  if (cpl_frame_get_group(f) == CPL_FRAME_GROUP_PRODUCT)
1097  {
1098  /* Remove this product */
1099  uves_msg("Removing product %s in '%s' from frameset. "
1100  "Not needed later",
1101  cpl_frame_get_tag(f), cpl_frame_get_filename(f));
1102 
1103  cpl_frameset_erase_frame(frames, f);
1104 
1105  }
1106  }
1107  } /* if re-classify... */
1108 
1109  cleanup:
1110  return cpl_error_get_code();
1111 }
1112 
1113 
1114 /* Retruns true if either frame 1 or frame 2 is not in the
1115  provided frame set
1116 
1117  fixme: reverse the logic of this function, i.e. rename to 'contains'
1118 */
1119 static bool
1120 is_missing(const cpl_frameset *frames, const char *frame1, const char *frame2)
1121 {
1122  bool result = false;
1123  if (cpl_frameset_find_const(frames, frame1) == NULL)
1124  {
1125  uves_msg("checking for %s... no", frame1);
1126  result = true;
1127  }
1128  else
1129  {
1130  uves_msg("checking for %s... yes", frame1);
1131  }
1132 
1133  if (frame2 != NULL && strcmp(frame1, frame2) != 0)
1134  {
1135  if (cpl_frameset_find_const(frames, frame2) == NULL)
1136  {
1137  uves_msg("checking for %s... no", frame2);
1138  result = true;
1139  }
1140  else
1141  {
1142  uves_msg("checking for %s... yes", frame2);
1143  }
1144  }
1145 
1146  return result;
1147 }
1148 
1149 
1150 /* Remove input frames (e.g. bias frames) along the way */
1151 static void
1152 remove_input_frame(cpl_frameset *frames, const char *tag)
1153 {
1154  int removed = cpl_frameset_erase(frames, tag);
1155 
1156  if (removed > 0)
1157  {
1158  uves_msg("Removing %d %s frame(s) from frame set", removed, tag);
1159  }
1160 
1161  return;
1162 }