UVES Pipeline Reference Manual  5.4.0
uves_wavecal_firstsolution.c
1 /* *
2  * This file is part of the ESO UVES Pipeline *
3  * Copyright (C) 2004,2005 European Southern Observatory *
4  * *
5  * This library is free software; you can redistribute it and/or modify *
6  * it under the terms of the GNU General Public License as published by *
7  * the Free Software Foundation; either version 2 of the License, or *
8  * (at your option) any later version. *
9  * *
10  * This program is distributed in the hope that it will be useful, *
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13  * GNU General Public License for more details. *
14  * *
15  * You should have received a copy of the GNU General Public License *
16  * along with this program; if not, write to the Free Software *
17  * Foundation, 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA *
18  * */
19 
20 /*
21  * $Author: amodigli $
22  * $Date: 2010-09-24 09:32:09 $
23  * $Revision: 1.23 $
24  * $Name: not supported by cvs2svn $
25  * $Log: not supported by cvs2svn $
26  * Revision 1.21 2007/08/21 13:08:26 jmlarsen
27  * Removed irplib_access module, largely deprecated by CPL-4
28  *
29  * Revision 1.20 2007/06/06 08:17:33 amodigli
30  * replace tab with 4 spaces
31  *
32  * Revision 1.19 2007/05/25 07:05:21 jmlarsen
33  * Decreased warning verbosity
34  *
35  * Revision 1.18 2007/04/26 13:21:04 jmlarsen
36  * Made more robust against inaccurate abs_order polynomial
37  *
38  * Revision 1.17 2007/04/10 07:11:56 jmlarsen
39  * Changed interface of polynomial_regression_2d()
40  *
41  * Revision 1.16 2007/03/05 10:22:24 jmlarsen
42  * Fixed bug in computation of max/min physical order number
43  *
44  * Revision 1.15 2007/01/15 08:58:20 jmlarsen
45  * More robust polynomial fitting
46  *
47  * Revision 1.14 2007/01/10 12:40:12 jmlarsen
48  * Removed unused parameter
49  *
50  * Revision 1.13 2006/12/07 08:29:58 jmlarsen
51  * Compute correct Ynew column for FLAMES
52  *
53  * Revision 1.12 2006/11/24 16:24:32 jmlarsen
54  * Added check of abs order polynomial
55  *
56  * Revision 1.11 2006/11/15 15:02:15 jmlarsen
57  * Implemented const safe workarounds for CPL functions
58  *
59  * Revision 1.9 2006/11/15 14:04:08 jmlarsen
60  * Removed non-const version of parameterlist_get_first/last/next which is
61  * already in CPL, added const-safe wrapper, unwrapper and deallocator functions
62  *
63  * Revision 1.8 2006/11/06 15:19:42 jmlarsen
64  * Removed unused include directives
65  *
66  * Revision 1.7 2006/08/18 07:07:43 jmlarsen
67  * Switched order of cpl_calloc arguments
68  *
69  * Revision 1.6 2006/07/14 12:43:47 jmlarsen
70  * Documentation update
71  *
72  * Revision 1.5 2006/07/03 13:29:45 jmlarsen
73  * Reduced max line length
74  *
75  * Revision 1.4 2006/03/03 13:54:11 jmlarsen
76  * Changed syntax of check macro
77  *
78  * Revision 1.3 2006/02/15 13:19:15 jmlarsen
79  * Reduced source code max. line length
80  *
81  * Revision 1.2 2006/02/08 09:25:05 jmlarsen
82  * Fixed bug caused by == comparison of doubles
83  *
84  * Revision 1.1 2006/02/03 07:46:30 jmlarsen
85  * Moved recipe implementations to ./uves directory
86  *
87  * Revision 1.27 2005/12/20 08:11:44 jmlarsen
88  * Added CVS entry
89  *
90  */
91 /*----------------------------------------------------------------------------*/
92 /*
93  * @addtogroup uves_wavecal
94  */
95 /*----------------------------------------------------------------------------*/
98 #ifdef HAVE_CONFIG_H
99 # include <config.h>
100 #endif
101 
102 #include <uves_wavecal_firstsolution.h>
103 
104 #include <uves_utils.h>
105 #include <uves_utils_wrappers.h>
106 #include <uves_dump.h>
107 #include <uves_error.h>
108 #include <uves_msg.h>
109 
110 #include <cpl.h>
111 
112 #include <math.h>
113 
114 static int *
115 write_physical_order(cpl_table *linetable,
116  const polynomial *absolute_order,
117  const cpl_table *ordertable,
118  const polynomial *order_locations,
119  int *first_abs_order, int *last_abs_order);
120 
121 static double
122 calculate_shift(const cpl_table *linetable, const cpl_table *previous,
123  const char *column, const char *reference_column,
124  double range, double step, double tolerance);
125 
126 static double
127 cross_correlation(double shift,
128  const cpl_table *t1, const cpl_table *t2,
129  const char *column, const char* reference_column,
130  int minref, int maxref, double tolerance);
131 
132 static polynomial *apply_shift(const cpl_table *previous,
133  const double shift, const int degree, double *mse);
134 
135 /*----------------------------------------------------------------------------*/
182 /*----------------------------------------------------------------------------*/
183 polynomial *
184 uves_wavecal_firstsolution(cpl_table *linetable,
185  const cpl_table *guess,
186  polynomial **absolute_order,
187  const cpl_table *ordertable,
188  const polynomial *order_locations,
189  bool flames,
190  double offset,
191  int **relative_order,
192  int DEGREE, double CORREL_RANGE, double CORREL_STEP,
193  double CORREL_TOLERANCE, double MAXERROR,
194  int *first_abs_order, int *last_abs_order)
195 {
196  polynomial *initial_dispersion = NULL;
197  polynomial *new_absorder = NULL;
198  const char *er_msg = NULL;
199  double shift;
200  double mse;
201 
202  /* Get physical order numbering */
203  check( *relative_order = write_physical_order(linetable, *absolute_order,
204  ordertable,
205  order_locations,
206  first_abs_order,
207  last_abs_order),
208  "Could not calculate absolute order numbers");
209 
210  /* Update the 'absolute_order' map */
211  {
212  int row;
213 
214  /* Create column for Y-location (in pixels) of order */
215  cpl_table_new_column(linetable, "Ynew", CPL_TYPE_DOUBLE);
216  for (row = 0; row < cpl_table_get_nrow(linetable); row++)
217  {
218  /* For historical reasons, the column 'Y' contains the
219  (relative) order number while 'Ynew' contains
220  the y-coordinate (in pixels) of the emission line. */
221  int order = cpl_table_get_int (linetable, "Y", row, NULL);
222  double x = cpl_table_get_double(linetable, "X", row, NULL);
223 
224  cpl_table_set_double(
225  linetable, "Ynew", row,
226  uves_polynomial_evaluate_2d(order_locations, x, order));
227  }
228 
229  assure_nomsg( cpl_error_get_code() == CPL_ERROR_NONE,
230  cpl_error_get_code() );
231 
232  new_absorder =
233  uves_polynomial_regression_2d(linetable, "X", "Ynew", "Order",
234  NULL, /* uncertainty of order number */
235  DEGREE, DEGREE,
236  NULL, NULL, NULL, /* New columns */
237  NULL, NULL, /* mse, chi^2 */
238  NULL, /* variance pol. */
239  -1, -1); /* kappa */
240 
241  if (cpl_error_get_code() != CPL_ERROR_NONE) /* Singular matrix, or too few points */
242  {
243  er_msg = uves_sprintf("%s", cpl_error_get_message());
244 
246  uves_msg_warning("Could not make global fit of absolute order number (%s). "
247  "Polynomial is not updated",
248  er_msg);
249  }
250  else
251  {
252  uves_polynomial_delete(absolute_order);
253  *absolute_order = uves_polynomial_duplicate(new_absorder);
254  }
255 
256  /* Calculate absolute_order wrt center of orders, but add offset to Ynew column */
257  if (flames)
258  {
259  cpl_table_add_scalar(linetable, "Ynew", + offset);
260  }
261  }
262 
263  /* Sort linetable by 'Order' (ascending), then 'X' (ascending) */
264  uves_sort_table_2(linetable, "Order", "X", false, false);
265 
266  /* Cross correlation of guess (linetable) and linetable */
267  /* Step size should not be less than 2*tolerance */
268  check( shift = calculate_shift(guess, linetable, "X", "Order",
269  CORREL_RANGE, CORREL_STEP, CORREL_TOLERANCE),
270  "Could not calculate shift of position w.r.t. guess solution");
271 
272  /* Apply shift to guess solution
273  * Note that it doesn't help to simply call uves_polynomial_shift()
274  * on the guess solution
275  * because the requested 'DEGREE' might be different from
276  * the degree used in the guess solution
277  */
278 
279  check( initial_dispersion = apply_shift(guess, shift, DEGREE, &mse),
280  "Could not calculate initial dispersion relation");
281  /* This fit may fail if the input guess table has too few or badly
282  distributed points, but there is not much to do about that */
283 
284  /* Check quality of initial solution */
285  if(mse > MAXERROR*MAXERROR)
286  {
287  uves_msg_warning("RMS of initial fit (%f pixels) is greater "
288  "than tolerance (%f pixels)", sqrt(mse), MAXERROR);
289  }
290 
291  cleanup:
292  uves_free_string_const(&er_msg);
293  uves_polynomial_delete(&new_absorder);
294  if (cpl_error_get_code() != CPL_ERROR_NONE)
295  {
296  uves_polynomial_delete(&initial_dispersion);
297  }
298 
299  return initial_dispersion;
300 }
301 
302 /*----------------------------------------------------------------------------*/
317 /*----------------------------------------------------------------------------*/
318 static polynomial *
319 apply_shift(const cpl_table *guess, double shift, int degree, double *mse)
320 {
321  polynomial *result = NULL;
322  cpl_table *t = NULL;
323 
324  /* Copy guess table */
325  check( t = cpl_table_duplicate(guess),
326  "Error duplicating table");
327 
328  /* Create auxillary column Ident*Order */
329  check(( cpl_table_duplicate_column(t, "ident_order", t, "Ident"),
330  cpl_table_multiply_columns(t, "ident_order", "Order")),
331  /* ident_order = Ident * Order */
332  "Error creating auxillary column");
333 
334  /* Shift x values */
335  check( cpl_table_add_scalar(t, "X", shift), "Error shifting column 'X'");
336 
337  /* Fit lambda*m = f(x, m) */
338  /* Don't use uncertainties because they might not exist in guess solution */
339  result = uves_polynomial_regression_2d(t, "X", "Order", "ident_order", NULL,
340  degree, degree,
341  NULL, NULL, NULL,
342  mse, NULL,
343  NULL, -1, -1);
344 
345  /* If failed, set error to SINGULAR_MATRIX */
346  if (cpl_error_get_code() != CPL_ERROR_NONE) /* Singular matrix or too few points */
347  {
349 
350  assure( false, CPL_ERROR_SINGULAR_MATRIX,
351  "Polynomial fitting failed");
352  }
353 
354  cleanup:
355  uves_free_table(&t);
356  return result;
357 }
358 
359 /*----------------------------------------------------------------------------*/
382 /*----------------------------------------------------------------------------*/
383 
384 static double
385 calculate_shift(const cpl_table *linetable, const cpl_table *guess, const char *column,
386  const char *reference_column, double range, double step, double tolerance)
387 {
388  cpl_type t;
389  int minorder, maxorder;
390  int N, i;
391  double shift, max_corr, median_corr, maxpos = 0;
392  cpl_table *temp = NULL;
393 
394  assure( cpl_table_has_column(linetable, column),
395  CPL_ERROR_ILLEGAL_INPUT, "Table has no '%s' column", column);
396  assure( cpl_table_has_column(guess , column),
397  CPL_ERROR_ILLEGAL_INPUT, "Table has no '%s' column", column);
398  assure( cpl_table_has_column(linetable, reference_column),
399  CPL_ERROR_ILLEGAL_INPUT, "Table has no '%s' column", reference_column);
400  assure( cpl_table_has_column(guess , reference_column),
401  CPL_ERROR_ILLEGAL_INPUT, "Table has no '%s' column", reference_column);
402  assure( range > 0, CPL_ERROR_ILLEGAL_INPUT, "Range = %f", range);
403 
404  t = cpl_table_get_column_type(linetable, column);
405  assure( t == CPL_TYPE_DOUBLE, CPL_ERROR_TYPE_MISMATCH,
406  "Column '%s' has type '%s'. Double expected", column, uves_tostring_cpl_type(t));
407 
408  t = cpl_table_get_column_type(guess, column);
409  assure( t == CPL_TYPE_DOUBLE, CPL_ERROR_TYPE_MISMATCH,
410  "Column '%s' has type '%s'. Double expected", column, uves_tostring_cpl_type(t));
411 
412  t = cpl_table_get_column_type(linetable, reference_column);
413  assure( t == CPL_TYPE_INT, CPL_ERROR_TYPE_MISMATCH,
414  "Ref. column '%s' has type '%s'. Integer expected",
415  reference_column, uves_tostring_cpl_type(t));
416 
417  t = cpl_table_get_column_type(guess, reference_column);
418  assure( t == CPL_TYPE_INT, CPL_ERROR_TYPE_MISMATCH,
419  "Ref. column '%s' has type '%s'. Integer expected",
420  reference_column, uves_tostring_cpl_type(t));
421 
422  /* Identify common orders */
423  check(( minorder =
424  uves_max_int(cpl_table_get_column_min(guess, reference_column),
425  cpl_table_get_column_min(linetable, reference_column)),
426  maxorder =
427  uves_min_int(cpl_table_get_column_max(guess, reference_column),
428  cpl_table_get_column_max(linetable, reference_column))),
429  "Error reading column '%s'", reference_column);
430 
431  assure(maxorder >= minorder, CPL_ERROR_ILLEGAL_INPUT, "No common orders found");
432 
433  uves_msg("Min/max common absolute orders = %d - %d", minorder, maxorder);
434 
435  /* Find maximum of cross correlation function
436  for shifts in [-range ; range]
437  */
438 
439  /* Count number of candidates,
440  so we can create a table of the correct size
441  which is used to get the median of
442  all cross-correlation values */
443  N = 0;
444  for (shift = -range; shift <= range; shift += step)
445  {
446  N += 1;
447  }
448 
449  temp = cpl_table_new(N);
450  cpl_table_new_column(temp, "Corr", CPL_TYPE_DOUBLE);
451 
452  max_corr = -1;
453  maxpos = 0;
454  for (shift = -range, i = 0;
455  i < N;
456  shift += step , i++)
457  {
458  double corr;
459  check( corr = cross_correlation(shift, linetable, guess, column,
460  reference_column, minorder, maxorder, tolerance),
461  "Error calculating spectrum cross correlation for shift = %f pixel(s)",
462  shift);
463 
464  /* Update table */
465  check( cpl_table_set_double(temp, "Corr", i, corr),
466  "Error updating table");
467 
468  uves_msg_debug("Correlation(shift=%f) = %f", shift, corr);
469 
470  if (corr > max_corr)
471  {
472  max_corr = corr;
473  maxpos = shift;
474  }
475  }
476 
477  /* To estimate significance,
478  compare the detected max cross-correlation
479  value to "no correlation" estimated as the
480  median of all cross-corr. values */
481 
482  median_corr = cpl_table_get_column_median(temp, "Corr");
483 
484  /* Correlation value is integer ; don't divide by zero */
485  if (median_corr < 0.5)
486  {
487  median_corr = 1;
488  }
489 
490  uves_msg("Estimated shift compared to guess solution is %f pixels (%.2f sigma detection)",
491  maxpos, max_corr / median_corr);
492 
493  /* The correlation peak is usually
494  ~30 or more times the background,
495  so warn if peak value is less than, say,
496  10 times background. */
497  if (max_corr / median_corr < 10)
498  {
499  uves_msg_warning("Cross-correlation with guess solution is "
500  "only %f times no correlation (usually >30). "
501  "Make sure that the guess solution is within ~10 pixels "
502  "of the real dispersion relation; otherwise the following "
503  "wavelength calibration is likely to fail or converge "
504  "to a wrong solution",
505  max_corr / median_corr);
506  }
507 
508  cleanup:
509  uves_free_table(&temp);
510  return maxpos;
511 }
512 
513 /*----------------------------------------------------------------------------*/
535 /*----------------------------------------------------------------------------*/
536 static double
537 cross_correlation(double shift,
538  const cpl_table *t1, const cpl_table *t2,
539  const char *column, const char* reference_column,
540  int minref, int maxref, double tolerance)
541 {
542  double result = 0; /* The result */
543  int i1 = 0; /* Pointers to table rows */
544  int i2 = 0;
545 
546  /* For efficiency reasons, retrieve the pointers to the columns */
547  const double *col1 = cpl_table_get_data_double_const(t1, column);
548  const double *col2 = cpl_table_get_data_double_const(t2, column);
549  const int *ref1 = cpl_table_get_data_int_const(t1, reference_column);
550  const int *ref2 = cpl_table_get_data_int_const(t2, reference_column);
551 
552  int N1 = cpl_table_get_nrow(t1);
553  int N2 = cpl_table_get_nrow(t2);
554 
555  assure( cpl_error_get_code() == CPL_ERROR_NONE, cpl_error_get_code(),
556  "Error reading input table");
557 
558  /* Search for matching rows */
559  while (i1 < N1 && ref1[i1] <= maxref &&
560  i2 < N2 && ref2[i2] <= maxref) {
561  if (i1 < minref || ref1[i1] < ref2[i2])
562  i1++;
563  else if (i2 < minref || ref1[i1] > ref2[i2])
564  i2++;
565  else {
566  /* Reference values match */
567  double difference = col2[i2] - (col1[i1] + shift);
568 
569  if (difference > tolerance)
570  {
571  i1++;
572  }
573  else if (difference < -tolerance)
574  {
575  i2++;
576  }
577  else {
578  /* Matching rows found: |col2-col1-shift| <= tolerance.
579  Update result and continue search */
580  result += 1.0;
581  i2++;
582  }
583  }
584  }
585 
586 
587  cleanup:
588  return result;
589 }
590 
591 
592 /*----------------------------------------------------------------------------*/
611 /*----------------------------------------------------------------------------*/
612 static int *
613 write_physical_order(cpl_table *linetable, const polynomial *absolute_order,
614  const cpl_table *ordertable,
615  const polynomial *order_locations,
616  int *first_abs_order, int *last_abs_order)
617 {
618  int *relative_order = NULL; /* Result */
619  int *physical_order = NULL;
620  int minorder, maxorder;
621  int maxphysical;
622  cpl_table *temp = NULL;
623  const polynomial *map = NULL;
624 
625  double *sum = NULL; /* Auxillary variables used to calculate the average */
626  int *N = NULL;
627 
628  int i;
629 
630  check( cpl_table_new_column(linetable, "Order", CPL_TYPE_INT),
631  "Error creating column");
632 
633  check( cpl_table_new_column(linetable, "AbsOrder", CPL_TYPE_DOUBLE),
634  "Error creating column");
635 
636  check(( minorder = cpl_table_get_column_min(ordertable, "Order"),
637  maxorder = cpl_table_get_column_max(ordertable, "Order")),
638  "Could not read min. and max. order numbers");
639 
640  assure( minorder > 0, CPL_ERROR_ILLEGAL_INPUT,
641  "Non-positive order number (%d) in linetable", minorder);
642 
643  physical_order = cpl_calloc(maxorder + 1, sizeof(int));
644  assure_mem( physical_order );
645 
646  /* First calculate the estimation of the
647  absolute order number at each line position */
648  for (i = 0; i < cpl_table_get_nrow(linetable); i++) {
649  double x, y;
650  double absorder;
651  int order;
652 
653  order = cpl_table_get_int (linetable, "Y", i, NULL);
654  /* The column 'Y' contains the (relative) order number */
655 
656  x = cpl_table_get_double(linetable, "X", i, NULL);
657 
658  y = uves_polynomial_evaluate_2d(order_locations, x, order);
659 
660  absorder = uves_polynomial_evaluate_2d(absolute_order, x, y);
661 
662  uves_msg_debug("Order #%d: Absolute order = %f at x = %f",
663  order, absorder, x);
664 
665  cpl_table_set_double(linetable, "AbsOrder", i, absorder);
666  }
667 
668  {
669  int degree = 1;
670  int coeff1, coeff2; /* absorder = coeff1 + coeff2 * relative_order */
671  int order;
672  int relorder_median;
673  int absorder_median;
674 
675  check_nomsg( map =
677  "Y", "AbsOrder", NULL,
678  degree,
679  NULL, NULL, NULL, -1));
680 
681  relorder_median = uves_round_double(cpl_table_get_column_median(linetable, "Y"));
682  absorder_median = uves_round_double(uves_polynomial_evaluate_1d(map, relorder_median));
683 
684  if (uves_polynomial_derivative_1d(map, relorder_median) > 0) {
685  coeff2 = 1;
686  }
687  else {
688  coeff2 = -1;
689  }
690 
691  coeff1 = absorder_median - coeff2 * relorder_median;
692 
693  uves_msg_debug("Assuming relation: abs.order = %d + (%d) * rel.order",
694  coeff1, coeff2);
695 
696  maxphysical = -1;
697  for (order = minorder; order <= maxorder; order++) {
698  physical_order[order] = coeff1 + coeff2 * order;
699 
700  assure(physical_order[order] > 0, CPL_ERROR_ILLEGAL_OUTPUT,
701  "Estimated physical order number is non-positive (%d)",
702  physical_order[order]);
703 
704  if (physical_order[order] > maxphysical)
705  {
706  maxphysical = physical_order[order];
707  }
708 
709  uves_msg_debug("Mapping relative order #%d to absolute order #%d",
710  order, physical_order[order]);
711  }
712 
713  /* Get first and last physical orders */
714  *first_abs_order = physical_order[minorder];
715  *last_abs_order = physical_order[maxorder];
716 
717  passure( *first_abs_order - *last_abs_order == coeff2*(minorder - maxorder),
718  "%d %d %d %d %d",
719  *first_abs_order, *last_abs_order, coeff2, minorder, maxorder);
720 
721  }
722 
723  /* Then write this rounded mean value to every row of the table */
724  for (i = 0; i < cpl_table_get_nrow(linetable); i++) {
725  int order;
726  order = cpl_table_get_int (linetable, "Y", i, NULL);
727  cpl_table_set_int(linetable, "Order", i, physical_order[order]);
728  }
729 
730  /* Calculate the inverse of 'physical_order' */
731  relative_order = cpl_calloc(maxphysical + 1, sizeof(int));
732  for (i = 0; i <= maxorder; i++)
733  {
734  relative_order[physical_order[i]] = i;
735  }
736 
737  cleanup:
738  uves_free_table(&temp);
740  cpl_free(sum);
741  cpl_free(physical_order);
742  cpl_free(N);
743 
744  return relative_order;
745 }