MIDI Pipeline Reference Manual  2.8.3
procDetLin.c
1 
2 /******************************************************************************
3 *******************************************************************************
4 * European Southern Observatory
5 * VLTI MIDI Maintenance Templates Software
6 *
7 * Module name: procDetLin.c
8 * Description: Contains routines for processing the templates
9 *
10 * History:
11 * 16-Jun-04 (csabet) created
12 *******************************************************************************
13 ******************************************************************************/
14 
15 /******************************************************************************
16 * Compiler directives
17 ******************************************************************************/
18 
19 /******************************************************************************
20 * Include files
21 ******************************************************************************/
22 #include <math.h>
23 #include <stdio.h>
24 #include <cpl.h>
25 #include "midiGlobal.h"
26 #include "midiLib.h"
27 #include "imageProcessing.h"
28 #include "memoryHandling.h"
29 #include "errorHandling.h"
30 #include "createProdDetLin.h"
31 #include "midiFitsUtility.h"
32 #include "fitsAnalysisTec.h"
33 #include "procDetLin.h"
34 #include "statistics.h"
35 #include "diagnostics.h"
36 #include "cpl_polynomial.h"
37 #include "qfits.h"
38 #include "midi_cplutils.h"
39 #include "midi_cplupgrade.h"
40 
41 /**********************************************************
42 * Constant definitions
43 **********************************************************/
44 
45 /**********************************************************
46 * Global Variables
47 **********************************************************/
48 
49 /*============================ C O D E A R E A ===========================*/
50 
51 
52 
53 /******************************************************************************
54 * European Southern Observatory
55 * VLTI MIDI Maintenance Templates Software
56 *
57 * Module name: procDetLin
58 * Input/Output: See function arguments to avoid duplication
59 * Description: Computes the detector linearity. This is the main routine for this
60 * application.
61 *
62 * History:
63 * 03-May-05 (csabet) Created
64 * 09-May-06 (csabet) Modified to characterise each pixel
65 ******************************************************************************/
66 void procDetLin (
67  MidiFiles *fileNames, // In: Pointer to file names
68  int *error) // Ou: Error status
69 {
70 
71  // Local Declarations
72  // ------------------
73  const char routine[] = "procDetLin";
74  ImageFormat *format=NULL;
75  DetLinearity *linearity=NULL;
76  int numOfFiles;
77  FILE *signaturePtr=NULL;
78 
79  // Algorithm
80  // ---------
81  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
82  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
83 
84  // Write a signature
85  signaturePtr = fopen ("MIDI_sig_lin.log", "w");
86  fclose (signaturePtr);
87 
88  // Reset status
89  *error = 0;
90  numOfFiles = 0;
91 
92  // Allocate memory
93  format = callocImageFormat ();
94 
95  analyseFitsDetLin (fileNames, format, &numOfFiles, error);
96  if (*error)
97  {
98  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot analyse DETLIN");
99  freeImageFormat (format);
100  return;
101  }
102 
103  linearity = callocDetLin (numOfFiles, format);
104  computeDetLin (numOfFiles, fileNames, format, linearity, error);
105  if (*error)
106  {
107  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot process DETLIN");
108  freeDetLin (linearity);
109  freeImageFormat (format);
110  return;
111  }
112 
113  createDetLinProd (fileNames, format, numOfFiles, linearity, error);
114  if (*error) midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot create DETLIN products");
115 
116  // Release memory
117  freeDetLin (linearity);
118  freeImageFormat (format);
119 
120  return;
121 }
122 /*****************************************************************************/
123 
124 
125 /******************************************************************************
126 * European Southern Observatory
127 * VLTI MIDI Maintenance Templates Software
128 *
129 * Module name: computeDetLin
130 * Input/Output: See function arguments to avoid duplication
131 * Description: Computes the detector linearity.
132 * Compute average flux for each pixel in a given file. Exclude saturated pixels.
133 * Find coefficients of a polynomial describing the linearity over the integration
134 * time.
135 *
136 * History:
137 * 15-Jun-04 (csabet) Created
138 * 09-May-06 (csabet) Modified to characterise each pixel
139 ******************************************************************************/
140 void computeDetLin (
141  int numOfFiles, // In: Number of data files
142  MidiFiles *fileNames, // In: Pointer to the MIDI file structure
143  ImageFormat *format, // In: Data format
144  DetLinearity *linearity, // Ou: Pointer to the detector linearity data structure
145  int *error) // Ou: Error status
146 {
147 
148  // Local Declarations
149  // ------------------
150  const char routine[] = "computeDetLin";
151  char *fileTemp, *classification, *firstFitsFile;
152  FILE *inFitsBatchPtr;
153  ImageFormat *formatLocal;
154  int localError, fileNumber, extNumOfImagingDataFile;
155 
156  // Algorithm
157  // ---------
158  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
159  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
160 
161  // Allocate memory
162  classification = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
163  firstFitsFile = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
164  fileTemp = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
165  formatLocal = callocImageFormat ();
166 
167  // Reset status
168  *error = 0;
169  localError = 0;
170  fileNumber = 0;
171  linearity->exists = 0;
172  formatLocal->hasData = 0;
173 
174  // Open the list of files
175  if ((inFitsBatchPtr = fopen (fileNames->inFitsBatch, "r")) == NULL)
176  {
177  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
178  "Cannot open input FITS file list\n No compression has been carried out for this batch");
179  freeImageFormat (formatLocal);
180  free (fileTemp);
181  free (classification);
182  *error = 1;
183  return;
184  }
185 
186  // Loop through the files and analyse
187  while (fgets (fileTemp, MAX_STRING_LENGTH, inFitsBatchPtr) != NULL)
188  {
189  sprintf (classification, "%s", "");
190  sscanf (fileTemp, "%s%s", fileNames->inFitsName, classification);
191  if (diagnostic)cpl_msg_info(cpl_func,"\n Processing file %s \n", fileNames->inFitsName);
192  fprintf(midiReportPtr, "\n Processing file %s \n", fileNames->inFitsName);
193 
194  // Get 'extNumOfImagingDataFile' extension number of IMAGING_DATA in input file
195  extNumOfImagingDataFile = findImagingDataExtension (fileNames->inFitsName, TAB_IMAGING_DATA, &localError);
196  if (localError)
197  {
198  *error = 1;
199  break;
200  }
201 
202  // Get Image Format parameters
203  if (extNumOfImagingDataFile > 0)
204  {
205  getImageFormat (fileNames->inFitsName, extNumOfImagingDataFile, formatLocal, &localError);
206  if (localError)
207  {
208  *error = 1;
209  break;
210  }
211  }
212  else formatLocal->hasData = 0;
213 
214  // Check if the file has data
215  if (formatLocal->hasData)
216  {
217  // Check Categ, Tech and Type and then compute the image size
218  if ((strcmp (formatLocal->obsCatg, "CALIB") == 0) &&
219  ((strcmp (formatLocal->obsTech, "IMAGE") == 0) ||
220  (strcmp (formatLocal->obsTech, "SPECTRUM") == 0)) &&
221  (strcmp (formatLocal->obsType, "FLAT") == 0))
222  {
223  // Save grism ID
224  sprintf (linearity->grismId, "%s", formatLocal->grismId);
225  if (diagnostic)cpl_msg_info(cpl_func," Grism ID for file %d = %s\n", fileNumber+1, formatLocal->grismId);
226  fprintf (midiReportPtr, " Grism ID for file %d = %s\n", fileNumber+1, formatLocal->grismId);
227 
228  // Create and display averaged images
229  createAndDisplayImageDetLin (fileNumber, fileNames->inFitsName, extNumOfImagingDataFile,
230  format, linearity, &localError);
231  if (localError) *error = 1;
232 
233  // Calculate mean of counts. Ignore this file if pixels are saturated
234  getIntegrationTime (fileNumber, fileNames->inFitsName, linearity, &localError);
235  if (localError) *error = 1;
236 
237  // Set the flags
238  linearity->exists = 1;
239 
240  // Increment file number but make sure it is not greater than the allocated
241  fileNumber++;
242  if (fileNumber > numOfFiles)
243  {
244  *error = 1;
245  break;
246  }
247 
248  // Save name of the first FITS file
249  if (fileNumber == 1) sprintf (firstFitsFile, "%s", fileNames->inFitsName);
250  }
251  else
252  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, "The above file is not suitable for this task");
253  }
254  else
255  {
256  if (diagnostic)
257  {
258  sprintf (midiMessage, "No data tables in %s. Not processed", fileNames->inFitsName);
259  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
260  }
261  }
262  }
263 
264 
265  // Check if processing has been carried out
266  if ((linearity->exists) && !(*error) && (fileNumber == numOfFiles))
267  {
268  cpl_msg_info(cpl_func,"\nDetector Linearity Inventry: \n");
269  cpl_msg_info(cpl_func,"=========================== \n");
270  cpl_msg_info(cpl_func," Expected number of data files = %d\n", numOfFiles);
271  cpl_msg_info(cpl_func," Number of data files processed = %d\n", fileNumber);
272  cpl_msg_info(cpl_func,"\n");
273 
274  fprintf (midiReportPtr, "\nDetector Linearity Inventry: \n");
275  fprintf (midiReportPtr, "=========================== \n");
276  fprintf (midiReportPtr, " Expected number of data files = %d\n", numOfFiles);
277  fprintf (midiReportPtr, " Number of data files processed = %d\n", fileNumber);
278  fprintf (midiReportPtr, "\n");
279 
280  // Assess linerity
281  // ---------------
282  assessLinearity (format, firstFitsFile, linearity, error);
283  }
284 
285  if (*error || localError)
286  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot assess Detector Linearity");
287 
288  // Close the file list
289  fclose (inFitsBatchPtr);
290 
291  // Release memory
292  freeImageFormat (formatLocal);
293  free (fileTemp);
294  free (classification);
295  free (firstFitsFile);
296 
297  return;
298 }
299 /*****************************************************************************/
300 
301 
302 /******************************************************************************
303 * European Southern Observatory
304 * VLTI MIDI Maintenance Templates Software
305 *
306 * Module name: createAndDisplayImageDetLin
307 * Input/Output: See function arguments to avoid duplication
308 * Description: Creates averaged images for display and later processing
309 * purposes.
310 *
311 * History:
312 * 08-May-06 (csabet) Created
313 ******************************************************************************/
314 void createAndDisplayImageDetLin (
315  int fileNumber, // In: File number
316  char *fileName, // In: Name of file to process
317  int extensionNumber, // In: Extension number of the IMAGE_DATA
318  ImageFormat *format, // In: Pointer to the image format
319  DetLinearity *linearity, // Ou: Pointer to the detector linearity data structure
320  int *error) // Ou: Error status
321 
322 {
323 
324  // Local Declarations
325  // ------------------
326  const char routine[] = "createAndDisplayImageDetLin";
327  qfits_table *pTable = NULL;
328  short int *inData;
329  char *tempStr, *string, *title, *dataName;
330  int i, foundData = 0, scalingOffset,
331  indexData;
332 
333  // Algorithm
334  // ---------
335  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
336  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
337 
338  // Reset status
339  *error = 0;
340  linearity->saturated[fileNumber] = 0;
341 
342  // Open IMAGING_DATA = INDEX 2
343  pTable = qfits_table_open (fileName, extensionNumber);
344  if (!pTable)
345  {
346  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load IMAGING_DATA");
347  *error = 1;
348  return;
349  }
350 
351  // Get data table information
352  for (i = 0; i < pTable->nc; i++)
353  {
354  if (strcmp (pTable->col[i].tlabel, "DATA1") == 0)
355  {
356  foundData = 1;
357  indexData = i;
358  }
359  }
360  if (foundData == 0)
361  {
362  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find requested columns in data FITS file");
363  qfits_table_close (pTable);
364  *error = 1;
365  return;
366  }
367 
368  // Read column DATA
369  inData = (short int*) qfits_query_column (pTable, indexData, NULL);
370 
371  // Get the scaling offset
372  dataName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
373  for (i = 14; i < 25; i++)
374  {
375  sprintf (dataName, "TZERO%d", i);
376  tempStr = qfits_query_ext (fileName, dataName, extensionNumber);
377  if (tempStr != NULL)
378  {
379  if (diagnostic)cpl_msg_info(cpl_func,"Scaling Offset = %s\n", tempStr);
380  if (diagnostic) fprintf (midiReportPtr, "Scaling Offset = %s\n", tempStr);
381  sscanf (tempStr, "%d", &scalingOffset);
382  break;
383  }
384  }
385  if (tempStr == NULL)
386  {
387  scalingOffset = 0;
388  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Scaling Offset. It is set to 0");
389  }
390  free (dataName);
391 
392  // Create averaged image for the whole file. This also allows a check on existence of a target
393  createAveragedImage (inData, scalingOffset, format, linearity->aveImage[fileNumber]);
394 
395  // Create an image FITS file
396  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
397  string = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
398  sprintf (title, "AveImg%d", fileNumber+1);
399  sprintf (string, "file %d", fileNumber+1);
400  createFitsImage (string, title, fileName, format->iXWidth, format->iYWidth, linearity->aveImage[fileNumber]);
401 
402  // Create plot file
403  if (plotFile)
404  {
405  sprintf (string, "3dAveImg%d", fileNumber+1);
406  sprintf (title, "Averaged Image, file %d", fileNumber+1);
407  midiCreatePlotFile3D (string, title, "X", "Y", "Flux",
408  0, linearity->aveImage[fileNumber], format->iXWidth, format->iYWidth, "lines", "3");
409  }
410 
411  // Create an averaged image. This time excluding the saturated pixels
412  checkSaturationDetLin (inData, scalingOffset, format, linearity->aveImage[fileNumber],
413  &(linearity->saturated[fileNumber]));
414 
415  // Create plot file excluding saturated pixels
416  if (diagnostic > 2)
417  {
418  sprintf (string, "3dAveImgClean%d", fileNumber+1);
419  sprintf (title, "Averaged Image. Excluding saturated pixels, file %d", fileNumber+1);
420  if (plotFile) midiCreatePlotFile3D (string, title, "X", "Y", "Flux",
421  0, linearity->aveImage[fileNumber], format->iXWidth, format->iYWidth, "lines", "3");
422  }
423  free (title);
424  free (string);
425 
426  // Release memory
427  qfits_table_close (pTable);
428  free (inData);
429 
430  return;
431 }
432 /*****************************************************************************/
433 
434 
435 /******************************************************************************
436 * European Southern Observatory
437 * VLTI MIDI Maintenance Templates Software
438 *
439 * Module name: getIntegrationTime
440 * Input/Output: See function arguments to avoid duplication
441 * Description: Retrieves Integration time as well as the Number of Integrations.
442 *
443 * History:
444 * 09-May-06 (csabet) Created
445 ******************************************************************************/
446 void getIntegrationTime (
447  int fileNumber, // In: File number
448  char *fileName, // In: Name of file to process
449  DetLinearity *linearity, // Ou: Pointer to the detector linearity data structure
450  int *error) // Ou: Error status
451 
452 {
453 
454  // Local Declarations
455  // ------------------
456  const char routine[] = "getIntegrationTime";
457  char *tempStr;
458  float numOfSubInteg, subIntegTime;
459 
460  // Algorithm
461  // ---------
462  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
463  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
464 
465  // Reset status
466  *error = 0;
467 
468  // Get integration time
469  tempStr = qfits_query_hdr (fileName, "HIERARCH ESO DET DIT");
470  if (tempStr != NULL)
471  sscanf(tempStr, "%f", &subIntegTime);
472  else
473  {
474  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Integration time");
475  *error = 1;
476  return;
477  }
478  tempStr = qfits_query_hdr (fileName, "HIERARCH ESO DET NDIT");
479  if (tempStr != NULL)
480  sscanf(tempStr, "%f", &numOfSubInteg);
481  else
482  {
483  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Number of Integrations");
484  *error = 1;
485  return;
486  }
487  linearity->integTime[fileNumber] = numOfSubInteg * subIntegTime;
488 
489  return;
490 }
491 /*****************************************************************************/
492 
493 
494 
495 /******************************************************************************
496 * European Southern Observatory
497 * VLTI MIDI Maintenance Templates Software
498 *
499 * Module name: assessLinearity
500 * Input/Output: See function arguments to avoid duplication
501 * Description: The dependence of the count rate on the integration time and therewith
502 * the number of detected photons is over wide range a linear function.
503 * It is known, that the function above a count rate of 50000 up to
504 * saturation (65536 counts) differs from linarity.
505 * Fit a second or third order polynomial to the count levels as a function
506 * of the actual exposure time. That is y = A0 + A1 * t + A2 * t^2 + A3 * t^3
507 * The values of the coeficients are the required relevant QC parameters.
508 *
509 * 1. Now we have the following 10 images (im)
510 * linearity->aveImage[im][pix] im = 0 to 9, pix = 0 to subWinSize
511 *
512 * 2. We know which file has saturation from
513 * linearity->saturated[im] im = 0 to 9
514 *
515 * 3. We have Integration Time
516 * linearity->integTime[im] im = 0 to 9
517 *
518 * 4. Mean flux for each pixel (pix) of each image (im) is of course
519 * given directly from (1) above. That is
520 * linearity->aveImage[im][pix] im = 0 to 9, pix = 0 to subWinSize
521 *
522 * 5. We can first create a 3D plot of raw linearity for each pixel. The
523 * releveant axes are:
524 * x = Integration Time (given from 3 above)
525 * y = Pixel position pix = 0 to subWinSize
526 * z = Mean Flux
527 *
528 * 6. For each pixel in 5 above we can compute the linearity coefficients:
529 * A0[pix], A1[pix], A2[pix], A3[pix]
530 *
531 * 7. Finally we find the averages of the above coefficients over a desired
532 * region of the image.
533 *
534 * History:
535 * 23-Jun-04 (csabet) Created
536 * 09-May-06 (csabet) Modified to characterise each pixel
537 ******************************************************************************/
538 void assessLinearity (
539  ImageFormat *format, // In: Pointer to the image format
540  char *firstFitsFile, // In: Name of first FITS file
541  DetLinearity *linearity, // Ou: Pointer to the detector linearity data structure
542  int *error) // Ou: Error status
543 {
544  // Local Declarations
545  // ------------------
546  const char routine[] = "assessLinearity";
547  cpl_polynomial *coefficients;
548  cpl_vector *xPosition, *values;
549  int j, f, x, y, p, xLength, yLength, xCoord, dxCoord, yCoord, dyCoord,
550  polyDeg = 3, numOfUnsatImages, imageSize, subImageSize;
551  cpl_size power=0;
552  double *integTime, *pixel, *cA0, *cA1, *cA2, *cA3, *linFit;
553  char *fileName, *title;
554  float *buffer;
555 
556  // Statistics for pixel regression test
557  float sigData = 0.0, coeff1, coeff2, siga, sigb, chi2, q;
558  int sigDataAvailable = 0;
559 
560  // Algorithm
561  // ---------
562  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
563  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
564 
565  // Reset status
566  *error = 0;
567 
568  // Count number of unsaturated images
569  numOfUnsatImages = 0;
570  for (j = 0; j < linearity->size; j++)
571  {
572  if (!(linearity->saturated[j])) numOfUnsatImages++;
573  }
574 
575  // Compute linearity coefficients for each pixel in a well illuminated central area
576  // --------------------------------------------------------------------------------
577  xLength = DET_LIN_WIN_PU * format->iXWidth;
578  if (xLength < DET_LIN_WIN_MIN) xLength = format->iXWidth;
579  xCoord = (format->iXWidth - xLength) / 2;
580  dxCoord = format->iXWidth - xCoord;
581  xLength = dxCoord - xCoord; // Avoing round-off error
582 
583  yLength = DET_LIN_WIN_PU * format->iYWidth;
584  if (yLength < DET_LIN_WIN_MIN) yLength = format->iYWidth;
585  yCoord = (format->iYWidth - yLength) / 2;
586  dyCoord = format->iYWidth - yCoord;
587  yLength = dyCoord - yCoord; // Avoing round-off error
588 
589  imageSize = format->iXWidth * format->iYWidth;
590 
591  // Load values of the window for QC log
592  linearity->winx = xCoord;
593  linearity->windx = dxCoord;
594  linearity->winy = yCoord;
595  linearity->windy = dyCoord;
596  if (diagnostic)
597  {
598  cpl_msg_info(cpl_func,"\nSelected window size \n");
599  cpl_msg_info(cpl_func,"-------------------- \n");
600  cpl_msg_info(cpl_func,"xCoord = %3d \n", linearity->winx);
601  cpl_msg_info(cpl_func,"dxCoord = %3d \n", linearity->windx);
602  cpl_msg_info(cpl_func,"yCoord = %3d \n", linearity->winy);
603  cpl_msg_info(cpl_func,"dyCoord = %3d \n", linearity->windy);
604  }
605  fprintf (midiReportPtr, "\nSelected window size QCLOG \n");
606  fprintf (midiReportPtr, "-------------------- QCLOG \n");
607  fprintf (midiReportPtr, "xCoord = %3d QCLOG \n", linearity->winx);
608  fprintf (midiReportPtr, "dxCoord = %3d QCLOG \n", linearity->windx);
609  fprintf (midiReportPtr, "yCoord = %3d QCLOG \n", linearity->winy);
610  fprintf (midiReportPtr, "dyCoord = %3d QCLOG \n", linearity->windy);
611 
612  // Allocate memory
613  fileName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
614  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
615  buffer = (float *) calloc (imageSize, sizeof (float));
616  cA0 = (double *) calloc (imageSize, sizeof (double));
617  cA1 = (double *) calloc (imageSize, sizeof (double));
618  cA2 = (double *) calloc (imageSize, sizeof (double));
619  cA3 = (double *) calloc (imageSize, sizeof (double));
620  linFit = (double *) calloc (numOfUnsatImages, sizeof (double));
621 
622  // Create a sub-image from each unsaturated image
623  for (j = 0; j < numOfUnsatImages; j++)
624  {
625  for (y = 0; y < format->iYWidth; y++)
626  {
627  f = y * format->iXWidth;
628  for (x = 0; x < format->iXWidth; x++)
629  {
630  if (!((x >= xCoord) && (x < dxCoord) && (y >= yCoord) && (y < dyCoord)))
631  {
632  p = f + x;
633  linearity->aveImage[j][p] = 0.0;
634  }
635  }
636  }
637  if (plotFile && diagnostic)
638  {
639  sprintf (fileName, "3dCentralImage%d", j+1);
640  sprintf (title, "Averaged central image, file %d", j+1);
641  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0, linearity->aveImage[j],
642  format->iXWidth, format->iYWidth, "lines", "3");
643  }
644  }
645 
646  // Allocate memory
647  pixel = (double *) calloc (numOfUnsatImages, sizeof (double));
648  integTime = (double *) calloc (numOfUnsatImages, sizeof (double));
649 
650  // For all unsaturated images Load integration time into a buffer
651  for (j = 0; j < numOfUnsatImages; j++)
652  integTime[j] = (double) (linearity->integTime[j]);
653 
654  for (y = 0; y < format->iYWidth; y++)
655  {
656  f = y * format->iXWidth;
657  for (x = 0; x < format->iXWidth; x++)
658  {
659  // For all unsaturated images Load pixel flux into a buffer
660  if ((x >= xCoord) && (x < dxCoord) &&
661  (y >= yCoord) && (y < dyCoord))
662  {
663  p = f + x;
664  for (j = 0; j < numOfUnsatImages; j++)
665  pixel[j] = (double) (linearity->aveImage[j][p]);
666 
667  // Plot raw linearity and delete plot file
668  if (plotFile && diagnostic > 2)
669  {
670  sprintf (title, "Flux versus integration time. Pixel %d, %d", x, y);
671  midiCreatePlotFileDouble2D2P ("temp1", title, "Integration time", "Pixel flux",
672  1, integTime, pixel, 0, numOfUnsatImages, 1, "lines 3");
673  }
674 
675  // Compute coefficients of a 3rd order polynomial
676  xPosition = cpl_vector_wrap (numOfUnsatImages, integTime);
677  values = cpl_vector_wrap (numOfUnsatImages, pixel);
678  coefficients = cpl_polynomial_fit_1d_create (xPosition, values, polyDeg, NULL);
679  power = 0;
680  cA0[p] = cpl_polynomial_get_coeff (coefficients, &power);
681  power = 1;
682  cA1[p] = cpl_polynomial_get_coeff (coefficients, &power);
683  power = 2;
684  cA2[p] = cpl_polynomial_get_coeff (coefficients, &power);
685  power = 3;
686  cA3[p] = cpl_polynomial_get_coeff (coefficients, &power);
687  cpl_vector_unwrap (xPosition);
688  cpl_vector_unwrap (values);
689  cpl_polynomial_delete (coefficients);
690 
691  // Plot fit linearity and delete plot file
692  if (plotFile && diagnostic > 2)
693  {
694  // Fill in the result
695  for (j = 0; j < numOfUnsatImages; j++)
696  linFit[j] = cA0[p] + cA1[p] * j + cA2[p] * (j * j) + cA3[p] * (j * j * j);
697 
698  sprintf (title, "Flux versus integration time (polynomial fit). Pixel %d, %d", x, y);
699  midiCreatePlotFileDouble2D2P ("temp1", title, "Integration time", "Pixel flux",
700  1, integTime, linFit, 0, numOfUnsatImages, 1, "lines 1");
701  }
702 
703  // Now do a regression test (degree of deviation from linearity)
704  midiGetLinearFit (integTime, pixel, numOfUnsatImages, sigData, sigDataAvailable,
705  &coeff1, &coeff2, &siga, &sigb, &chi2, &q, error);
706 
707  // Load a measure of linear deviation. Here we use the
708  // combined standard deviation of the linear coefficients
709  linearity->deviation[p] = sqrt ((siga+sigb)/2.0);
710 
711  if (diagnostic > 2)
712  {
713  cpl_msg_info(cpl_func,"\nPixel (%d, %d) statistics \n", x, y);
714  cpl_msg_info(cpl_func,"------------------------- \n");
715  cpl_msg_info(cpl_func,"coeff1 = %f \n", coeff1);
716  cpl_msg_info(cpl_func,"coeff2 = %f \n", coeff2);
717  cpl_msg_info(cpl_func,"siga = %f \n", siga);
718  cpl_msg_info(cpl_func,"sigb = %f \n", sigb);
719  cpl_msg_info(cpl_func,"chi2 = %f \n", chi2);
720  cpl_msg_info(cpl_func,"q = %f \n", q);
721  fprintf (midiReportPtr, "\nPixel (%d, %d) statistics \n", x, y);
722  fprintf (midiReportPtr, "------------------------- \n");
723  fprintf (midiReportPtr, "coeff1 = %f \n", coeff1);
724  fprintf (midiReportPtr, "coeff2 = %f \n", coeff2);
725  fprintf (midiReportPtr, "siga = %f \n", siga);
726  fprintf (midiReportPtr, "sigb = %f \n", sigb);
727  fprintf (midiReportPtr, "chi2 = %f \n", chi2);
728  fprintf (midiReportPtr, "q = %f \n", q);
729  }
730  }
731  }
732  }
733 
734  // Create plots and FITS images of coefficients
735  if (plotFile && diagnostic)
736  {
737  midiCreatePlotFileDouble3D ("3dCoefficientMapA0", "Map of A0 coefficients", "X", "Y", "Coefficient value",
738  0, cA0, format->iXWidth, format->iYWidth, "points", "1");
739  midiCreatePlotFileDouble3D ("3dCoefficientMapA1", "Map of A1 coefficients", "X", "Y", "Coefficient value",
740  0, cA1, format->iXWidth, format->iYWidth, "points", "1");
741  midiCreatePlotFileDouble3D ("3dCoefficientMapA2", "Map of A2 coefficients", "X", "Y", "Coefficient value",
742  0, cA2, format->iXWidth, format->iYWidth, "points", "1");
743  midiCreatePlotFileDouble3D ("3dCoefficientMapA3", "Map of A3 coefficients", "X", "Y", "Coefficient value",
744  0, cA3, format->iXWidth, format->iYWidth, "points", "1");
745  }
746  for (p = 0; p < imageSize; p++) buffer[p] = (float) (cA0[p]);
747  removeDc (imageSize, buffer, buffer);
748  createFitsImage ("Coefficients A0", "CoefficientMapA0", firstFitsFile, format->iXWidth, format->iYWidth, buffer);
749  for (p = 0; p < imageSize; p++) buffer[p] = (float) (cA1[p]);
750  removeDc (imageSize, buffer, buffer);
751  createFitsImage ("Coefficients A1", "CoefficientMapA1", firstFitsFile, format->iXWidth, format->iYWidth, buffer);
752  for (p = 0; p < imageSize; p++) buffer[p] = (float) (cA2[p]);
753  removeDc (imageSize, buffer, buffer);
754  createFitsImage ("Coefficients A2", "CoefficientMapA2", firstFitsFile, format->iXWidth, format->iYWidth, buffer);
755  for (p = 0; p < imageSize; p++) buffer[p] = (float) (cA3[p]);
756  removeDc (imageSize, buffer, buffer);
757  createFitsImage ("Coefficients A3", "CoefficientMapA3", firstFitsFile, format->iXWidth, format->iYWidth, buffer);
758 
759  // Create plot and FITS image of the linear deviation
760  if (plotFile) midiCreatePlotFile3D ("3dLinearitySigmaMap", "Map of Linearity Standard Deviation", "X", "Y", "Sigma",
761  0, linearity->deviation, format->iXWidth, format->iYWidth, "lines", "1");
762 
763  // Suitably prepare image for FITS
764  removeDc (imageSize, linearity->deviation, buffer);
765  createFitsImage ("Map of Linearity Standard Deviation", "LinearitySigmaMap", firstFitsFile, format->iXWidth,
766  format->iYWidth, buffer);
767 
768  // Compute mean value of the coefficients and the overall linearity deviation
769  linearity->meanCoeffA0 = 0.0;
770  linearity->meanCoeffA1 = 0.0;
771  linearity->meanCoeffA2 = 0.0;
772  linearity->meanCoeffA3 = 0.0;
773  subImageSize = 0;
774  for (y = 0; y < format->iYWidth; y++)
775  {
776  f = y * format->iXWidth;
777  for (x = 0; x < format->iXWidth; x++)
778  {
779  if ((x >= xCoord) && (x < dxCoord) && (y >= yCoord) && (y < dyCoord))
780  {
781  p = f + x;
782  linearity->meanCoeffA0 += cA0[p];
783  linearity->meanCoeffA1 += cA1[p];
784  linearity->meanCoeffA2 += cA2[p];
785  linearity->meanCoeffA3 += cA3[p];
786  linearity->meanSigma += linearity->deviation[p];
787  subImageSize++;
788  }
789  }
790  }
791  if (subImageSize)
792  {
793  linearity->meanCoeffA0 /= subImageSize;
794  linearity->meanCoeffA1 /= subImageSize;
795  linearity->meanCoeffA2 /= subImageSize;
796  linearity->meanCoeffA3 /= subImageSize;
797  linearity->meanSigma /= subImageSize;
798  }
799 
800  cpl_msg_info(cpl_func,"\nMean coefficients for polynomial fit: y = a0 + a1 * t + a2 * t^2 + a3 * t^3 \n");
801  cpl_msg_info(cpl_func,"--------------------------------------------------------------------------- \n");
802  cpl_msg_info(cpl_func,"a0, a1, a2, a3 = %f, %f, %f, %f \n",
803  linearity->meanCoeffA0, linearity->meanCoeffA1, linearity->meanCoeffA2, linearity->meanCoeffA3);
804  fprintf (midiReportPtr, "\nMean coefficients for polynomial fit: y = a0 + a1 * t + a2 * t^2 + a3 * t^3 (QCLOG) \n");
805  fprintf (midiReportPtr, "--------------------------------------------------------------------------- (QCLOG) \n");
806  fprintf (midiReportPtr, "a0, a1, a2, a3 = %f, %f, %f, %f (QCLOG) \n",
807  linearity->meanCoeffA0, linearity->meanCoeffA1, linearity->meanCoeffA2, linearity->meanCoeffA3);
808 
809  cpl_msg_info(cpl_func,"\nMean linearity standard deviation = %f \n\n", linearity->meanSigma);
810  fprintf (midiReportPtr, "\nMean linearity standard deviation = %f (QCLOG) \n\n", linearity->meanSigma);
811 
812  // Plot linear fit of the mean
813  for (j = 0; j < numOfUnsatImages; j++)
814  {
815  linFit[j] = linearity->meanCoeffA0 + linearity->meanCoeffA1 * j + linearity->meanCoeffA2 * (j * j) +
816  linearity->meanCoeffA3 * (j * j * j);
817  linearity->mean[j] = (float) linFit[j];
818 
819  }
820  midiCreatePlotFileDouble2D2P ("MeanFluxVersusTimePolyFit", "Meam flux versus time (polynomial fit)",
821  "Integration time", "Mean flux", 0, integTime, linFit, 0, numOfUnsatImages, 1, "lines 1");
822 
823  // Release memory
824  free (linFit);
825  free (cA0);
826  free (cA1);
827  free (cA2);
828  free (cA3);
829  free (fileName);
830  free (title);
831  free (pixel);
832  free (integTime);
833  free (buffer);
834 
835  return;
836 
837 }
838 /*****************************************************************************/
839 
840 
841 
842 /******************************************************************************
843 * European Southern Observatory
844 * VLTI MIDI Maintenance Templates Software
845 *
846 * Module name: checkSaturationDetLin
847 * Input/Output: See function arguments to avoid duplication
848 * Description: Checks if any pixel is saturated. If so this exposure
849 * will not be included in the later processing
850 *
851 * History:
852 * 05-May-06 (csabet) Created
853 ******************************************************************************/
854 void checkSaturationDetLin (
855  short int *inData, // In: Pointer to the input data
856  float scalingOffset, // In: Scaling Offset
857  ImageFormat *format, // In: Pointer to the image format
858  float *image, // Ou: Output image
859  int *saturated) // Ou: Saturation indicator
860 
861 {
862  // Local Declarations
863  // ------------------
864  const char routine[] = "checkSaturationDetLin";
865  int i, count, pixel, frame, imageSize;
866 
867  // Algorithm
868  // ---------
869  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
870  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
871 
872  // Reset status
873  imageSize = format->iXWidth * format->iYWidth;
874  *saturated = 0;
875 
876  for (pixel = 0; pixel < imageSize; pixel++)
877  {
878  // Along the time axis (frame)
879  image[pixel] = 0;
880  count = 0;
881  for (frame = 0; frame < format->numOfFrames; frame++)
882  {
883  // Load each pixel data along the time axis
884  i = frame * imageSize + pixel;
885  if (isnan (inData[i]))
886  {
887  sprintf (midiMessage, "Found bad pixel %d on frame %d", frame, pixel);
888  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
889  }
890  else if (((float) (inData[i] + scalingOffset)) >= PIXEL_SATURATION)
891  {
892  if (diagnostic > 2)
893  {
894  sprintf (midiMessage, "Found saturated pixel %d on frame %d", frame, pixel);
895  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
896  }
897  *saturated = 1;
898  }
899  else
900  {
901  count++;
902  image[pixel] += (float) (inData[i] + scalingOffset);
903  }
904  }
905 
906  // Compute average image
907  if (count) image[pixel] /= count;
908  }
909 
910  if (*saturated)
911  {
912  sprintf (midiMessage, "Found saturated pixels in this image");
913  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
914  }
915 
916  return;
917 }
918 /*****************************************************************************/