MIDI Pipeline Reference Manual  2.8.3
procDetRon.c
1 
2 /******************************************************************************
3 *******************************************************************************
4 * European Southern Observatory
5 * VLTI MIDI Maintenance Templates Software
6 *
7 * Module name: procDetRon.c
8 * Description: Contains routines for processing the templates
9 *
10 * History:
11 * 16-Jun-04 (csabet) created
12 *******************************************************************************
13 ******************************************************************************/
14 
15 /******************************************************************************
16 * Compiler directives
17 ******************************************************************************/
18 
19 /******************************************************************************
20 * Include files
21 ******************************************************************************/
22 #include <math.h>
23 #include <stdio.h>
24 #include <cpl.h>
25 #include "midiGlobal.h"
26 #include "midiLib.h"
27 #include "memoryHandling.h"
28 #include "statistics.h"
29 #include "createProdDetRon.h"
30 #include "cpl_image_stats.h"
31 #include "cpl_image_io.h"
32 #include "imageProcessing.h"
33 #include "midiFitsUtility.h"
34 #include "fitsAnalysisTec.h"
35 #include "procDetRon.h"
36 #include "errorHandling.h"
37 #include "diagnostics.h"
38 #include "qfits.h"
39 
40 /**********************************************************
41 * Constant definitions
42 **********************************************************/
43 
44 /**********************************************************
45 * Global Variables
46 **********************************************************/
47 
48 /*============================ C O D E A R E A ===========================*/
49 
50 
51 
52 /******************************************************************************
53 * European Southern Observatory
54 * VLTI MIDI Maintenance Templates Software
55 *
56 * Module name: procDetRon
57 * Input/Output: See function arguments to avoid duplication
58 * Description: Computes the detector read-out noise. This is the main routine for this
59 * application.
60 *
61 * History:
62 * 03-May-05 (csabet) Created
63 ******************************************************************************/
64 void procDetRon (
65  MidiFiles *fileNames, // In: Pointer to file names
66  int *error) // Ou: Error status
67 {
68 
69  // Local Declarations
70  // ------------------
71  const char routine[] = "procDetRon";
72  ImageFormat *format=NULL;
73  DetRonNoise *noiseProfile=NULL;
74  int numOfFiles;
75  FILE *signaturePtr=NULL;
76 
77  // Algorithm
78  // ---------
79  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
80  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
81 
82  // Write a signature
83  signaturePtr = fopen ("MIDI_sig_ron.log", "w");
84  fclose (signaturePtr);
85 
86  // Reset status
87  *error = 0;
88  numOfFiles = 0;
89 
90  // Allocate memory
91  format = callocImageFormat ();
92 
93  analyseFitsDetRon (fileNames, format, &numOfFiles, error);
94  if (*error)
95  {
96  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot analyse DETRON");
97  freeImageFormat (format);
98  return;
99  }
100 
101  noiseProfile = callocDetRon (format);
102  computeDetRon (numOfFiles, format, fileNames, noiseProfile, error);
103  if (*error)
104  {
105  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot process DETRON");
106  freeDetRon (noiseProfile);
107  freeImageFormat (format);
108  return;
109  }
110 
111  createDetRonProd (fileNames, format, noiseProfile, error);
112  if (*error) midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot create DETRON products");
113 
114  // Release memory
115  freeDetRon (noiseProfile);
116  freeImageFormat (format);
117 
118  return;
119 }
120 /*****************************************************************************/
121 
122 
123 
124 /******************************************************************************
125 * European Southern Observatory
126 * VLTI MIDI Maintenance Templates Software
127 *
128 * Module name: computeDetRon
129 * Input/Output: See function arguments to avoid duplication
130 * Description: Computes the detector read-out noise.
131 * Compute the standard deviation pixel by pixel along the time sequence:
132 * This method allows one to obtain a noise map and distribution of noise
133 * values. All noise components contribute to the final value, including
134 * a frame to frame variation in the offset that is easily removed in the
135 * observations. To remove it, one can subtract from each frame the mean
136 * value, channel by channel:
137 *
138 * IMPORTANT:
139 * To obtain the noise in electrons, multiply the result in ADUs by the
140 * conversion factor (145 e-/ADU).
141 *
142 * Summary: Find standard deviation of each pixel along the time (frame axis).
143 * Essentially you will end up with 320 X 240 number of standard deviations
144 *
145 *
146 * History:
147 * 23-Jun-04 (csabet) Created
148 ******************************************************************************/
149 void computeDetRon (
150  int numOfFiles, // In: Number of data files
151  ImageFormat *maxFormat, // In: Pointer to the maximum image format
152  MidiFiles *fileNames, // In: Pointer to the MIDI file structure
153  DetRonNoise *noiseProfile, // Ou: Pointer to the detector noise data structure
154  int *error) // Ou: Error status
155 {
156 
157  // Local Declarations
158  // ------------------
159  const char routine[] = "computeDetRon";
160  char *fileTemp, *classification, *firstFile;
161  FILE *inFitsBatchPtr;
162  ImageFormat *format;
163  int fileNumber, extNumOfImagingDataFile, subWindowSize;
164  float *arrayPixel;
165 
166  // Algorithm
167  // ---------
168  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
169  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
170 
171  // Reset status
172  *error = 0;
173  fileNumber = 0;
174  noiseProfile->exists = 0;
175  subWindowSize = maxFormat->iXWidth * maxFormat->iYWidth;
176 
177  // Allocate memory
178  classification = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
179  firstFile = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
180  fileTemp = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
181  format = callocImageFormat ();
182  arrayPixel = (float *) calloc (subWindowSize * maxFormat->numOfFrames, sizeof (float));
183 
184  // Initialise
185  format->hasData = 0;
186 
187  // Open the list of files
188  if ((inFitsBatchPtr = fopen (fileNames->inFitsBatch, "r")) == NULL)
189  {
190  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
191  "Cannot open input FITS file list\n No preprocessing has been carried out for this batch");
192  freeImageFormat (format);
193  free (fileTemp);
194  free (classification);
195  free (firstFile);
196  free (arrayPixel);
197  *error = 1;
198  return;
199  }
200 
201  // Loop through the files and analyse
202  while (fgets (fileTemp, MAX_STRING_LENGTH, inFitsBatchPtr) != NULL)
203  {
204  sprintf (classification, "%s", "");
205  sscanf (fileTemp, "%s%s", fileNames->inFitsName, classification);
206  if (diagnostic)cpl_msg_info(cpl_func,"\n Processing file %s \n", fileNames->inFitsName);
207  fprintf(midiReportPtr, "\n Processing file %s \n", fileNames->inFitsName);
208 
209  // Get 'extNumOfImagingDataFile' extension number of IMAGING_DATA in input file
210  extNumOfImagingDataFile = findImagingDataExtension (fileNames->inFitsName, TAB_IMAGING_DATA, error);
211  if (*error) break;
212 
213  // Get Image Format parameters
214  if (extNumOfImagingDataFile > 0)
215  {
216  getImageFormat (fileNames->inFitsName, extNumOfImagingDataFile, format, error);
217  if (*error) break;
218  }
219  else format->hasData = 0;
220 
221  // Check if the file has data
222  if (format->hasData)
223  {
224  // Check Categ, Tech and Type and then compute the image size
225  if ((strcmp (format->obsCatg, "CALIB") == 0) &&
226  (strcmp (format->obsTech, "IMAGE") == 0) &&
227  (strcmp (format->obsType, "BIAS") == 0))
228  {
229  // Increment file number but make sure it is not greater than the allocated
230  fileNumber++;
231  if (fileNumber > numOfFiles)
232  {
233  *error = 1;
234  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Inconsistent file quantity");
235  break;
236  }
237 
238  // Save the name of the first file
239  if (fileNumber == 1) sprintf (firstFile, "%s", fileNames->inFitsName);
240 
241  // Compress data
242  compressDetRon (fileNumber, numOfFiles, maxFormat->numOfFrames, fileNames->inFitsName,
243  extNumOfImagingDataFile, format, arrayPixel, noiseProfile, error);
244  if (*error)
245  {
246  sprintf (midiMessage, "Cannot compress data in %s", fileNames->inFitsName);
247  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
248  break;
249  }
250  }
251  else
252  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, "The above file is not suitable for this task");
253  }
254  else
255  {
256  if (diagnostic)
257  {
258  sprintf (midiMessage, "No data tables in %s. Not processed", fileNames->inFitsName);
259  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
260  }
261  }
262  }
263 
264 
265  // Check if reduction has been carried out
266  if (*error || (fileNumber != numOfFiles))
267  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot create a noise pattern for this batch");
268  else
269  {
270  cpl_msg_info(cpl_func,"\nDetector Readout Noise Inventry: \n");
271  cpl_msg_info(cpl_func,"=============================== \n");
272  cpl_msg_info(cpl_func," Expected number of data files = %d\n", numOfFiles);
273  cpl_msg_info(cpl_func," Number of data files processed = %d\n", fileNumber);
274  cpl_msg_info(cpl_func,"\n");
275 
276  fprintf (midiReportPtr, "\nDetector Readout Noise Inventry: \n");
277  fprintf (midiReportPtr, "=============================== \n");
278  fprintf (midiReportPtr, " Expected number of data files = %d\n", numOfFiles);
279  fprintf (midiReportPtr, " Number of data files processed = %d\n", fileNumber);
280  fprintf (midiReportPtr, "\n");
281  }
282 
283  // Close the file list
284  fclose (inFitsBatchPtr);
285  free (arrayPixel);
286 
287  // Create Noise Pattern
288  createNoisePattern (fileNumber, firstFile, maxFormat, noiseProfile, error);
289 
290  // If no error set the exists flag
291  if (!(*error)) noiseProfile->exists = 1;
292 
293  // Release memory
294  freeImageFormat (format);
295  free (fileTemp);
296  free (classification);
297  free (firstFile);
298 
299  return;
300 }
301 /*****************************************************************************/
302 
303 
304 
305 /******************************************************************************
306 * European Southern Observatory
307 * VLTI MIDI Maintenance Templates Software
308 *
309 * Module name: compressDetRon
310 * Input/Output: See function arguments to avoid duplication
311 * Description: Compresses the DETRON data
312 *
313 * History:
314 * 25-Apr-06 (csabet) Created
315 ******************************************************************************/
316 void compressDetRon (
317  int fileNumber, // In: File number
318  int numOfFiles, // In: Total number of files expected
319  int maxNumOfFrames, // In: Maximum number of frames expected
320  char *fileName, // In: Name of file to process
321  int extensionNumber, // In: Extension number of the IMAGE_DATA
322  ImageFormat *format, // In: Pointer to the image format
323  float *arrayPixel, // In: Pointer to the accumulated data
324  DetRonNoise *noiseProfile, // Ou: Pointer to the detector noise data structure
325  int *error) // Ou: Error status
326 
327 {
328 
329  // Local Declarations
330  // ------------------
331  const char routine[] = "compressDetRon";
332  qfits_table *pTable=NULL;
333  short int *inData;
334  char *tempStr, *dataName;
335  int i, k, frame, foundData = 0, indexData, pixel, subWindowSize, scalingOffset;
336  static int indexFrame;
337  float *arrayFrame, standDev, variance;
338 
339 
340  // Algorithm
341  // ---------
342  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
343  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
344 
345  // Reset status
346  *error = 0;
347  subWindowSize = format->iXWidth * format->iYWidth;
348  if (fileNumber == 1) indexFrame = 0;
349 
350  // Open IMAGING_DATA = INDEX 2
351  pTable = qfits_table_open (fileName, extensionNumber);
352  if (!pTable)
353  {
354  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load IMAGING_DATA");
355  *error = 1;
356  return;
357  }
358 
359  // Get data table information
360  for (i = 0; i < pTable->nc; i++)
361  {
362  if (strcmp (pTable->col[i].tlabel, "DATA1") == 0)
363  {
364  foundData = 1;
365  indexData = i;
366  }
367  }
368  if (foundData == 0)
369  {
370  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find requested columns in data FITS file");
371  qfits_table_close (pTable);
372  *error = 1;
373  return;
374  }
375 
376  // Read column DATA
377  inData = (short int*) qfits_query_column (pTable, indexData, NULL);
378 
379  // Get the scaling offset
380  dataName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
381  for (i = 14; i < 25; i++)
382  {
383  sprintf (dataName, "TZERO%d", i);
384  tempStr = qfits_query_ext (fileName, dataName, extensionNumber);
385  if (tempStr != NULL)
386  {
387  if (diagnostic)cpl_msg_info(cpl_func,"Scaling Offset = %s\n", tempStr);
388  if (diagnostic) fprintf (midiReportPtr, "Scaling Offset = %s\n", tempStr);
389  sscanf (tempStr, "%d", &scalingOffset);
390  break;
391  }
392  }
393  if (tempStr == NULL)
394  {
395  scalingOffset = 0;
396  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Scaling Offset. It is set to 0");
397  }
398  free (dataName);
399 
400  // Now load or/and compress
401  for (pixel = 0; pixel < subWindowSize; pixel++)
402  {
403  // Along the time axis (frame)
404  for (frame = 0; frame < format->numOfFrames; frame++)
405  {
406  // Load each pixel data along the time axis
407  i = frame * subWindowSize + pixel;
408  k = (frame+indexFrame) * subWindowSize + pixel;
409  if (isnan (inData[i]))
410  {
411  sprintf (midiMessage, "inData has an INVALID value at frame %d", (indexFrame+frame));
412  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
413  }
414  noiseProfile->aveImage[pixel] += (float) (inData[i] + scalingOffset);
415  arrayPixel[k] = (float) (inData[i] + scalingOffset);
416  }
417  }
418 
419  // When all files have been read, compute an average
420  // image and create an image from pixels standard deviation
421  if (fileNumber == numOfFiles)
422  {
423  arrayFrame = (float *) calloc (maxNumOfFrames, sizeof (float));
424  for (pixel = 0; pixel < subWindowSize; pixel++)
425  {
426  noiseProfile->aveImage[pixel] /= maxNumOfFrames;
427 
428  // Along the time axis (frame)
429  for (frame = 0; frame < maxNumOfFrames; frame++)
430  {
431  // Load each pixel data along the time axis
432  i = frame * subWindowSize + pixel;
433  arrayFrame[frame] = arrayPixel[i];
434  }
435 
436  // Compute the standard deviation for the pixel array
437  variance = signalVariance (arrayFrame, 0, maxNumOfFrames, &standDev);
438  noiseProfile->pattern[pixel] = standDev;
439  }
440  free (arrayFrame);
441  }
442 
443  // Set it correctly for the next call to this function
444  indexFrame += format->numOfFrames;
445 
446  // Release memory
447  qfits_table_close (pTable);
448  free (inData);
449 
450  return;
451 }
452 /*****************************************************************************/
453 
454 
455 
456 /******************************************************************************
457 * European Southern Observatory
458 * VLTI MIDI Maintenance Templates Software
459 *
460 * Module name: createNoisePattern
461 * Input/Output: See function arguments to avoid duplication
462 * Description: Creates a noise pattern made up of standard deviations of image pixels
463 *
464 * History:
465 * 15-Jun-04 (csabet) Created
466 ******************************************************************************/
467 void createNoisePattern (
468  int fileNumber, // In: File number
469  char *inFile, // In: Name of the input FITS file
470  ImageFormat *format, // In: Pointer to the image format
471  DetRonNoise *noiseProfile, // Ou: Pointer to the detector noise data structure
472  int *error) // Ou: Error status
473 
474 {
475 
476  // Local Declarations
477  // ------------------
478  const char routine[] = "createNoisePattern";
479  char *title, *fileName;
480  int subWindowSize;
481  cpl_image *cplImage;
482 
483 
484  // Algorithm
485  // ---------
486  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
487  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
488 
489  // Reset status
490  *error = 0;
491  subWindowSize = format->iXWidth * format->iYWidth;
492 
493  // Create an image FITS file
494  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
495  fileName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
496  sprintf (title, "AveImg");
497  sprintf (fileName, "file %d", fileNumber);
498  createFitsImage (fileName, title, inFile, format->iXWidth, format->iYWidth, noiseProfile->aveImage);
499  free (fileName);
500  free (title);
501 
502  // Compute the median of the noise pattern
503  cplImage = cpl_image_wrap_float (format->iXWidth, format->iYWidth, noiseProfile->pattern);
504  noiseProfile->median = cpl_image_get_median (cplImage);
505  cpl_image_unwrap (cplImage);
506 
507  if (diagnostic)cpl_msg_info(cpl_func," Median of the noise pattern = %8.4f\n", noiseProfile->median);
508  fprintf (midiReportPtr, " Median of the noise pattern = %8.4f\n", noiseProfile->median);
509 
510  // Create plot file
511  if (plotFile) midiCreatePlotFile3D ("3dSigmaMap", "Sigma Map", "X", "Y", "Sigma",
512  0, noiseProfile->pattern, format->iXWidth, format->iYWidth, "lines", "0");
513 
514  return;
515 }
516 /*****************************************************************************/
517