MIDI Pipeline Reference Manual  2.8.3
procAcq.c
1 
2 /******************************************************************************
3 *******************************************************************************
4 * European Southern Observatory
5 * VLTI MIDI Data Reduction Image Qulaity Software
6 *
7 * Module name: procAcq.c
8 * Description: Contains routines for processing the Image Quality
9 *
10 * History:
11 * 25-Apr-05 (csabet) created
12 *******************************************************************************
13 ******************************************************************************/
14 
15 /******************************************************************************
16 * Compiler directives
17 ******************************************************************************/
18 
19 /******************************************************************************
20 * Include files
21 ******************************************************************************/
22 #include <stdio.h>
23 #include <cpl.h>
24 #include <math.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include "midiGlobal.h"
28 #include "qfits.h"
29 #include "midiLib.h"
30 #include "imageProcessing.h"
31 #include "midiFitsUtility.h"
32 #include "createProdAcq.h"
33 #include "diagnostics.h"
34 #include "memoryHandling.h"
35 #include "errorHandling.h"
36 #include "procAcq.h"
37 
38 /**********************************************************
39 * Constant definitions
40 **********************************************************/
41 
42 /**********************************************************
43 * Global Variables
44 **********************************************************/
45 
46 /*============================ C O D E A R E A ===========================*/
47 
48 
49 
50 
51 /******************************************************************************
52 * European Southern Observatory
53 * VLTI MIDI Data Reduction Software
54 *
55 * Module name: procAcq
56 * Input/Output: See function arguments to avoid duplication
57 * Description: Performs complete post processing for the Image Quality mode
58 *
59 * History:
60 * 25-Apr-05 (csabet) Created
61 ******************************************************************************/
62 void procAcq (
63  MidiFiles *fileNames, // In: Pointer to file names
64  int *error) // Ou: Status
65 {
66 
67  // Local Declarations
68  // ------------------
69  const char routine[] = "procAcq";
70  char *fileTemp, *classification;
71  FILE *inFitsBatchPtr;
72  ImageFormat *format;
73  int extNumOfImagingDataFile;
74  ImageQuality *image=NULL;
75  FILE *signaturePtr=NULL;
76 
77  // Algorithm
78  // ---------
79  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
80  if (diagnostic > 4) fprintf(midiReportPtr, "Invoking routine '%s' \n", routine);
81 
82  cpl_msg_info(cpl_func,"\nProcessing Image Quality for batch %d \n", batchNumber);
83  cpl_msg_info(cpl_func,"---------------------------------- \n");
84  fprintf (midiReportPtr, "\nProcessing Image Quality for batch %d \n", batchNumber);
85  fprintf (midiReportPtr, "---------------------------------- \n");
86 
87  // Write a signature
88  signaturePtr = fopen ("MIDI_sig_acq.log", "w");
89  fclose (signaturePtr);
90 
91  // Allocate memory
92  classification = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
93  fileTemp = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
94  format = callocImageFormat ();
95 
96  // Reset status
97  *error = 0;
98 
99  // Open the list of files
100  if ((inFitsBatchPtr = fopen (fileNames->inFitsBatch, "r")) == NULL)
101  {
102  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot open input FITS file list");
103  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "No data preprocessing has been carried out for this batch");
104  freeImageFormat (format);
105  free (fileTemp);
106  free (classification);
107  *error = 1;
108  return;
109  }
110 
111  // Extract file name and its classification
112  fgets (fileTemp, MAX_STRING_LENGTH, inFitsBatchPtr);
113  sprintf (classification, "%s", "");
114  sscanf (fileTemp, "%s%s", fileNames->inFitsName, classification);
115  if (diagnostic)cpl_msg_info(cpl_func,"\n Processing file %s \n", fileNames->inFitsName);
116  fprintf(midiReportPtr, "\n Processing file %s \n", fileNames->inFitsName);
117 
118  // Get 'extNumOfImagingDataFile' extension number of IMAGING_DATA in input file
119  extNumOfImagingDataFile = findImagingDataExtension (fileNames->inFitsName, TAB_IMAGING_DATA, error);
120  if (*error || !extNumOfImagingDataFile)
121  {
122  fclose (inFitsBatchPtr);
123  freeImageFormat (format);
124  free (fileTemp);
125  free (classification);
126  return;
127  }
128 
129  // Get Image Format parameters
130  getImageFormat (fileNames->inFitsName, extNumOfImagingDataFile, format, error);
131  if (*error || !(format->hasData))
132  {
133  fclose (inFitsBatchPtr);
134  freeImageFormat (format);
135  free (fileTemp);
136  free (classification);
137  return;
138  }
139 
140  // Check Categ, Tech and Type and then compute the image size
141  if (((strcmp (format->obsCatg, "CALIB") == 0) ||
142  (strcmp (format->obsCatg, "SCIENCE") == 0)) &&
143  ((strcmp (format->obsTech, "IMAGE,WINDOW") == 0) ||
144  (strcmp (format->obsType, "COARSE,OBJECT") == 0)))
145  {
146  // Compress Acquisition Image
147  image = callocImageQuality (format);
148  compressAcq (fileNames->inFitsName, extNumOfImagingDataFile, image, format, error);
149  if (*error)
150  {
151  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot compress data in FITS file");
152  fclose (inFitsBatchPtr);
153  freeImageQuality (format, image);
154  freeImageFormat (format);
155  free (fileTemp);
156  free (classification);
157  return;
158  }
159 
160  // Evaluate Image quality
161  assessImageQuality (fileNames->inFitsName, image, format, error);
162  if (*error)
163  {
164  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot assess image quality in FITS file");
165  fclose (inFitsBatchPtr);
166  freeImageQuality (format, image);
167  freeImageFormat (format);
168  free (fileTemp);
169  free (classification);
170  return;
171  }
172 
173  // Create Output products
174  createAcqProd (fileNames, format, image, error);
175  if (*error)
176  {
177  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot create product FITS file");
178  fclose (inFitsBatchPtr);
179  freeImageQuality (format, image);
180  freeImageFormat (format);
181  free (fileTemp);
182  free (classification);
183  return;
184  }
185  freeImageQuality (format, image);
186  }
187  else
188  {
189  *error = 1;
190  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Unknown Catg, Type, or Tech");
191  }
192 
193  // Close the file list
194  fclose (inFitsBatchPtr);
195 
196  // Release memory
197  freeImageFormat (format);
198  free (fileTemp);
199  free (classification);
200 
201  return;
202 }
203 /*****************************************************************************/
204 
205 
206 
207 /******************************************************************************
208 * European Southern Observatory
209 * VLTI MIDI Maintenance Templates Software
210 *
211 * Module name: compressAcq
212 * Input/Output: See function arguments to avoid duplication
213 * Description: Prepares Image data
214 *
215 * History:
216 * 25-Apr-05 (csabet) Created
217 ******************************************************************************/
218 void compressAcq (
219  char *inFitsFile, // In: Name of the input FITS file
220  int extNumOfImagingDataFile,// In: Extension number of the IMAGING_DATA in input file
221  ImageQuality *compressed, // In: Compressed data
222  ImageFormat *format, // In: File format
223  int *error) // Ou: Error status
224 {
225 
226  // Local Declarations
227  // ------------------
228  const char routine[] = "compressAcq";
229  qfits_table *pTable=NULL;
230  short int **inData;
231  char *inTarType=NULL, *tempStr, *dataName, lastTT;
232  int i, pixel, *foundData, scalingOffset, *indexData, numOfSkyFrames, numOfFramesUsed,
233  frame, region, indexTarType, foundTarType=0, transitions, numOfUndefinedFrames, numOfBadFrames,
234  tartypMult = 2; // Signifies that there are 2 (instead of 1!) characters per tartyp value (only
235  // first is useful). In future, if the problem producing the MIDI files is solved,
236  // can be changed to 1 (only for newer files) or made variable (with some way to
237  // determine whether it is an "old" type file). Note: this problem really has
238  // nothing to do with Qfits, but with the (unintended) way the files are written.
239 
240 
241  // Algorithm
242  // ---------
243  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
244  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
245 
246  // Reset status
247  *error = 0;
248 
249  // Allocate memory
250  inData = (short int **) calloc (format->numOfDetectorRegions, sizeof (short int *));
251  foundData = (int *) calloc (format->numOfDetectorRegions, sizeof (int));
252  indexData = (int *) calloc (format->numOfDetectorRegions, sizeof (int));
253  dataName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
254 
255  pTable = qfits_table_open (inFitsFile, extNumOfImagingDataFile);
256  if (!pTable)
257  {
258  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load IMAGING_DATA from");
259  free (inData);
260  free (foundData);
261  free (indexData);
262  free (dataName);
263  *error = 1;
264  return;
265  }
266 
267  // Get data table information
268  for (region = 0; region < format->numOfDetectorRegions; region++)
269  {
270  foundData[region] = 0;
271  indexData[region] = 0;
272  }
273  for (i = 0; i < pTable->nc; i++)
274  {
275  for (region = 0; region < format->numOfDetectorRegions; region++)
276  {
277  sprintf (dataName, "DATA%d", region+1);
278  if (strcmp (pTable->col[i].tlabel, dataName) == 0)
279  {
280  foundData[region] = 1;
281  indexData[region] = i;
282  if (diagnostic)cpl_msg_info(cpl_func,"Found 'DATA%d' at column %d in data file %s \n", region+1, i+1, inFitsFile);
283  if (diagnostic) fprintf(midiReportPtr, "Found 'DATA%d' at column %d in data file %s \n", region+1, i+1, inFitsFile);
284  }
285  }
286  if (strcmp (pTable->col[i].tlabel, "TARTYP2") == 0) // Now changed to TARTYP2, for all newer files
287  {
288  foundTarType = 1;
289  indexTarType = i;
290  }
291  }
292 
293  // Now issue warnings
294  if (foundTarType == 0)
295  {
296  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find TARTYP2 in data FITS file");
297  *error = 1;
298  }
299  for (region = 0; region < format->numOfDetectorRegions; region++)
300  {
301  if (foundData[region] == 0)
302  {
303  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find requested DATA column in data FITS file");
304  *error = 1;
305  }
306  }
307 
308  // Release memory and exit this module if there has been a warning. No point going any further
309  if (*error)
310  {
311  qfits_table_close (pTable);
312  free (inData);
313  free (foundData);
314  free (indexData);
315  free (dataName);
316  return;
317  }
318 
319  // Get TARTYP2
320  inTarType = (char *) qfits_query_column (pTable, indexTarType, NULL);
321  for (frame = 0; frame < format->numOfFrames; frame++)
322  compressed->tarType[frame] = inTarType[frame*tartypMult]; // New way around tartype problem (JM)
323 
324  // Check if the data is chopped
325  transitions = 0;
326  lastTT = compressed->tarType[0];
327  for (frame = 0; frame < format->numOfFrames; frame++)
328  {
329  if (compressed->tarType[frame] == lastTT) continue;
330  transitions++; // INCLUDES transitions to U etc.
331  lastTT = compressed->tarType[frame];
332  }
333  if ((format->chopped = (transitions > 3))) // Apparently is a chopped file
334  {
335  if (diagnostic)cpl_msg_info(cpl_func,"\nAcquision data is CHOPPED, since %d 'TARTYP2' transitions were found \n", transitions);
336  fprintf (midiReportPtr, "\nAcquision data is CHOPPED, since %d 'TARTYP2' transitions were found \n", transitions);
337  }
338  else
339  {
340  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Acquisition data is NOT chopped");
341  *error = 1;
342  qfits_table_close (pTable);
343  free (inData);
344  free (foundData);
345  free (indexData);
346  free (dataName);
347  return;
348  }
349 
350  // Read all regions from input files
351  for (region = 0; region < format->numOfDetectorRegions; region++)
352  inData[region] = (short int*) qfits_query_column (pTable, indexData[region], NULL);
353 
354  // Get the scaling offset
355  for (i = 14; i < 25; i++)
356  {
357  sprintf (dataName, "TZERO%d", i);
358  tempStr = qfits_query_ext (inFitsFile, dataName, extNumOfImagingDataFile);
359  if (tempStr != NULL)
360  {
361  if (diagnostic)cpl_msg_info(cpl_func,"Scaling Offset = %s\n", tempStr);
362  if (diagnostic) fprintf (midiReportPtr, "Scaling Offset = %s\n", tempStr);
363  sscanf (tempStr, "%d", &scalingOffset);
364  break;
365  }
366  }
367  if (tempStr == NULL)
368  {
369  scalingOffset = 0;
370  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Scaling Offset. It is set to 0");
371  }
372 
373  // Compression starts here
374  for (region = 0; region < format->numOfDetectorRegions; region++)
375  {
376  for (pixel = 0; pixel < format->subWindowSize; pixel++)
377  {
378  // Along the time axis (frame)
379  numOfUndefinedFrames = 0;
380  numOfBadFrames = 0;
381  numOfFramesUsed = 0;
382  numOfSkyFrames = 0;
383  (compressed->aveImage)[region][pixel] = 0.0;
384  (compressed->aveImageSky)[region][pixel] = 0.0;
385  for (frame = 0; frame < format->numOfFrames; frame++)
386  {
387  // Only consider Target, "T" and Sky, "S"
388  if (compressed->tarType[frame] == 'U')
389  {
390  if (diagnostic > 2 && !pixel)
391  {
392  sprintf (midiMessage, "Found undefined, 'U' 'TARTYP2' at frame %d of DATA%d in %s \n",
393  frame+1, region+1, inFitsFile);
394  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
395  }
396  numOfUndefinedFrames++;
397  continue;
398  }
399 
400  // Load each pixel data along the time axis
401  i = frame * format->subWindowSize + pixel;
402  if (isnan (inData[region][i]))
403  {
404  numOfBadFrames++;
405  sprintf (midiMessage, "Found bad pixel %d of DATA%d in %s \n", i, region, inFitsFile);
406  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
407  }
408  else
409  {
410  numOfFramesUsed++;
411  (compressed->aveImage)[region][pixel] += (float) (inData[region][i]);
412  if (compressed->tarType[frame] == 'S')
413  {
414  numOfSkyFrames++;
415  (compressed->aveImageSky)[region][pixel] += (float) (inData[region][i]);
416  }
417  }
418  }
419 
420  // Compute average image
421  if (diagnostic && !pixel)
422  {
423  cpl_msg_info(cpl_func,"\nData Statistics \n");
424  cpl_msg_info(cpl_func,"--------------- \n");
425  cpl_msg_info(cpl_func,"Number of frames used in DATA%d = %d \n", region+1, numOfFramesUsed);
426  cpl_msg_info(cpl_func,"Number of Sky frames in DATA%d = %d \n", region+1, numOfSkyFrames);
427  cpl_msg_info(cpl_func,"Number of Bad frames in DATA%d = %d \n", region+1, numOfBadFrames);
428  cpl_msg_info(cpl_func,"Number of Undefined frames in DATA%d = %d \n", region+1, numOfUndefinedFrames);
429  fprintf (midiReportPtr, "\nData Statistics \n");
430  fprintf (midiReportPtr, "--------------- \n");
431  fprintf (midiReportPtr, "Number of frames used in DATA%d = %d \n", region+1, numOfFramesUsed);
432  fprintf (midiReportPtr, "Number of Sky frames in DATA%d = %d \n", region+1, numOfSkyFrames);
433  fprintf (midiReportPtr, "Number of Bad frames in DATA%d = %d \n", region+1, numOfBadFrames);
434  fprintf (midiReportPtr, "Number of Undefined frames in DATA%d = %d \n", region+1, numOfUndefinedFrames);
435  }
436  if (numOfFramesUsed) (compressed->aveImage)[region][pixel] /= numOfFramesUsed; // Target + Sky
437  (compressed->aveImage)[region][pixel] += scalingOffset; // Target + Sky
438  if (numOfSkyFrames) (compressed->aveImageSky)[region][pixel] /= numOfSkyFrames; // Sky
439  (compressed->aveImageSky)[region][pixel] += scalingOffset; // Sky
440  (compressed->aveImage)[region][pixel] -= (compressed->aveImageSky)[region][pixel]; // Target
441  }
442  }
443 
444  // Clean up now:
445  for (region = 0; region < format->numOfDetectorRegions; region++) free (inData[region]);
446  if (pTable) qfits_table_close (pTable);
447  if (inTarType) free(inTarType);
448  free (inData);
449  free (foundData);
450  free (indexData);
451  free (dataName);
452 
453  return;
454 }
455 /*****************************************************************************/
456 
457 
458 
459 /******************************************************************************
460 * European Southern Observatory
461 * VLTI MIDI Maintenance Templates Software
462 *
463 * Module name: assessImageQuality
464 * Input/Output: See function arguments to avoid duplication
465 * Description: Evaluates the image quality in acquisition mode
466 *
467 * History:
468 * 25-Apr-05 (csabet) Created
469 ******************************************************************************/
470 void assessImageQuality (
471  char *fileName, // In: Input FITS name
472  ImageQuality *compressed, // In: Compressed data
473  ImageFormat *format, // In: File format
474  int *error) // Ou: Error status
475 
476 {
477 
478  // Local Declarations
479  // ------------------
480  const char routine[] = "assessImageQuality";
481  int region, searchSize;
482  char *title=NULL, *fileString=NULL;
483  MidiCoords *target;
484 
485  // Algorithm
486  // ---------
487  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
488  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
489 
490  // Reset status
491  *error = 0;
492  searchSize = SIZE_SEARCH_ACQ;
493 
494  // Allocate memory
495  target = (MidiCoords *) calloc (1, sizeof (MidiCoords));
496 
497  // Analyse iamges
498  for (region = 0; region < format->numOfDetectorRegions; region++)
499  {
500  // Create image FITS files
501  if (diagnostic)
502  {
503  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
504  fileString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
505  sprintf (title , "AveImgSkyDATA%d", region+1);
506  sprintf (fileString , "region %d", region+1);
507  createFitsImage (fileString, title, fileName, format->iXWidth, format->iYWidth,
508  (compressed->aveImageSky)[region]);
509  sprintf (title , "AveImgDATA%d", region+1);
510  createFitsImage (fileString, title, fileName, format->iXWidth, format->iYWidth,
511  (compressed->aveImage)[region]);
512  free (fileString);
513  free (title);
514  }
515 
516  if (plotFile)
517  {
518  fileString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
519  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
520  sprintf (fileString, "3dImgDATA%d", region+1);
521  sprintf (title, "Dark removed, region %d", region+1);
522  midiCreatePlotFile3D (fileString, title, "X", "Y", "Flux", 0,
523  (compressed->aveImage)[region], format->iXWidth, format->iYWidth, "lines", "3");
524  if (diagnostic)
525  {
526  sprintf (fileString, "3dImgSkyDATA%d", region+1);
527  sprintf (title, "Sky, region %d", region+1);
528  midiCreatePlotFile3D (fileString, title, "X", "Y", "Flux", 0,
529  (compressed->aveImageSky)[region], format->iXWidth, format->iYWidth, "lines", "3");
530  }
531  free (fileString);
532  free (title);
533  }
534 
535  // Determine target size and location
536  midiGetFWHM (region+1, (compressed->aveImage)[region],
537  format->iXWidth, format->iYWidth, searchSize, &(compressed->coordX[region]),
538  &(compressed->coordY[region]), &(compressed->sizeX[region]), &(compressed->sizeY[region]), error);
539  cpl_msg_info(cpl_func,"Target Coordinates and size (X, Y, X-Width, Y-Width): %0.2f, %0.2f, %0.2f %0.2f \n",
540  compressed->coordX[region], compressed->coordY[region], compressed->sizeX[region], compressed->sizeY[region]);
541  fprintf (midiReportPtr, "Target Coordinates and size (X, Y, X-Width, Y-Width): %0.2f, %0.2f, %0.2f %0.2f \n",
542  compressed->coordX[region], compressed->coordY[region], compressed->sizeX[region], compressed->sizeY[region]);
543 
544  // Compute target Intensity
545  target->xCoord = compressed->coordX[region] - (0.5 * compressed->sizeX[region]);
546  target->yCoord = compressed->coordY[region] - (0.5 * compressed->sizeY[region]);
547  target->dxCoord = compressed->coordX[region] + (0.5 * compressed->sizeX[region]);
548  target->dyCoord = compressed->coordY[region] + (0.5 * compressed->sizeY[region]);
549 // cpl_msg_info(cpl_func,"Target Coordinates (x, y, dx, dy): %f, %f, %f %f \n", target->xCoord, target->yCoord,
550 // target->dxCoord, target->dyCoord);
551 // fprintf (midiReportPtr, "Target Coordinates (x, y, dx, dy): %f, %f, %f %f \n", target->xCoord, target->yCoord,
552 // target->dxCoord, target->dyCoord);
553 
554  computeImageFlux ((compressed->aveImage)[region], format, target, &((compressed->targetFlux)[region]),
555  &((compressed->targetPixelCount)[region]), error);
556  cpl_msg_info(cpl_func,"Target flux in region %d = %0.2f \n", region+1, (compressed->targetFlux)[region]);
557  cpl_msg_info(cpl_func,"Target pixel count in region %d = %d \n", region+1, (compressed->targetPixelCount)[region]);
558  fprintf (midiReportPtr, "Target flux in region %d = %0.2f \n", region+1, (compressed->targetFlux)[region]);
559  fprintf (midiReportPtr, "Target pixel count in region %d = %d \n", region+1, (compressed->targetPixelCount)[region]);
560  }
561 
562  // Release memory
563  free (target);
564 
565  return;
566 }
567 /*****************************************************************************/
568