MIDI Pipeline Reference Manual  2.8.3
preProcKappa.c
1 /******************************************************************************
2 *******************************************************************************
3 * European Southern Observatory
4 * VLTI MIDI Data Reduction Software
5 *
6 * Module name: preProcKappa.c
7 * Description: Contains routines for pre processing files related to
8 * Kappa computation
9 *
10 * History:
11 * 05-Dec-05 (csabet) Created
12 *******************************************************************************
13 ******************************************************************************/
14 
15 /******************************************************************************
16 * Compiler directives
17 ******************************************************************************/
18 #define SIMULATE_DATA_x // Only used when proper data was not available
19 
20 /******************************************************************************
21 * Include files
22 ******************************************************************************/
23 #include <stdio.h>
24 #include <cpl.h>
25 #include <math.h>
26 #include "midiGlobal.h"
27 #include "midiLib.h"
28 #include "memoryHandling.h"
29 #include "errorHandling.h"
30 #include "midiFitsUtility.h"
31 #include "diagnostics.h"
32 #include "preProcKappa.h"
33 
34 /**********************************************************
35 * Constant definitions
36 **********************************************************/
37 
38 /**********************************************************
39 * Global Variables
40 **********************************************************/
41 
42 /*============================ C O D E A R E A ===========================*/
43 
44 
45 /******************************************************************************
46 * European Southern Observatory
47 * VLTI MIDI Data Reduction Software
48 *
49 * Module name: preProcKappa
50 * Input/Output: See function arguments to avoid duplication
51 * Description: Prepares the data for the DISPERSED modes of processing.
52 * This is configured for SCI_PHOT Kappa computation
53 *
54 * History:
55 * 05-Dec-05 (csabet) Created
56 ******************************************************************************/
57 void preProcKappa (
58  UserOptions *options, // In: User parameters
59  FilterData *filterInfo, // IO: Pointer to the filter data structure
60  MidiFiles *fileNames, // In: Pointer to the MIDI file structure
61  CompressedData *compressedInterfA, // Ou: Pointer to the compressed interferometry data structure
62  CompressedData *compressedInterfB, // Ou: Pointer to the compressed interferometry data structure
63  CompressedData *compressedPhotomA, // Ou: Pointer to the compressed photom A data structure
64  CompressedData *compressedPhotomB, // Ou: Pointer to the compressed photom B data structure
65  ImageFormat *formatInterfA, // In: Interf size parameters
66  ImageFormat *formatInterfB, // In: Interf size parameters
67  ImageFormat *formatPhotomA, // In: PhotA size parameters
68  ImageFormat *formatPhotomB, // In: PhotB size parameters
69  int *error) // Ou: Error status
70 {
71 
72  // Local Declarations
73  // ------------------
74  const char routine[] = "preProcKappa";
75  FILE *inFitsBatchPtr=NULL;
76  int newFile, extNumOfImagingDataFile, extNumOfImagingDataMask, R;
77  char *maskFile, *cleanString, *classification, *stringTemp, *fileName, *title;
78  unsigned int loopCount = 0;
79  ImageFormat *localFormat;
80  enum ObsTechnique obsTech;
81 
82  // Algorithm
83  // ---------
84  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
85  if (diagnostic > 4) fprintf(midiReportPtr, "Invoking routine '%s' \n", routine);
86 
87  cpl_msg_info(cpl_func,"\nReducing data of batch %d \n", batchNumber);
88  cpl_msg_info(cpl_func,"---------------------- \n");
89  fprintf (midiReportPtr, "\nReducing data of batch %d \n", batchNumber);
90  fprintf (midiReportPtr, "---------------------- \n");
91 
92  // Reset status
93  *error = 0;
94  compressedInterfA->exists = 0;
95  compressedInterfB->exists = 0;
96  compressedPhotomA->exists = 0;
97  compressedPhotomB->exists = 0;
98  newFile = 1;
99  obsTech = UNKNOWN;
100 
101  // Allocate memory
102  stringTemp = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
103  classification = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
104  cleanString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
105  localFormat = callocImageFormat ();
106  maskFile = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
107 
108  // Open the list of files
109  if ((inFitsBatchPtr = fopen (fileNames->inFitsBatch, "r")) == NULL)
110  {
111  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
112  "Cannot open input FITS file list. No compression has been carried out for this batch");
113  free (cleanString);
114  free (localFormat);
115  free (stringTemp);
116  free (classification);
117  freeImageFormat (localFormat);
118  *error = 1;
119  return;
120  }
121 
122  // Loop through the list of files and compress data
123  while (fgets (stringTemp, MAX_STRING_LENGTH, inFitsBatchPtr) != NULL)
124  {
125  sprintf (classification, "%s", "");
126  sscanf (stringTemp, "%s%s", fileNames->inFitsName, classification);
127  cpl_msg_info(cpl_func,"\nProcessing file %s \n", fileNames->inFitsName);
128  fprintf (midiReportPtr, "\nProcessing file %s \n", fileNames->inFitsName);
129 
130  // Get 'extNumOfImagingDataFile' extension number of IMAGING_DATA in input file
131  extNumOfImagingDataFile = findImagingDataExtension (fileNames->inFitsName, TAB_IMAGING_DATA, error);
132  if (*error) break;
133 
134  // Get Image Format parameters
135  if (extNumOfImagingDataFile > 0)
136  {
137  getImageFormat (fileNames->inFitsName, extNumOfImagingDataFile, localFormat, error);
138  if (*error) break;
139  }
140  else localFormat->hasData = 0;
141 
142  // Get filter data and mask file name from the first FITS file
143  if (loopCount == 0)
144  {
145  getFilterData (fileNames->inFitsName, filterInfo, error);
146  if (*error) break;
147  else fprintf (midiReportPtr, "First file of batch = %s (QCLOG)\n", fileNames->inFitsName);
148 
149  // Select mask file
150  selectMask (options->maskMode, fileNames, maskFile, error);
151  if (*error) break;
152 
153  // Get 'extNumOfImagingDataMask' extension number of IMAGING_DATA in mask file
154  extNumOfImagingDataMask = findImagingDataExtension (maskFile, MASK_IMAGING_DATA, error);
155  if (*error) break;
156 
157  cpl_msg_info(cpl_func,"Mask File is %s\n", maskFile);
158  fprintf( midiReportPtr, "Mask File is %s\n", maskFile);
159  }
160 
161  // Check if the file has data
162  if (localFormat->hasData)
163  {
164  // Check file Category and Type. For kappa computation, only Photometry files are used
165  if ((strcmp (localFormat->obsTech, "INTERFEROMETRY") == 0))
166  {
167  if (diagnostic > 1) midiReportInfo (midiReportPtr,
168  routine, __FILE__, __LINE__, "The above Interferometry file will not be processed");
169  obsTech = INTERF;
170  }
171  else if ((strcmp (localFormat->obsTech, "IMAGE,WINDOW,CHOPNOD") == 0) &&
172  (strcmp(localFormat->shutterId,"AOPEN") == 0))
173  {
174  // Check previous TYPE
175  if (obsTech != PHOTOMA) newFile = 1;
176  else newFile = 0;
177 
178  // Set the TYPE flag
179  obsTech = PHOTOMA;
180 
181  // Check consistency
182  if ((formatPhotomA->numOfDetectorRegions != localFormat->numOfDetectorRegions) ||
183  (formatPhotomA->iXWidth != localFormat->iXWidth) ||
184  (formatPhotomA->iYWidth != localFormat->iYWidth))
185  {
186  midiReportWarning (midiReportPtr,
187  routine, __FILE__, __LINE__, "Expected format is incorrect");
188  *error = 1;
189  break;
190  }
191 
192  /* The Photometry files contribute to the computation of the
193  kappa matrix. Therefore here only formatPhotomA will be used */
194  organiseKappa ("AOPEN", newFile, extNumOfImagingDataFile, fileNames,
195  extNumOfImagingDataMask, maskFile, localFormat, formatInterfA,
196  compressedInterfA, compressedPhotomA, error);
197  if (*error) break;
198  newFile = 0; // So the next time through, will know not to start at frame 0
199  }
200  else if ((strcmp (localFormat->obsTech, "IMAGE,WINDOW,CHOPNOD") == 0) &&
201  (strcmp(localFormat->shutterId,"BOPEN") == 0))
202  {
203  // Check previous TYPE
204  if (obsTech != PHOTOMB) newFile = 1;
205  else newFile = 0;
206 
207  // Set the TYPE flag
208  obsTech = PHOTOMB;
209 
210  // Check consistency
211  if ((formatPhotomA->numOfDetectorRegions != localFormat->numOfDetectorRegions) ||
212  (formatPhotomA->iXWidth != localFormat->iXWidth) ||
213  (formatPhotomA->iYWidth != localFormat->iYWidth))
214  {
215  midiReportWarning (midiReportPtr,
216  routine, __FILE__, __LINE__, "Expected format is incorrect");
217  *error = 1;
218  break;
219  }
220 
221  /* The Photometry files contribute to the computation of the
222  kappa matrix. Therefore here only formatPhotomB will be used */
223  organiseKappa ("BOPEN", newFile, extNumOfImagingDataFile, fileNames,
224  extNumOfImagingDataMask, maskFile, localFormat, formatInterfB,
225  compressedInterfB, compressedPhotomB, error);
226  if (*error) break;
227  newFile = 0; // So the next time through, will know not to start at frame 0
228  }
229  else
230  {
231  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Unknown Catg, Type or Tech");
232  *error = 1;
233  break;
234  }
235  }
236  else
237  {
238  sprintf (midiMessage, "No data tables in %s. Not processed", fileNames->inFitsName);
239  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
240  }
241  loopCount++;
242  }
243 
244  // Close files and release memory
245  fclose (inFitsBatchPtr);
246  free (cleanString);
247  free (stringTemp);
248  free (classification);
249  freeImageFormat (localFormat);
250  free (maskFile);
251 
252  // Check if reduction has been carried out
253  cpl_msg_info(cpl_func,"\nCompression status \n");
254  cpl_msg_info(cpl_func,"------------------ \n");
255  if (compressedInterfA->exists)cpl_msg_info(cpl_func,"Created Compressed Interferometry A Data\n");
256  if (compressedInterfB->exists)cpl_msg_info(cpl_func,"Created Compressed Interferometry B Data\n");
257  if (compressedPhotomA->exists)cpl_msg_info(cpl_func,"Created Compressed Photometry A Data\n");
258  if (compressedPhotomB->exists)cpl_msg_info(cpl_func,"Created Compressed Photometry B Data\n");
259  cpl_msg_info(cpl_func,"\n");
260 
261  fprintf (midiReportPtr, "\nCompression status \n");
262  fprintf (midiReportPtr, "------------------ \n");
263  if (compressedInterfA->exists) fprintf (midiReportPtr, "Created Compressed Interferometry A Data\n");
264  if (compressedInterfA->exists) fprintf (midiReportPtr, "Created Compressed Interferometry B Data\n");
265  if (compressedPhotomA->exists) fprintf (midiReportPtr, "Created Compressed Photometry A Data\n");
266  if (compressedPhotomB->exists) fprintf (midiReportPtr, "Created Compressed Photometry B Data\n");
267  fprintf (midiReportPtr, "\n");
268 
269  if (!(compressedInterfA->exists) || !(compressedInterfB->exists) ||
270  !(compressedPhotomA->exists) || !(compressedPhotomB->exists) || *error)
271  {
272  *error = 1;
273  sprintf (midiMessage,
274  "Cannot continue. Need the following compressed data: InterfA, InterfB, PhotomA, PhotomB");
275  midiReportWarning (midiReportPtr,
276  routine, __FILE__, __LINE__, midiMessage);
277  return;
278  }
279 
280  // Display images
281  if (diagnostic > 1 && plotFile)
282  {
283  fileName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
284  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
285 
286  for (R = 0; R < formatInterfA->numOfRegionsToProcess; R++)
287  {
288  sprintf (fileName, "3dInterfADATA%d", R+2);
289  sprintf (title, "Interferometry A DATA %d (Masked and Sky removed)", R+2);
290  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
291  compressedInterfA->image[R], formatInterfA->iXWidth, formatInterfA->iYWidth, "lines", "3");
292  }
293 
294  for (R = 0; R < formatInterfB->numOfRegionsToProcess; R++)
295  {
296  sprintf (fileName, "3dInterfBDATA%d", R+2);
297  sprintf (title, "Interferometry B DATA %d (Masked and Sky removed)", R+2);
298  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
299  compressedInterfB->image[R], formatInterfB->iXWidth, formatInterfB->iYWidth, "lines", "3");
300  }
301 
302  for (R = 0; R < formatPhotomA->numOfRegionsToProcess; R++)
303  {
304  sprintf (fileName, "3dPhotomADATA%d", R+1);
305  sprintf (title, "Photometry A DATA %d (Masked and Sky removed)", R+1);
306  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
307  compressedPhotomA->image[R], formatPhotomA->iXWidth, formatPhotomA->iYWidth, "lines", "3");
308  }
309 
310  for (R = 0; R < formatPhotomB->numOfRegionsToProcess; R++)
311  {
312  sprintf (fileName, "3dPhotomBDATA%d", R+4);
313  sprintf (title, "Photometry B DATA %d (Masked and Sky removed)", R+4);
314  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
315  compressedPhotomB->image[R], formatPhotomB->iXWidth, formatPhotomB->iYWidth, "lines", "3");
316  }
317 
318  free (fileName);
319  free (title);
320  }
321 
322  return;
323 }
324 /*****************************************************************************/
325 
326 
327 
328 
329 /******************************************************************************
330 * European Southern Observatory
331 * VLTI MIDI Data Reduction Software
332 *
333 * Module name: organiseKappa
334 * Input/Output: See function arguments to avoid duplication
335 * Description: Prepares SCI_PHOT Kappa files for compression. All files may be
336 * split.
337 *
338 * History:
339 * 05-Dec-05 (csabet) Created
340 ******************************************************************************/
341 void organiseKappa (
342  const char *shutter, // In: Whether AOPEN or BOPEN
343  int newFile, // In: Flag indicating continuation
344  int extNumOfImagingDataFile, // In: Extention number
345  MidiFiles *fileNames, // In: Pointer to the MIDI file structure
346  int extNumOfImagingDataMask, // In: Extention number
347  char *maskFile, // In: Mask file
348  ImageFormat *localFormat, // In: Local format size
349  ImageFormat *formatInterf, // In: Interf size parameters
350  CompressedData *compressedInterf, // Ou: Pointer to the compressed interferometry data structure
351  CompressedData *compressedPhotom, // Ou: Pointer to the compressed photom A data structure
352  int *error) // Ou: Error status
353 {
354 
355  // Local Declarations
356  // ------------------
357  const char routine[] = "organiseKappa";
358  int frame0;
359 
360  // Algorithm
361  // ---------
362  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
363  if (diagnostic > 4) fprintf(midiReportPtr, "Invoking routine '%s' \n", routine);
364 
365  // Initialise
366  *error = 0;
367 
368  // Compress data
369  frame0 = compressKappa (shutter, fileNames->inFitsName, maskFile,
370  extNumOfImagingDataFile, extNumOfImagingDataMask, compressedInterf,
371  compressedPhotom, newFile, localFormat, formatInterf->numOfFrames, error);
372  if (*error) return;
373 
374  // Report compression status
375  if (diagnostic)
376  {
377  sprintf (midiMessage, "\nWrote %d frames into Interf and Photom, starting at frame %d\nThus now %d frames total,"
378  " expecting %d altogether. Error=%d \n\n", localFormat->numOfFrames, frame0,
379  localFormat->numOfFrames+frame0, formatInterf->numOfFrames, *error);
380  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
381  }
382 
383  // Set the flags
384  if ((frame0 + localFormat->numOfFrames) == formatInterf->numOfFrames) // ALL have been read in
385  {
386  compressedInterf->exists = 1; // Mark it as completed
387  compressedPhotom->exists = 1; // Mark it as completed
388  }
389 
390  return;
391 }
392 /*****************************************************************************/
393 
394 
395 
396 /******************************************************************************
397 * European Southern Observatory
398 * VLTI MIDI Data Reduction Software
399 *
400 * Module name: compressKappa
401 * Input/Output: See function arguments to avoid duplication
402 * Description: This routine compresses data files for kappa computation
403 * iDispFringe points to a 3D structure. One dimension is time (frame),
404 * the second dimension is wave (produced by collapsing each X, Y image
405 * into one line) and the third dimension is the number of detector regions.
406 *
407 * Note:
408 * if AOPEN
409 * ========
410 * DATA1 = PHOT A
411 * DATA2 = INTERF
412 * DATA3 = INTERF
413 * DATA4 = 0
414 *
415 * if BOPEN
416 * ========
417 * DATA1 = 0
418 * DATA2 = INTERF
419 * DATA3 = INTERF
420 * DATA4 = PHOT B
421 *
422 * All 4 data regions are chopped
423 *
424 * History:
425 * 05-Dec-05 (csabet) Created
426 ******************************************************************************/
427 int compressKappa ( // Ou: Status. Returns number of first frame compressed; -1 if error
428  const char *shutter, // In: Whether AOPEN or BOPEN
429  char *inFitsFile, // In: Name of the input FITS file
430  char *maskFile, // In: Name of the mask file
431  int extNumOfImagingDataFile,// In: Extension number of the IMAGING_DATA in input file
432  int extNumOfImagingDataMask,// In: Extension number of the IMAGING_DATA in mask file
433  CompressedData *compressedInterf, // IO: Compressed Interf data
434  CompressedData *compressedPhotom, // IO: Compressed Photom data
435  int newSet, // In: Flag indicating the arrival of a new set of FITS files
436  ImageFormat *format, // In: Split file format
437  int numOfFramesMax, // In: Maximum number of frames
438  int *error) // Ou: Error status
439 {
440 
441  // Local Declarations
442  // ------------------
443  const char routine[] = "compressKappa";
444  qfits_table *pTable = NULL, *pMask = NULL;
445  short int **inData, *inSteppingPhase;
446  float **inMask, accum, current,
447  *normalization; // Normalization factor. Becomes an array in X for the normalization of the
448  // compressedInterf data, valid for each subRegion then overwritten
449  char **inTARTYP = NULL, *tempStr, fitsColumnString[10], *dataName, *title=NULL, *fileString=NULL;
450  double *inTIME = NULL;
451  int frame0 = - 1; // Gets set to a valid number UNLESS routine terminates in error
452  double (*inLOCALOPD)[2] = NULL, (*inOPD)[2] = NULL;
453  static int aprioriSteppingPhase, iF, channelSelected = 0;
454  static double zeroTime;
455  int i, k, *foundData, foundSteppingPhase = 0, indexSteppingPhase, scalingOffset, *indexData,
456  *indexMask, maskWidthX, maskWidthY, maskSubWindow, F, X, Y, R,
457  *indexTARTYP, *foundTARTYP, startframe=0, frameOffset, indexOPD= -1, indexLOCALOPD= -1,
458  indexTIME= -1, maxstep,
459  tartypMult=2; // Signifies that there are 2 (instead of 1!) characters per tartyp value (only
460  // first is useful). In future, if the problem producing the MIDI files is solved,
461  // can be changed to 1 (only for newer files) or made variable (with some way to
462  // determine whether it is an "old" type file). Note: this problem really has
463  // nothing to do with Qfits, but with the (unintended) way the files are written.
464  int startRegion, endRegion, i2, fst, snd, found;
465 
466 
467  // Algorithm
468  // ---------
469  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
470  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
471 
472  // Reset status
473  *error = 0;
474  if (format->numOfDetectorRegions != 4)
475  {
476  *error = 1;
477  sprintf (midiMessage, "Incorrect number of regions. Expected 4, found %d", format->numOfDetectorRegions);
478  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
479  return (-1);
480  }
481 
482  // Determine region's start and end. See description
483  if (strcmp (shutter, "AOPEN") == 0)
484  {
485  startRegion = 0;
486  endRegion = format->numOfDetectorRegions - 1;
487  }
488  else
489  {
490  startRegion = 1;
491  endRegion = format->numOfDetectorRegions;
492  }
493 
494  // Allocate memory
495  inData = (short int **) calloc (format->numOfDetectorRegions, sizeof (short int *));
496  inTARTYP = (char **) calloc (format->numOfDetectorRegions, sizeof (char *));
497  inMask = (float **) calloc (format->numOfDetectorRegions, sizeof (float *)); // Now has each OUTPUT region allocated
498  foundData = (int *) calloc (format->numOfDetectorRegions, sizeof (int));
499  indexData = (int *) calloc (format->numOfDetectorRegions, sizeof (int));
500  foundTARTYP = (int *) calloc (format->numOfDetectorRegions, sizeof (int));
501  indexTARTYP = (int *) calloc (format->numOfDetectorRegions, sizeof (int));
502  indexMask = (int *) calloc (format->numOfDetectorRegions, sizeof (int));
503  normalization = (float *) calloc (format->iXWidth, sizeof (float));
504  dataName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
505 
506  // Initialise stepping phase for a new set of FITS files
507  if (newSet)
508  {
509  aprioriSteppingPhase = 0;
510  iF = 0;
511  }
512 
513  // Make sure the sum of the split-files numOfFrames is not more than the allowed maximum
514  if ((iF + format->numOfFrames) > numOfFramesMax)
515  format->numOfFrames = numOfFramesMax - iF;
516 
517  // Open IMAGING_DATA = INDEX 2
518  pMask = qfits_table_open (maskFile, extNumOfImagingDataMask);
519  if (!pMask)
520  {
521  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load Mask's IMAGING_DATA");
522  free (inData);
523  free (inTARTYP);
524  free (inMask);
525  free (foundData);
526  free (indexData);
527  free (foundTARTYP);
528  free (indexTARTYP);
529  free (indexMask);
530  free (normalization);
531  free (dataName);
532  *error = 1;
533  return (-1);
534  }
535  pTable = qfits_table_open (inFitsFile, extNumOfImagingDataFile);
536  if (!pTable)
537  {
538  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load Data's IMAGING_DATA");
539  qfits_table_close (pMask);
540  free (inData);
541  free (inTARTYP);
542  free (inMask);
543  free (foundData);
544  free (indexData);
545  free (foundTARTYP);
546  free (indexTARTYP);
547  free (indexMask);
548  free (normalization);
549  free (dataName);
550  *error = 1;
551  return (-1);
552  }
553 
554  // Get data table information
555  for (R = startRegion; R < endRegion; R++)
556  {
557  foundData[R] = 0;
558  indexData[R] = 0;
559  }
560  for (i = 0; i < pTable->nc; i++)
561  {
562  for (R = startRegion; R < endRegion; R++)
563  {
564  sprintf (dataName, "DATA%d", R+1);
565  if (strcmp (pTable->col[i].tlabel, dataName) == 0)
566  {
567  foundData[R] = 1;
568  indexData[R] = i;
569  if (diagnostic)
570  {
571  cpl_msg_info(cpl_func,"Found 'DATA%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
572  fprintf(midiReportPtr, "Found 'DATA%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
573  }
574  }
575 
576  sprintf (dataName, "TARTYP%d", R+1);
577  if (strcmp (pTable->col[i].tlabel, dataName) == 0)
578  {
579  foundTARTYP[R] = 1;
580  indexTARTYP[R] = i;
581  if (diagnostic)
582  {
583  cpl_msg_info(cpl_func,"Found 'TARTYP%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
584  fprintf(midiReportPtr, "Found 'TARTYP%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
585  }
586  }
587  }
588  if (strcmp (pTable->col[i].tlabel, "STEPPING_PHASE") == 0)
589  {
590  foundSteppingPhase = 1;
591  indexSteppingPhase = i;
592  }
593  if (strcmp (pTable->col[i].tlabel, "OPD") == 0)
594  {
595  indexOPD = i;
596  }
597  if (strcmp (pTable->col[i].tlabel, "LOCALOPD") == 0)
598  {
599  indexLOCALOPD = i;
600  }
601  if (strcmp (pTable->col[i].tlabel, "TIME") == 0)
602  {
603  indexTIME = i;
604  }
605  }
606 
607  // Now issue warnings
608  if (foundSteppingPhase == 0)
609  {
610  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find STEPPING_PHASE in data FITS file");
611  *error = 1;
612  }
613  if (indexOPD < 0)
614  {
615  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for OPD in data FITS file");
616  *error = 1;
617  }
618  if (indexLOCALOPD < 0)
619  {
620  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for LOCALOPD in data FITS file");
621  *error = 1;
622  }
623  if (indexTIME < 0)
624  {
625  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for TIME in data FITS file");
626  *error = 1;
627  }
628  for (R = startRegion; R < endRegion; R++)
629  {
630  if (foundData[R] == 0)
631  {
632  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
633  "Cannot find requested DATA column in data FITS file");
634  *error = 1;
635  }
636  if (foundTARTYP[R] == 0)
637  {
638  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
639  "Cannot find requested TARTYP column in data FITS file");
640  *error = 1;
641  }
642  }
643 
644  // Get mask data table information
645  for (R = startRegion; R < endRegion; R++)
646  {
647  foundData[R] = 0;
648  indexMask[R] = 0;
649  }
650  for (i = 0; i < pMask->nc; i++)
651  {
652  for (R = startRegion; R < endRegion; R++)
653  {
654  sprintf (dataName, "DATA%d", R+1);
655  if (strcmp (pMask->col[i].tlabel, dataName) == 0)
656  {
657  foundData[R] = 1;
658  indexMask[R] = i;
659  if (diagnostic)
660  {
661  cpl_msg_info(cpl_func,"Found 'DATA%d' at column %d in mask file %s \n", R+1, i+1, maskFile);
662  fprintf(midiReportPtr, "Found 'DATA%d' at column %d in mask file %s \n", R+1, i+1, maskFile);
663  }
664  }
665  }
666  }
667 
668  // Now issue warnings
669  for (R = startRegion; R < endRegion; R++)
670  {
671  if (foundData[R] == 0)
672  {
673  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
674  "Cannot find requested DATA column in mask FITS file");
675  *error = 1;
676  }
677  }
678 
679  // Get subwindow format for the mask file
680  sprintf (fitsColumnString, "TDIM%d", indexMask[1]+1); // Point it to the column of a valid region
681  tempStr = qfits_query_ext (maskFile, fitsColumnString, extNumOfImagingDataMask);
682  sscanf (tempStr, "'(%d,%d) '", &maskWidthX, &maskWidthY);
683  maskSubWindow = maskWidthX * maskWidthY;
684  if (diagnostic)cpl_msg_info(cpl_func,"Mask sub-window size = %d\n", maskSubWindow);
685  if (diagnostic)cpl_msg_info(cpl_func,"Data sub-window size = %d\n", format->subWindowSize);
686  fprintf (midiReportPtr, "Mask sub-window size = %d\n", maskSubWindow);
687  fprintf (midiReportPtr, "Data sub-window size = %d\n", format->subWindowSize);
688  if (maskSubWindow != format->subWindowSize)
689  {
690  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Mask has incompatible sub window size");
691  *error = 1;
692  }
693 
694  // Get MAXSTEP value
695  tempStr = qfits_query_ext (inFitsFile, "MAXSTEP", extNumOfImagingDataFile);
696  if (tempStr != NULL)
697  {
698  if (diagnostic)cpl_msg_info(cpl_func,"MAXSTEP = %s\n", tempStr);
699  fprintf( midiReportPtr, "MAXSTEP = %s\n", tempStr);
700  sscanf (tempStr, "%d", &maxstep);
701  }
702  else
703  {
704  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read MAXSTEP");
705  *error = 1;
706  }
707 
708  // Read column COL_STEPPING_PHASE and check compatibility
709  inSteppingPhase = (short int*)qfits_query_column (pTable, indexSteppingPhase, NULL);
710  if (inSteppingPhase[0] != ((aprioriSteppingPhase % format->framesPerScan) + 1))
711  {
712  sprintf (midiMessage, "Incorrect Stepping Phase. Expected %d to %d. Instead found %d to %d",
713  aprioriSteppingPhase+1, maxstep, inSteppingPhase[0], maxstep-1);
714  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
715  *error = 1;
716  }
717 
718  // Release memory and exit this module if there has been a warning. No point going any further
719  if (*error)
720  {
721  qfits_table_close (pTable);
722  qfits_table_close (pMask);
723  free (inData);
724  free (inTARTYP);
725  free (inMask);
726  free (foundData);
727  free (indexData);
728  free (foundTARTYP);
729  free (indexTARTYP);
730  free (indexMask);
731  free (inSteppingPhase);
732  free (normalization);
733  free (dataName);
734  return (-1);
735  }
736 
737  // Read DATA and TARTYP for all regions from input files
738  for (R = startRegion; R < endRegion; R++)
739  {
740  inData[R] = (short int*) qfits_query_column (pTable, indexData[R], NULL);
741  inTARTYP[R] = (char *) qfits_query_column (pTable, indexTARTYP[R], NULL);
742 
743  found = 0;
744  for (F = 0; F < format->numOfFrames; F++)
745  {
746  // Correct the initial anomalies
747  if (newSet)
748  {
749  if (inTARTYP[R][F*tartypMult] == 'U')
750  found = 1;
751  if (!found)
752  inTARTYP[R][F*tartypMult] = 'U';
753  }
754 
755  // Load data and correct tartype problem
756  if (strcmp (shutter, "AOPEN") == 0)
757  {
758  if (R == 0)
759  (compressedPhotom->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
760  else if (R == 1)
761  (compressedInterf->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
762  }
763  else
764  {
765  if (R == 1)
766  (compressedInterf->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
767  else if (R == 3)
768  (compressedPhotom->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
769  }
770  }
771  }
772 
773  // Get TIME
774  if(indexTIME >= 0) // Valid ...
775  {
776  startframe = 0;
777  inTIME = (double *) qfits_query_column (pTable, indexTIME, NULL);
778  // Stuff it into time, subtracting off the zero time
779  if (iF == 0) zeroTime = inTIME[0];
780 
781  // Fix a bug found in the data:
782  if (iF == 0)
783  {
784  for (startframe = 0; startframe < ARB_NUM_OF_FRAMES; startframe++)
785  {
786  if ((zeroTime = inTIME[startframe]) > 1.0) // Should be more like 53000 or something!
787  break;
788  }
789  }
790  if (startframe)
791  {
792  if (diagnostic)
793  {
794  cpl_msg_info(cpl_func,"\nLOOK: frames 0 - %d had ZERO for their time field!!\n", startframe-1);
795  fprintf(midiReportPtr, "\nLOOK: frames 0 - %d had ZERO for their time field!!\n", startframe-1);
796  }
797  }
798 
799  for (F = startframe; F < format->numOfFrames; F++)
800  {
801  compressedInterf->time[iF+F] = (float)(inTIME[F] - zeroTime);
802  compressedPhotom->time[iF+F] = (float)(inTIME[F] - zeroTime);
803  if (isnan (compressedInterf->time[iF+F]) ||
804  isnan (compressedPhotom->time[iF+F]))
805  {
806  // All channels and regions are affected
807  for (i = 0; i < format->iXWidth; i++)
808  {
809  compressedInterf->rejectList[i][iF+F] |= BSL_TIME_ERROR;
810  compressedPhotom->rejectList[i][iF+F] |= BSL_TIME_ERROR;
811  }
812  sprintf (midiMessage, "inTIME has an INVALID value at frame %d", iF+F);
813  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
814  }
815  }
816  }
817 
818  // Get LOCALOPD
819  if (indexLOCALOPD >= 0) // Valid ...
820  {
821  inLOCALOPD = (double (*)[2]) qfits_query_column (pTable, indexLOCALOPD, NULL);
822  // Add up the 2 delay lines, and stuff it into localOPD:
823  for (F = 0; F < format->numOfFrames; F++)
824  {
825  compressedInterf->localOPD[iF+F] = (float)(inLOCALOPD[F][0] + inLOCALOPD[F][1]);
826  compressedPhotom->localOPD[iF+F] = (float)(inLOCALOPD[F][0] + inLOCALOPD[F][1]);
827  if (isnan (compressedInterf->localOPD[iF+F]) ||
828  isnan (compressedPhotom->localOPD[iF+F]))
829  {
830  // All channels and regions are affected
831  for (i = 0; i < format->iXWidth; i++)
832  {
833  compressedInterf->rejectList[i][iF+F] |= BSL_LOCALOPD_ERROR;
834  compressedPhotom->rejectList[i][iF+F] |= BSL_LOCALOPD_ERROR;
835  }
836  sprintf (midiMessage, "localOPD has an INVALID value at frame %d", iF+F);
837  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
838  }
839  }
840  }
841 
842  // Get OPD
843  if (indexOPD >= 0) // Valid ...
844  {
845  inOPD = (double (*)[2]) qfits_query_column (pTable, indexOPD, NULL);
846  // Add up the 2 delay lines, and stuff it into bigDL:
847  for (F = 0; F < format->numOfFrames; F++)
848  {
849  (compressedInterf->bigDL)[iF+F] = (float)(inOPD[F][0] + inOPD[F][1]);
850  (compressedPhotom->bigDL)[iF+F] = (float)(inOPD[F][0] + inOPD[F][1]);
851  if (isnan (compressedInterf->bigDL[iF+F]) ||
852  isnan (compressedPhotom->bigDL[iF+F]))
853  {
854  // All channels and regions are affected
855  for (i = 0; i < format->iXWidth; i++)
856  {
857  compressedInterf->rejectList[i][iF+F] |= BSL_OPD_ERROR;
858  compressedPhotom->rejectList[i][iF+F] |= BSL_OPD_ERROR;
859  }
860  sprintf (midiMessage, "bigDL has an INVALID value at frame %d", iF+F);
861  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
862  }
863  }
864  }
865 
866  // Read all subregions from mask file
867  for (R = startRegion; R < endRegion; R++)
868  {
869  inMask[R] = (float*) qfits_query_column (pMask, indexMask[R], NULL);
870 
871  // Display mask data
872  if (diagnostic && plotFile && newSet)
873  {
874  fileString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
875  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
876  sprintf (fileString, "3dMaskDATA%d", R+1);
877  sprintf (title, "3D Mask DATA %d", R+1);
878  midiCreatePlotFile3D (fileString, title, "X", "Y", "Flux", 0,
879  inMask[R], format->iXWidth, format->iYWidth, "lines", "3");
880  free (fileString);
881  free (title);
882  }
883  }
884 
885  // Select valid channels
886  if (newSet || !channelSelected)
887  {
888  selectChannels (startRegion, endRegion, format, inMask);
889  channelSelected = 1;
890  }
891 
892  // Get the scaling offset
893  for (i = 14; i < 25; i++)
894  {
895  sprintf (dataName, "TZERO%d", i);
896  tempStr = qfits_query_ext (inFitsFile, dataName, extNumOfImagingDataFile);
897  if (tempStr != NULL)
898  {
899  if (diagnostic)cpl_msg_info(cpl_func,"Scaling Offset = %s\n", tempStr);
900  if (diagnostic) fprintf (midiReportPtr, "Scaling Offset = %s\n", tempStr);
901  sscanf (tempStr, "%d", &scalingOffset);
902  break;
903  }
904  }
905  if (tempStr == NULL)
906  {
907  scalingOffset = 0;
908  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Scaling Offset. It is set to 0");
909  }
910 
911  // ACTUAL COMPRESSION OF DATA HERE
912  for (R = startRegion; R < endRegion; R++)
913  {
914  // Determine the normalization for each X (DISPERSED mode)
915  for (X = 0; X < format->iXWidth; X++)
916  {
917  accum = 0.0F;
918  for (Y = 0; Y < format->iYWidth; Y++)
919  accum += inMask[R][Y * format->iXWidth + X];
920 
921  if (accum > 0.0)
922  normalization[X] = 1.F/accum;
923  else
924  normalization[X] = 1.F;
925  }
926 
927  for (F = 0; F < format->numOfFrames; F++)
928  {
929  frameOffset = F * format->subWindowSize;
930  for (X = 0; X < format->iXWidth; X++)
931  {
932  accum = 0.0F;
933  for (Y = 0; Y < format->iYWidth; Y++)
934  {
935  k = Y * format->iXWidth + X;
936  i = frameOffset + k;
937 
938 #ifdef SIMULATE_DATA
939  // If suitable data is not available
940  if ((compressedInterf->tarType[F] == 'T') || (compressedPhotom->tarType[F] == 'T'))
941  {
942  if ((R == 1) || (R == 2))
943  inData[R][i] = 20000;
944  else
945  inData[R][i] = 30000;
946  }
947  else
948  {
949  if ((R == 1) || (R == 2))
950  inData[R][i] = 7000;
951  else
952  inData[R][i] = 10000;
953  }
954 #endif
955  // If data is bad reject it
956  if (isnan (inData[R][i]))
957  {
958  compressedInterf->rejectList[X][iF+F] |= BSL_DATA_ERROR;
959  compressedPhotom->rejectList[X][iF+F] |= BSL_DATA_ERROR;
960  sprintf (midiMessage, "inData has an INVALID value at frame %d", (iF+F));
961  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
962  }
963  else
964  {
965  current = (inData[R][i] + scalingOffset) * inMask[R][k];
966  accum += current;
967  }
968  }
969 
970  // Accumulate further for kappa computation
971  if ((R == 0) && (strcmp (shutter, "AOPEN") == 0))
972  {
973  (((compressedPhotom->iDispFringe)[R])[X])[iF+F] =
974  accum * normalization[X];
975  (compressedPhotom->iFringe1)[iF+F] += accum;
976  }
977  if (R == 1)
978  {
979  (((compressedInterf->iDispFringe)[R-1])[X])[iF+F] =
980  accum * normalization[X];
981  (compressedInterf->iFringe1)[iF+F] += accum;
982  (compressedInterf->iFringe)[iF+F] += accum;
983  }
984  if (R == 2)
985  {
986  (((compressedInterf->iDispFringe)[R-1])[X])[iF+F] =
987  accum * normalization[X];
988  (compressedInterf->iFringe2)[iF+F] += accum;
989  (compressedInterf->iFringe)[iF+F] -= accum;
990  }
991  if ((R == 3) && (strcmp (shutter, "BOPEN") == 0))
992  {
993  (((compressedPhotom->iDispFringe)[R-3])[X])[iF+F] =
994  accum * normalization[X];
995  (compressedPhotom->iFringe1)[iF+F] += accum;
996  }
997  }
998  }
999 
1000  // Diagnostic
1001  if (diagnostic > 1)
1002  {
1003  for (F = 0; F < format->numOfFrames-maxstep; F++)
1004  {
1005  fst = F * format->subWindowSize;
1006  snd = (F+maxstep) * format->subWindowSize;
1007  if (compressedInterf->tarType[F] != 'U')
1008  {
1009  for (Y = 0; Y < format->iYWidth; Y++)
1010  {
1011  for (X = 0; X < format->iXWidth; X++)
1012  {
1013  k = X * format->iYWidth + Y;
1014  i = fst + k;
1015  i2 = snd + k;
1016  if ((compressedInterf->tarType[F] == 'T') && (compressedInterf->tarType[F+maxstep] == 'S'))
1017  current = (inData[R][i] + scalingOffset) - (inData[R][i2] + scalingOffset);
1018  else if ((compressedInterf->tarType[F] == 'S') && (compressedInterf->tarType[F+maxstep] == 'T'))
1019  current = (inData[R][i2] + scalingOffset) - (inData[R][i] + scalingOffset);
1020 
1021  // Load
1022  if ((R == 0) && (strcmp (shutter, "AOPEN") == 0))
1023  compressedPhotom->image[R][k] += (current * inMask[R][k]);
1024  if (R == 1 || R == 2)
1025  compressedInterf->image[R-1][k] += (current * inMask[R][k]);
1026  if ((R == 3) && (strcmp (shutter, "BOPEN") == 0))
1027  compressedPhotom->image[R-3][k] += (current * inMask[R][k]);
1028  }
1029  }
1030  }
1031  }
1032  F += maxstep;
1033  }
1034  }
1035 
1036  // Save local parameters and indices
1037  aprioriSteppingPhase = inSteppingPhase[F-1]; // F-1 because of the above loop
1038  if (aprioriSteppingPhase == format->framesPerScan) aprioriSteppingPhase = 0;
1039 
1040  frame0 = iF; // To output on return, below
1041  iF += format->numOfFrames; // Set it correctly for the next call to this function
1042 
1043  // Clean up now:
1044  for (R = 0; R < format->numOfDetectorRegions; R++) free (inData[R]);
1045  for (R = 0; R < format->numOfDetectorRegions; R++) free (inTARTYP[R]);
1046  for (R = 0; R < format->numOfDetectorRegions; R++) free (inMask[R]);
1047  if (pMask) qfits_table_close (pMask);
1048  if (pTable) qfits_table_close (pTable);
1049  if (inLOCALOPD) free(inLOCALOPD);
1050  if (inOPD) free(inOPD);
1051  if (inTIME) free(inTIME);
1052  free (inData);
1053  free (inTARTYP);
1054  free (inMask);
1055  free (inSteppingPhase);
1056  free (foundData);
1057  free (indexData);
1058  free (foundTARTYP);
1059  free (indexTARTYP);
1060  free (indexMask);
1061  free (normalization);
1062  free (dataName);
1063 
1064  return (frame0);
1065 }
1066 /*****************************************************************************/
1067