MIDI Pipeline Reference Manual  2.8.3
preProcFrgHS.c
1 /******************************************************************************
2 *******************************************************************************
3 * European Southern Observatory
4 * VLTI MIDI Data Reduction Software
5 *
6 * Module name: preProcFrgHS.c
7 * Description: Contains routines for pre processing
8 *
9 * History:
10 * 13-Jul-05 (csabet) Created
11 *******************************************************************************
12 ******************************************************************************/
13 
14 /******************************************************************************
15 * Compiler directives
16 ******************************************************************************/
17 
18 /******************************************************************************
19 * Include files
20 ******************************************************************************/
21 #include <sys/types.h>
22 #include <sys/stat.h>
23 #include <unistd.h>
24 #include <stdio.h>
25 #include <cpl.h>
26 #include <math.h>
27 #include "midiGlobal.h"
28 #include "midiLib.h"
29 #include "memoryHandling.h"
30 #include "errorHandling.h"
31 #include "midiFitsUtility.h"
32 #include "diagnostics.h"
33 #include "preProcFrgHS.h"
34 #include "qfits.h"
35 
36 /**********************************************************
37 * Constant definitions
38 **********************************************************/
39 
40 /**********************************************************
41 * Global Variables
42 **********************************************************/
43 
44 /*============================ C O D E A R E A ===========================*/
45 
46 
47 
48 /******************************************************************************
49 * European Southern Observatory
50 * VLTI MIDI Data Reduction Software
51 *
52 * Module name: preProcFrgHS
53 * Input/Output: See function arguments to avoid duplication
54 * Description: Prepares the data for both DISPERSED as well as UNDISPERSED modes of
55 * processing
56 *
57 * History:
58 * 21-Jul-03 (csabet) Created
59 * 25 aug 04 (JM) Modified by combining several other modules to handle both DISPERSED & UNDISPERSED modes
60 * 16-Jan-05 (csabet) Cleaned up as much as possible, removed GOTO instructions! and integrated into MIDI
61 ******************************************************************************/
62 void preProcFrgHS (
63  UserOptions *options, // In: User parameters
64  FilterData *filterInfo, // IO: Pointer to the filter data structure
65  MidiFiles *fileNames, // In: Pointer to the MIDI file structure
66  CompressedData *compressedInterf, // Ou: Pointer to the compressed interferometry data structure
67  CompressedData *compressedPhotomA, // Ou: Pointer to the compressed photom A data structure
68  CompressedData *compressedPhotomB, // Ou: Pointer to the compressed photom B data structure
69  ImageFormat *formatInterf, // In: Interf size parameters
70  ImageFormat *formatPhotomA, // In: PhotA size parameters
71  ImageFormat *formatPhotomB, // In: PhotB size parameters
72  int *error) // Ou: Error status
73 {
74 
75  /* Local Declarations
76  --------------------*/
77  const char routine[] = "preProcFrgHS";
78  FILE *inFitsBatchPtr = NULL;
79  int newFile, extNumOfImagingDataFile, extNumOfImagingDataMask, R;
80  char *maskFile, *cleanString, *classification, *stringTemp, *obsTech, *fileName, *title;
81  unsigned int loopCount = 0;
82  ImageFormat *localFormat;
83 
84  /* Algorithm
85  -----------*/
86  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
87  if (diagnostic > 4) fprintf(midiReportPtr, "Invoking routine '%s' \n", routine);
88 
89  cpl_msg_info(cpl_func,"\nReducing data of batch %d \n", batchNumber);
90  cpl_msg_info(cpl_func,"---------------------- \n");
91  fprintf (midiReportPtr, "\nReducing data of batch %d \n", batchNumber);
92  fprintf (midiReportPtr, "---------------------- \n");
93 
94  /* Allocate memory */
95  obsTech = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
96  stringTemp = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
97  classification = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
98  cleanString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
99  localFormat = callocImageFormat ();
100  maskFile = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
101 
102  /* Reset status */
103  *error = 0;
104  compressedInterf->exists = 0;
105  compressedPhotomA->exists = 0;
106  compressedPhotomB->exists = 0;
107  sprintf (obsTech, "UNKNOWN");
108  newFile = 1;
109 
110  /* Open the list of files */
111  if ((inFitsBatchPtr = fopen (fileNames->inFitsBatch, "r")) == NULL)
112  {
113  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
114  "Cannot open input FITS file list. No compression has been carried out for this batch");
115  free (cleanString);
116  free (localFormat);
117  free (stringTemp);
118  free (classification);
119  freeImageFormat (localFormat);
120  free (maskFile);
121  free (obsTech);
122  *error = 1;
123  return;
124  }
125 
126  // Loop through the list of files and compress data
127  while (fgets (stringTemp, MAX_STRING_LENGTH, inFitsBatchPtr) != NULL)
128  {
129  sprintf (classification, "%s", "");
130  sscanf (stringTemp, "%s%s", fileNames->inFitsName, classification);
131  cpl_msg_info(cpl_func,"\nProcessing file %s \n", fileNames->inFitsName);
132  fprintf (midiReportPtr, "\nProcessing file %s \n", fileNames->inFitsName);
133 
134  // Get 'extNumOfImagingDataFile' extension number of IMAGING_DATA in input file
135  extNumOfImagingDataFile = findImagingDataExtension (fileNames->inFitsName, TAB_IMAGING_DATA, error);
136  if (*error) break;
137 
138  // Get Image Format parameters
139  if (extNumOfImagingDataFile > 0)
140  {
141  getImageFormat (fileNames->inFitsName, extNumOfImagingDataFile, localFormat, error);
142  if (*error) break;
143  }
144  else localFormat->hasData = 0;
145 
146  // Get filter data and mask file name from the first FITS file
147  if (loopCount == 0)
148  {
149  getFilterData (fileNames->inFitsName, filterInfo, error);
150  if (*error) break;
151  else fprintf (midiReportPtr, "First file of batch = %s (QCLOG)\n", fileNames->inFitsName);
152 
153  // Select mask file
154  selectMask (options->maskMode, fileNames, maskFile, error);
155  if (*error) break;
156 
157  // Get 'extNumOfImagingDataMask' extension number of IMAGING_DATA in mask file
158  extNumOfImagingDataMask = findImagingDataExtension (maskFile, MASK_IMAGING_DATA, error);
159  if (*error) break;
160 
161  cpl_msg_info(cpl_func,"Mask File is %s\n", maskFile);
162  fprintf( midiReportPtr, "Mask File is %s\n", maskFile);
163  }
164 
165  // Check if the file has data
166  if (localFormat->hasData)
167  {
168  // Check "Observation Technique". We expect either INTERFEROMETRY or IMAGE
169  if (strcmp (localFormat->obsTech, "INTERFEROMETRY") == 0)
170  {
171  // Check previous TYPE
172  if (strcmp (obsTech, "INTERF") != 0) newFile = 1;
173 
174  // Set the TYPE flag
175  sprintf (obsTech, "INTERF");
176 
177  // Check consistency
178  if ((formatInterf->numOfDetectorRegions != localFormat->numOfDetectorRegions) ||
179  (formatInterf->iXWidth != localFormat->iXWidth) || (formatInterf->iYWidth != localFormat->iYWidth))
180  {
181  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Expected format is incorrect");
182  *error = 1;
183  break;
184  }
185 
186  // Send Interferometry file for compression
187  organiseFrgHS (newFile, obsTech, fileNames->inFitsName, maskFile,
188  extNumOfImagingDataFile, extNumOfImagingDataMask, localFormat, formatInterf,
189  compressedInterf, error);
190 
191  if (*error) break;
192  newFile = 0; // So the next time through, will know not to start at frame 0
193  }
194  else if ((strcmp (localFormat->obsTech, "IMAGE,WINDOW,CHOPNOD") == 0) &&
195  (strcmp(localFormat->shutterId,"AOPEN") == 0))
196  {
197  // Check previous TYPE
198  if (strcmp (obsTech, "PHOTOMA") != 0) newFile = 1;
199 
200  // Set the TYPE flag
201  sprintf (obsTech, "PHOTOMA");
202 
203  // Check consistency
204  if ((formatPhotomA->numOfDetectorRegions != localFormat->numOfDetectorRegions) ||
205  (formatPhotomA->iXWidth != localFormat->iXWidth) || (formatPhotomA->iYWidth != localFormat->iYWidth))
206  {
207  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Expected format is incorrect");
208  *error = 1;
209  break;
210  }
211 
212  // Send Photometry A file for compression
213  organiseFrgHS (newFile, obsTech, fileNames->inFitsName, maskFile,
214  extNumOfImagingDataFile, extNumOfImagingDataMask, localFormat, formatPhotomA,
215  compressedPhotomA, error);
216 
217  if (*error) break;
218  newFile = 0; // So the next time through, will know not to start at frame 0
219  }
220  else if ((strcmp (localFormat->obsTech, "IMAGE,WINDOW,CHOPNOD") == 0) &&
221  (strcmp(localFormat->shutterId,"BOPEN") == 0))
222  {
223  // Check previous TYPE
224  if (strcmp (obsTech, "PHOTOMB") != 0) newFile = 1;
225 
226  // Set the TYPE flag
227  sprintf (obsTech, "PHOTOMB");
228 
229  // Check consistency
230  if ((formatPhotomB->numOfDetectorRegions != localFormat->numOfDetectorRegions) ||
231  (formatPhotomB->iXWidth != localFormat->iXWidth) || (formatPhotomB->iYWidth != localFormat->iYWidth))
232  {
233  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Expected format is incorrect");
234  *error = 1;
235  break;
236  }
237 
238  // Send Photometry B file for compression
239  organiseFrgHS (newFile, obsTech, fileNames->inFitsName, maskFile,
240  extNumOfImagingDataFile, extNumOfImagingDataMask, localFormat, formatPhotomB,
241  compressedPhotomB, error);
242 
243  if (*error) break;
244  newFile = 0; // So the next time through, will know not to start at frame 0
245  }
246  else
247  {
248  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Unknown Catg, Type or Tech");
249  *error = 1;
250  break;
251  }
252  }
253  else
254  {
255  if (diagnostic)
256  {
257  sprintf (midiMessage, "No data tables in %s. Not processed", fileNames->inFitsName);
258  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
259  }
260  }
261  loopCount++;
262  }
263 
264  // Close files and release memory
265  fclose (inFitsBatchPtr);
266  free (cleanString);
267  free (stringTemp);
268  free (classification);
269  freeImageFormat (localFormat);
270  free (maskFile);
271  free (obsTech);
272 
273  // Check if reduction has been carried out
274  cpl_msg_info(cpl_func,"\nCompression status \n");
275  cpl_msg_info(cpl_func,"------------------ \n");
276  if (compressedInterf->exists)cpl_msg_info(cpl_func,"Created Compressed Interferometry Data\n");
277  if (compressedPhotomA->exists)cpl_msg_info(cpl_func,"Created Compressed Photometry A Data\n");
278  if (compressedPhotomB->exists)cpl_msg_info(cpl_func,"Created Compressed Photometry B Data\n");
279  cpl_msg_info(cpl_func,"\n");
280 
281  fprintf (midiReportPtr, "\nCompression status \n");
282  fprintf (midiReportPtr, "------------------ \n");
283  if (compressedInterf->exists) fprintf (midiReportPtr, "Created Compressed Interferometry Data\n");
284  if (compressedPhotomA->exists) fprintf (midiReportPtr, "Created Compressed Photometry A Data\n");
285  if (compressedPhotomB->exists) fprintf (midiReportPtr, "Created Compressed Photometry B Data\n");
286  fprintf (midiReportPtr, "\n");
287 
288  if (!(compressedInterf->exists) ||
289  !(compressedPhotomA->exists) || !(compressedPhotomB->exists) || *error)
290  {
291  *error = 1;
292  sprintf (midiMessage,
293  "Cannot continue. Need the following compressed data: Interf, PhotomA, PhotomB");
294  midiReportWarning (midiReportPtr,
295  routine, __FILE__, __LINE__, midiMessage);
296  return;
297  }
298 
299  // Display images
300  if (diagnostic > 1 && plotFile)
301  {
302  fileName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
303  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
304 
305  for (R = 0; R < formatPhotomA->numOfRegionsToProcess; R++)
306  {
307  sprintf (fileName, "3dPhotomADATA%d", R+1);
308  sprintf (title, "Photometry A DATA %d (Masked and Sky removed)", R+1);
309  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
310  compressedPhotomA->image[R], formatPhotomA->iXWidth, formatPhotomA->iYWidth, "lines", "3");
311  }
312 
313  for (R = 0; R < formatPhotomB->numOfRegionsToProcess; R++)
314  {
315  sprintf (fileName, "3dPhotomBDATA%d", R+1);
316  sprintf (title, "Photometry B DATA %d (Masked and Sky removed)", R+1);
317  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
318  compressedPhotomB->image[R], formatPhotomB->iXWidth, formatPhotomB->iYWidth, "lines", "3");
319  }
320 
321  free (fileName);
322  free (title);
323  }
324 
325  return;
326 }
327 /*****************************************************************************/
328 
329 
330 /******************************************************************************
331 * European Southern Observatory
332 * VLTI MIDI Data Reduction Software
333 *
334 * Module name: organiseFrgHS
335 * Input/Output: See function arguments to avoid duplication
336 * Description: Prepares HIGH_SENS files for compression
337 *
338 * History:
339 * 03-Nov-05 (csabet) Created
340 ******************************************************************************/
341 void organiseFrgHS (
342  int newFile, // In: Flag indicating continuation
343  char *obsTech, // In: Whether Interf or Photom
344  char *fitsFile, // In: Name of the input FITS file
345  char *maskFile, // In: Name of the Mask file
346  int extNumOfImagingDataFile, // In: Extention number
347  int extNumOfImagingDataMask, // In: Extention number
348  ImageFormat *localFormat, // In: Local split file format
349  ImageFormat *formatMax, // In: Maximum file format
350  CompressedData *compressed, // In: Compressed data
351  int *error) // Ou: Error status
352 {
353 
354  // Local Declarations
355  // ------------------
356  const char routine[] = "organiseFrgHS";
357  int frame0;
358 
359  // Algorithm
360  // ---------
361  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
362  if (diagnostic > 4) fprintf(midiReportPtr, "Invoking routine '%s' \n", routine);
363 
364  // Initialise
365  *error = 0;
366 
367  // Compress data
368  frame0 = compressFrgHS (newFile, obsTech, fitsFile, maskFile, extNumOfImagingDataFile,
369  extNumOfImagingDataMask, compressed, localFormat, formatMax->numOfFrames, error);
370  if (*error) return;
371 
372  // Report compression status
373  if (diagnostic)
374  {
375  sprintf (midiMessage, "\nWrote %d frames into %s, starting at frame %d\nThus now %d frames total,"
376  " expecting %d altogether. Error=%d \n\n", localFormat->numOfFrames, obsTech, frame0,
377  localFormat->numOfFrames+frame0, formatMax->numOfFrames, *error);
378  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
379  }
380 
381  // Set the flags
382  if ((frame0 + localFormat->numOfFrames) == formatMax->numOfFrames) // ALL have been read in
383  {
384  compressed->exists = 1; // Mark it as completed
385 
386  // Now with data read in, display some of the parameters:
387  if (plotFile && (strcmp (obsTech, "INTERF") == 0) && diagnostic)
388  {
389  midiCreatePlotFile2D ("Time", "TIME", "Frame", "Time", 0, compressed->time, 0,
390  formatMax->numOfFrames, 1, 0);
391  midiCreatePlotFile2D ("BigDelayLine", "Net VLTI Delay Line Positions", "Frame",
392  "Net Position", 0, compressed->bigDL, 0, formatMax->numOfFrames, 1, 0);
393  midiCreatePlotFile2D ("PiezoDelayLine", "Net Piezo Delay Line Position", "Frame", "Net Delay", 0,
394  compressed->localOPD, 0, formatMax->numOfFrames, 1, 0);
395  }
396  }
397 
398  return;
399 }
400 /*****************************************************************************/
401 
402 
403 
404 /******************************************************************************
405 * European Southern Observatory
406 * VLTI MIDI Data Reduction Software
407 *
408 * Module name: compressFrgHS
409 * Input/Output: See function arguments to avoid duplication
410 * Description: This routine prepares the interferomerty data for dispersed
411 * AND undispersed processing. In DISPERSED mode iDispFringe points to
412 * a 3D structure. One dimension is time (frame), the second dimension is
413 * wave (produced by collapsing each X, Y image into one line) and the
414 * third dimension is the number of sub-regions.
415 * History:
416 * 14-Aug-03 (csabet) Created. Derived from pballest July 2003
417 * 25-Aug-04 (JM) Modified by combining several other modules to handle both DISPERSED & UNDISPERSED modes
418 * 16-Jan-05 (csabet) Cleaned and added numerous features. This module is still far too big. Will
419 * modularise later
420 ******************************************************************************/
421 int compressFrgHS ( // Ou: Status. Returns number of first frame compressed; -1 if error
422  int newSet, // In: Flag indicating the arrival of a new set of FITS files
423  char *obsTech, // In: Whether Interf or Photom
424  char *inFitsFile, // In: Name of the input FITS file
425  char *maskFile, // In: Name of the mask file
426  int extNumOfImagingDataFile,// In: Extension number of the IMAGING_DATA in input file
427  int extNumOfImagingDataMask,// In: Extension number of the IMAGING_DATA in mask file
428  CompressedData *compressed, // In: Compressed data
429  ImageFormat *localFormat, // In: Local split file format
430  int numOfFramesMax, // In: Maximum number of frames
431  int *error) // Ou: Error status
432 {
433 
434  // Local Declarations
435  // ------------------
436  const char routine[] = "compressFrgHS";
437  qfits_table *pTable = NULL, *pMask = NULL;
438  short int **inData, *inSteppingPhase;
439  float **inMask;
440  float accum, current;
441  float *normalization; // Normalization factor. Becomes an array in X for the normalization of the
442  // compressed data, valid for each region then overwritten
443  int frameOffset, indexOPD = - 1, indexLOCALOPD = - 1, indexTIME = - 1, maxstep;
444  char **inTARTYP = NULL, *tempStr, fitsColumnString[10], *dataName, *title=NULL, *fileString=NULL;
445  double *inTIME = NULL;
446  int frame0 = - 1; // Gets set to a valid number UNLESS routine terminates in error
447  double (*inLOCALOPD)[2] = NULL, (*inOPD)[2] = NULL;
448  static int aprioriSteppingPhase, iF, chopInterval, channelSelected = 0;
449  static double zeroTime ;
450  int i, k, fps, *foundData, foundSteppingPhase = 0, indexSteppingPhase, scalingOffset, *indexData ,
451  *indexMask, maskWidthX, maskWidthY, maskSubWindow, F, X, Y, R,
452  *indexTARTYP, *foundTARTYP = 0, startframe=0;
453  int tartypMult = 2; // Signifies that there are 2 (instead of 1!) characters per tartyp value (only
454  // first is useful). In future, if the problem producing the MIDI files is solved,
455  // can be changed to 1 (only for newer files) or made variable (with some way to
456  // determine whether it is an "old" type file). Note: this problem really has
457  // nothing to do with Qfits, but with the (unintended) way the files are written.
458  int i2, fst, snd, found;
459  char lastType;
460 
461  // Algorithm
462  // ---------
463  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
464  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
465 
466  // Reset status
467  *error = 0;
468  fps = localFormat->framesPerScan;
469 
470  // Allocate memory
471  inData = (short int **) calloc (localFormat->numOfDetectorRegions, sizeof (short int *));
472  inTARTYP = (char **) calloc (localFormat->numOfDetectorRegions, sizeof (char *));
473  inMask = (float **) calloc (localFormat->numOfRegionsToProcess, sizeof (float *));
474  foundData = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
475  indexData = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
476  foundTARTYP = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
477  indexTARTYP = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
478  indexMask = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
479  normalization = (float *) calloc (localFormat->iXWidth, sizeof (float));
480  dataName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
481 
482  // Initialise the indices for a new set of FITS files. The
483  // files may be split the indices ensure continuity of split files
484  if (newSet)
485  {
486  aprioriSteppingPhase = 0;
487  iF = 0;
488  }
489 
490  // Make sure the sum of the split-files numOfFrames is not more than the allowed maximum
491  if ((iF + localFormat->numOfFrames) > numOfFramesMax)
492  localFormat->numOfFrames = numOfFramesMax - iF;
493 
494  // Open IMAGING_DATA = INDEX 2
495  pMask = qfits_table_open (maskFile, extNumOfImagingDataMask);
496  if (!pMask)
497  {
498  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load IMAGING_DATA");
499  free (inData);
500  free (inTARTYP);
501  free (inMask);
502  free (foundData);
503  free (indexData);
504  free (foundTARTYP);
505  free (indexTARTYP);
506  free (indexMask);
507  free (normalization);
508  free (dataName);
509  *error = 1;
510  return (-1);
511  }
512  pTable = qfits_table_open (inFitsFile, extNumOfImagingDataFile);
513  if (!pTable)
514  {
515  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load IMAGING_DATA");
516  qfits_table_close (pMask);
517  free (inData);
518  free (inTARTYP);
519  free (inMask);
520  free (foundData);
521  free (indexData);
522  free (foundTARTYP);
523  free (indexTARTYP);
524  free (indexMask);
525  free (normalization);
526  free (dataName);
527  *error = 1;
528  return (-1);
529  }
530 
531  // Get data table information
532  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
533  {
534  foundData[R] = 0;
535  indexData[R] = 0;
536  }
537  for (i = 0; i < pTable->nc; i++)
538  {
539  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
540  {
541  sprintf (dataName, "DATA%d", R+1);
542  if (strcmp (pTable->col[i].tlabel, dataName) == 0)
543  {
544  foundData[R] = 1;
545  indexData[R] = i;
546  if (diagnostic)
547  {
548  cpl_msg_info(cpl_func,"Found 'DATA%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
549  fprintf(midiReportPtr, "Found 'DATA%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
550  }
551  }
552 
553  sprintf (dataName, "TARTYP%d", R+1);
554  if (strcmp (pTable->col[i].tlabel, dataName) == 0)
555  {
556  foundTARTYP[R] = 1;
557  indexTARTYP[R] = i;
558  if (diagnostic)
559  {
560  cpl_msg_info(cpl_func,"Found 'TARTYP%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
561  fprintf(midiReportPtr, "Found 'TARTYP%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
562  }
563  }
564  }
565  if (strcmp (pTable->col[i].tlabel, "STEPPING_PHASE") == 0)
566  {
567  foundSteppingPhase = 1;
568  indexSteppingPhase = i;
569  }
570  if (strcmp (pTable->col[i].tlabel, "OPD") == 0)
571  {
572  indexOPD = i;
573  }
574  if (strcmp (pTable->col[i].tlabel, "LOCALOPD") == 0)
575  {
576  indexLOCALOPD = i;
577  }
578  if (strcmp (pTable->col[i].tlabel, "TIME") == 0)
579  {
580  indexTIME = i;
581  }
582  }
583 
584  // Now issue warnings
585  if (foundSteppingPhase == 0)
586  {
587  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find STEPPING_PHASE in data FITS file");
588  *error = 1;
589  }
590  if (indexOPD < 0)
591  {
592  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for OPD in data FITS file");
593  *error = 1;
594  }
595  if (indexLOCALOPD < 0)
596  {
597  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for LOCALOPD in data FITS file");
598  *error = 1;
599  }
600  if (indexTIME < 0)
601  {
602  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for TIME in data FITS file");
603  *error = 1;
604  }
605  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
606  {
607  if (foundData[R] == 0)
608  {
609  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
610  "Cannot find requested DATA column in data FITS file");
611  *error = 1;
612  }
613  if (foundTARTYP[R] == 0)
614  {
615  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
616  "Cannot find requested TARTYP column in data FITS file");
617  *error = 1;
618  }
619  }
620 
621  // Get mask data table information
622  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
623  {
624  foundData[R] = 0;
625  indexMask[R] = 0;
626  }
627  for (i = 0; i < pMask->nc; i++)
628  {
629  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
630  {
631  sprintf (dataName, "DATA%d", R+1);
632  if (strcmp (pMask->col[i].tlabel, dataName) == 0)
633  {
634  foundData[R] = 1;
635  indexMask[R] = i;
636  if (diagnostic)
637  {
638  cpl_msg_info(cpl_func,"Found 'DATA%d' at column %d in mask file %s \n", R+1, i+1, maskFile);
639  fprintf(midiReportPtr, "Found 'DATA%d' at column %d in mask file %s \n", R+1, i+1, maskFile);
640  }
641  }
642  }
643  }
644 
645  // Now issue warnings
646  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
647  {
648  if (foundData[R] == 0)
649  {
650  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
651  "Cannot find requested DATA column in mask FITS file");
652  *error = 1;
653  }
654  }
655 
656  // Get subwindow format for the mask file
657  sprintf (fitsColumnString, "TDIM%d", indexMask[1]+1); // Point it to the column of a valid region
658  tempStr = qfits_query_ext (maskFile, fitsColumnString, extNumOfImagingDataMask);
659  sscanf (tempStr, "'(%d,%d) '", &maskWidthX, &maskWidthY);
660  maskSubWindow = maskWidthX * maskWidthY;
661  if (newSet && diagnostic)cpl_msg_info(cpl_func,"Mask sub-window size = %d\n", maskSubWindow);
662  if (diagnostic)cpl_msg_info(cpl_func,"Data sub-window size = %d\n", localFormat->subWindowSize);
663  if (newSet) fprintf (midiReportPtr, "Mask sub-window size = %d\n", maskSubWindow);
664  fprintf (midiReportPtr, "Data sub-window size = %d\n", localFormat->subWindowSize);
665  if (maskSubWindow != localFormat->subWindowSize)
666  {
667  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Mask has incompatible sub window size");
668  *error = 1;
669  }
670 
671  // Get MAXSTEP value
672  tempStr = qfits_query_ext (inFitsFile, "MAXSTEP", extNumOfImagingDataFile);
673  if (tempStr != NULL)
674  {
675  if (diagnostic)cpl_msg_info(cpl_func,"MAXSTEP = %s\n", tempStr);
676  fprintf( midiReportPtr, "MAXSTEP = %s\n", tempStr);
677  sscanf (tempStr, "%d", &maxstep);
678  }
679  else
680  {
681  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read MAXSTEP");
682  *error = 1;
683  }
684 
685  // Read column COL_STEPPING_PHASE and check compatibility
686  inSteppingPhase = (short int*)qfits_query_column (pTable, indexSteppingPhase, NULL);
687  if (inSteppingPhase[0] != ((aprioriSteppingPhase % localFormat->framesPerScan) + 1))
688  {
689  sprintf (midiMessage, "Incorrect Stepping Phase. Expected %d to %d. Instead found %d to %d",
690  aprioriSteppingPhase+1, maxstep, inSteppingPhase[0], maxstep-1);
691  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
692  *error = 1;
693  }
694 
695  // Release memory and exit this module if there has been a warning. No point going any further
696  if (*error)
697  {
698  qfits_table_close (pTable);
699  qfits_table_close (pMask);
700  free (inData);
701  free (inTARTYP);
702  free (inMask);
703  free (foundData);
704  free (indexData);
705  free (foundTARTYP);
706  free (indexTARTYP);
707  free (indexMask);
708  free (inSteppingPhase);
709  free (normalization);
710  free (dataName);
711  return (-1);
712  }
713 
714  // Read all regions from input files
715  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
716  {
717  inData[R] = (short int*) qfits_query_column (pTable, indexData[R], NULL);
718  inTARTYP[R] = (char *) qfits_query_column (pTable, indexTARTYP[R], NULL);
719 
720  found = 0;
721  for (F = 0; F < localFormat->numOfFrames; F++)
722  {
723  // Correct the initial anomalies
724  if (newSet && (strcmp (obsTech, "INTERF") != 0))
725  {
726  if (inTARTYP[R][F*tartypMult] == 'U')
727  found = 1;
728  if (!found)
729  inTARTYP[R][F*tartypMult] = 'U';
730  }
731 
732  // Load data and correct tartype problem
733  (compressed->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
734 
735  if (R > 0)
736  {
737  if (inTARTYP[R][F*tartypMult] != inTARTYP[R-1][F*tartypMult])
738  {
739  sprintf (midiMessage, "Incompatible Interferometry TARTYP%d & TARTYP%d at frame %d", R, R-1, iF+F);
740  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
741 
742  // All channels are affected
743  for (i = 0; i < localFormat->iXWidth; i++)
744  compressed->rejectList[i][iF+F] |= BSL_TARTYPE_CROSS;
745 
746  // badScanList is only meaningful for Interferometry data
747  if (strcmp (obsTech, "INTERF") == 0) compressed->badScanList[(iF+F)/fps] |= BSL_TARTYPE_CROSS;
748  }
749  }
750  }
751  }
752 
753  // Get TIME
754  if (indexTIME >= 0) // Valid ...
755  {
756  startframe = 0;
757  inTIME = (double *) qfits_query_column (pTable, indexTIME, NULL);
758  // Stuff it into time, subtracting off the zero time
759  if (iF == 0) zeroTime = inTIME[0];
760 
761  // Fix a bug found in the data:
762  if (iF == 0)
763  {
764  for (startframe = 0; startframe < ARB_NUM_OF_FRAMES; startframe++)
765  {
766  if ((zeroTime = inTIME[startframe]) > 1.0) // Should be more like 53000 or something!
767  break;
768  }
769  }
770  if (startframe)
771  {
772  if (diagnostic)
773  {
774  cpl_msg_info(cpl_func,"\nLOOK: frames 0 - %d had ZERO for their time field!!\n", startframe-1);
775  fprintf(midiReportPtr, "\nLOOK: frames 0 - %d had ZERO for their time field!!\n", startframe-1);
776  }
777  }
778 
779  for (F = startframe; F < localFormat->numOfFrames; F++)
780  {
781  compressed->time[iF+F] = (float)(inTIME[F] - zeroTime);
782  if (isnan ((compressed->time[iF+F])))
783  {
784  sprintf (midiMessage, "inTIME has an INVALID value at frame %d", iF+F);
785  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
786 
787  // All channels and regions are affected
788  for (i = 0; i < localFormat->iXWidth; i++)
789  compressed->rejectList[i][iF+F] |= BSL_TIME_ERROR;
790 
791  // badScanList is only meaningful for Interferometry data
792  if (strcmp (obsTech, "INTERF") == 0) compressed->badScanList[(iF+F)/fps] |= BSL_TIME_ERROR;
793  }
794  }
795  }
796 
797  // Get LOCALOPD
798  if (indexLOCALOPD >= 0) // Valid ...
799  {
800  inLOCALOPD = (double (*)[2]) qfits_query_column (pTable, indexLOCALOPD, NULL);
801  // Add up the 2 delay lines, and stuff it into localOPD:
802  for (F = 0; F < localFormat->numOfFrames; F++)
803  {
804  compressed->localOPD[iF+F] = (float)(inLOCALOPD[F][0] + inLOCALOPD[F][1]);
805  if (isnan ((compressed->localOPD[iF+F])))
806  {
807  sprintf (midiMessage, "localOPD has an INVALID value at frame %d", iF+F);
808  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
809 
810  // All channels are affected
811  for (i = 0; i < localFormat->iXWidth; i++)
812  compressed->rejectList[i][iF+F] |= BSL_LOCALOPD_ERROR;
813 
814  // badScanList is only meaningful for Interferometry data
815  if (strcmp (obsTech, "INTERF") == 0) compressed->badScanList[(iF+F)/fps] |= BSL_LOCALOPD_ERROR;
816  }
817  }
818  }
819 
820  // Get OPD
821  if (indexOPD >= 0) // Valid ...
822  {
823  inOPD = (double (*)[2]) qfits_query_column (pTable, indexOPD, NULL);
824  // Add up the 2 delay lines, and stuff it into bigDL:
825  for (F = 0; F < localFormat->numOfFrames; F++)
826  {
827  (compressed->bigDL)[iF+F] = (float)(inOPD[F][0] + inOPD[F][1]);
828  if (isnan ((compressed->bigDL)[iF+F]))
829  {
830  sprintf (midiMessage, "bigDL has an INVALID value at frame %d", iF+F);
831  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
832 
833  // All channels are affected
834  for (i = 0; i < localFormat->iXWidth; i++)
835  compressed->rejectList[i][iF+F] |= BSL_OPD_ERROR;
836 
837  // badScanList is only meaningful for Interferometry data
838  if (strcmp (obsTech, "INTERF") == 0) compressed->badScanList[(iF+F)/fps] |= BSL_OPD_ERROR;
839  }
840  }
841  }
842 
843  // Compute chopping frequency. (Diagnostic)
844  if (newSet && (strcmp (obsTech, "INTERF") != 0))
845  {
846  lastType = ' ';
847  chopInterval = 0;
848  for (F = 0; F < localFormat->numOfFrames; F++)
849  {
850  if (compressed->tarType[F] == 'U' && (lastType == 'T' || lastType == 'S'))
851  {
852  break;
853  }
854  else
855  {
856  lastType = compressed->tarType[F];
857  chopInterval++;
858  }
859  }
860  }
861 
862  // Read all subregions from mask file
863  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
864  {
865  inMask[R] = (float*) qfits_query_column (pMask, indexMask[R], NULL);
866 
867  // Display mask data
868  if (diagnostic > 1 && plotFile && newSet && (strcmp(obsTech, "INTERF") == 0))
869  {
870  fileString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
871  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
872  sprintf (fileString, "3dMaskDATA%d", R+1);
873  sprintf (title, "3D Mask DATA %d", R+1);
874  midiCreatePlotFile3D (fileString, title, "X", "Y", "Flux", 0,
875  inMask[R], localFormat->iXWidth, localFormat->iYWidth, "lines", "3");
876  free (fileString);
877  free (title);
878  }
879  }
880 
881  // Select valid channels
882  if ((newSet || !channelSelected) && (strcmp(obsTech, "INTERF") == 0))
883  {
884  selectChannels (0, localFormat->numOfDetectorRegions, localFormat, inMask);
885  channelSelected = 1;
886  }
887 
888  // Get the scaling offset
889  for (i = 14; i < 25; i++)
890  {
891  sprintf (dataName, "TZERO%d", i);
892  tempStr = qfits_query_ext (inFitsFile, dataName, extNumOfImagingDataFile);
893  if (tempStr != NULL)
894  {
895  if (diagnostic)cpl_msg_info(cpl_func,"Scaling Offset = %s\n", tempStr);
896  if (diagnostic) fprintf (midiReportPtr, "Scaling Offset = %s\n", tempStr);
897  sscanf (tempStr, "%d", &scalingOffset);
898  break;
899  }
900  }
901  if (tempStr == NULL)
902  {
903  scalingOffset = 0;
904  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Scaling Offset. It is set to 0");
905  }
906 
907  // ACTUAL COMPRESSION OF DATA HERE
908  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
909  {
910  // Determine the normalization for each X (DISPERSED mode)
911  for (X = 0; X < localFormat->iXWidth; X++)
912  {
913  accum = 0.0F;
914  for (Y = 0; Y < localFormat->iYWidth; Y++)
915  accum += inMask[R][Y * localFormat->iXWidth + X];
916 
917  if (accum > 0.0)
918  normalization[X] = 1.F/accum;
919  else
920  normalization[X] = 1.F;
921  }
922 
923  for (F = 0; F < localFormat->numOfFrames; F++)
924  {
925  frameOffset = F * localFormat->subWindowSize;
926  for (X = 0; X < localFormat->iXWidth; X++)
927  {
928  accum = 0.0F;
929  for (Y = 0; Y < localFormat->iYWidth; Y++)
930  {
931  k = Y * localFormat->iXWidth + X;
932  i = frameOffset + k;
933 
934  // If data is bad reject it
935  if (isnan (inData[R][i]))
936  {
937  sprintf (midiMessage, "inData has an INVALID value at frame %d", (iF+F));
938  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
939 
940  compressed->rejectList[X][iF+F] |= BSL_DATA_ERROR;
941 
942  // badScanList is only meaningful for Interferometry data
943  if (strcmp (obsTech, "INTERF") == 0) compressed->badScanList[(iF+F)/fps] |= BSL_DATA_ERROR;
944  }
945  else
946  {
947  current = (inData[R][i] + scalingOffset) * inMask[R][k];
948  accum += current;
949  }
950  }
951 
952  // Accumulate further
953  (((compressed->iDispFringe)[R])[X])[iF+F] = accum * normalization[X];
954  if (R == 0)
955  {
956  (compressed->iFringe1)[iF+F] += accum;
957  (compressed->iFringe)[iF+F] += accum;
958  }
959  if (R == 1)
960  {
961  (compressed->iFringe2)[iF+F] += accum;
962  (compressed->iFringe)[iF+F] -= accum;
963  }
964  }
965  }
966 
967  // Diagnostic
968  if (diagnostic > 1 && (strcmp (obsTech, "INTERF") != 0))
969  {
970  for (F = 0; F < localFormat->numOfFrames-chopInterval; F++)
971  {
972  fst = F * localFormat->subWindowSize;
973  snd = (F+chopInterval) * localFormat->subWindowSize;
974  if (compressed->tarType[F] != 'U')
975  {
976  for (Y = 0; Y < localFormat->iYWidth; Y++)
977  {
978  for (X = 0; X < localFormat->iXWidth; X++)
979  {
980  k = X * localFormat->iYWidth + Y;
981  i = fst + k;
982  i2 = snd + k;
983  if ((compressed->tarType[F] == 'T') && (compressed->tarType[F+chopInterval] == 'S'))
984  current = (inData[R][i] + scalingOffset) - (inData[R][i2] + scalingOffset);
985  else if ((compressed->tarType[F] == 'S') && (compressed->tarType[F+chopInterval] == 'T'))
986  current = (inData[R][i2] + scalingOffset) - (inData[R][i] + scalingOffset);
987 
988  // Load
989  compressed->image[R][k] += current;
990  }
991  }
992  }
993  }
994  F += chopInterval;
995  }
996  }
997 
998  aprioriSteppingPhase = inSteppingPhase[F-1]; // F-1 because of the above loop
999  if (aprioriSteppingPhase == localFormat->framesPerScan) aprioriSteppingPhase = 0;
1000 
1001  frame0 = iF; // To output on return, below
1002  iF += localFormat->numOfFrames; // Set it correctly for the next call to this function
1003 
1004  // Clean up now:
1005  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
1006  {
1007  free (inData[R]);
1008  free (inTARTYP[R]);
1009  free (inMask[R]);
1010  }
1011  if (pMask) qfits_table_close (pMask);
1012  if (pTable) qfits_table_close (pTable);
1013  if (inLOCALOPD) free(inLOCALOPD);
1014  if (inOPD) free(inOPD);
1015  if (inTIME) free(inTIME);
1016  free (inData);
1017  free (inTARTYP);
1018  free (inMask);
1019  free (inSteppingPhase);
1020  free (foundData);
1021  free (indexData);
1022  free (foundTARTYP);
1023  free (indexTARTYP);
1024  free (indexMask);
1025  free (normalization);
1026  free (dataName);
1027 
1028  return (frame0);
1029 }
1030 /*****************************************************************************/
1031