MIDI Pipeline Reference Manual  2.8.3
preProcFrgSP.c
1 /******************************************************************************
2 *******************************************************************************
3 * European Southern Observatory
4 * VLTI MIDI Data Reduction Software
5 *
6 * Module name: preProcFrgSP.c
7 * Description: Contains routines for pre processing
8 *
9 * History:
10 * 08-Aug-05 (csabet) Created
11 *******************************************************************************
12 ******************************************************************************/
13 
14 /******************************************************************************
15 * Compiler directives
16 ******************************************************************************/
17 
18 /******************************************************************************
19 * Include files
20 ******************************************************************************/
21 #include <stdio.h>
22 #include <cpl.h>
23 #include <math.h>
24 #include "midiGlobal.h"
25 #include "fft.h"
26 #include "midiLib.h"
27 #include "memoryHandling.h"
28 #include "errorHandling.h"
29 #include "midiFitsUtility.h"
30 #include "diagnostics.h"
31 #include "preProcFrgSP.h"
32 
33 /**********************************************************
34 * Constant definitions
35 **********************************************************/
36 
37 /**********************************************************
38 * Global Variables
39 **********************************************************/
40 
41 /*============================ C O D E A R E A ===========================*/
42 
43 
44 
45 /******************************************************************************
46 * European Southern Observatory
47 * VLTI MIDI Data Reduction Software
48 *
49 * Module name: preProcFrgSP
50 * Input/Output: See function arguments to avoid duplication
51 * Description: Prepares the data for DISPERSED mode of processing.
52 * This is configured for SCI_PHOT
53 *
54 * History:
55 * 07-Jul-05 (csabet) Created
56 ******************************************************************************/
57 void preProcFrgSP (
58  UserOptions *options, // In: User parameters
59  FilterData *filterInfo, // IO: Pointer to the filter data structure
60  MidiFiles *fileNames, // In: Pointer to the MIDI file structure
61  CompressedData *compressedInterf, // Ou: Pointer to the compressed interferometry data structure
62  CompressedData *compressedPhotomA, // Ou: Pointer to the compressed photom A data structure
63  CompressedData *compressedPhotomB, // Ou: Pointer to the compressed photom B data structure
64  ImageFormat *formatInterf, // In: Interf size parameters
65  ImageFormat *formatPhotomA, // In: PhotA size parameters
66  ImageFormat *formatPhotomB, // In: PhotB size parameters
67  int *error) // Ou: Error status
68 {
69 
70  // Local Declarations
71  // ------------------
72  const char routine[] = "preProcFrgSP";
73  FILE *inFitsBatchPtr=NULL;
74  int newFile, extNumOfImagingDataFile, extNumOfImagingDataMask, R;
75  char *maskFile, *cleanString, *classification, *stringTemp, *fileName, *title;
76  unsigned int loopCount = 0;
77  ImageFormat *localFormat;
78  enum ObsTechnique obsTech;
79 
80  // Algorithm
81  // ---------
82  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
83  if (diagnostic > 4) fprintf(midiReportPtr, "Invoking routine '%s' \n", routine);
84 
85  cpl_msg_info(cpl_func,"\nReducing data of batch %d \n", batchNumber);
86  cpl_msg_info(cpl_func,"---------------------- \n");
87  fprintf (midiReportPtr, "\nReducing data of batch %d \n", batchNumber);
88  fprintf (midiReportPtr, "---------------------- \n");
89 
90  // Reset status
91  *error = 0;
92  compressedInterf->exists = 0;
93  compressedPhotomA->exists = 0;
94  compressedPhotomB->exists = 0;
95  newFile = 1;
96  obsTech = UNKNOWN;
97 
98  // Allocate memory
99  stringTemp = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
100  classification = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
101  cleanString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
102  localFormat = callocImageFormat ();
103  maskFile = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
104 
105  // Open the list of files
106  if ((inFitsBatchPtr = fopen (fileNames->inFitsBatch, "r")) == NULL)
107  {
108  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
109  "Cannot open input FITS file list. No compression has been carried out for this batch");
110  free (cleanString);
111  free (localFormat);
112  free (stringTemp);
113  free (classification);
114  freeImageFormat (localFormat);
115  *error = 1;
116  return;
117  }
118 
119  // Loop through the list of files and compress data
120  while (fgets (stringTemp, MAX_STRING_LENGTH, inFitsBatchPtr) != NULL)
121  {
122  sprintf (classification, "%s", "");
123  sscanf (stringTemp, "%s%s", fileNames->inFitsName, classification);
124  cpl_msg_info(cpl_func,"\nProcessing file %s \n", fileNames->inFitsName);
125  fprintf (midiReportPtr, "\nProcessing file %s \n", fileNames->inFitsName);
126 
127  // Get 'extNumOfImagingDataFile' extension number of IMAGING_DATA in input file
128  extNumOfImagingDataFile = findImagingDataExtension (fileNames->inFitsName, TAB_IMAGING_DATA, error);
129  if (*error) break;
130 
131  // Get Image Format parameters
132  if (extNumOfImagingDataFile > 0)
133  {
134  getImageFormat (fileNames->inFitsName, extNumOfImagingDataFile, localFormat, error);
135  if (*error) break;
136  }
137  else localFormat->hasData = 0;
138 
139  // Get filter data and mask file name from the first FITS file
140  if (loopCount == 0)
141  {
142  getFilterData (fileNames->inFitsName, filterInfo, error);
143  if (*error) break;
144  else fprintf (midiReportPtr, "First file of batch = %s (QCLOG)\n", fileNames->inFitsName);
145 
146  // Select mask file
147  selectMask (options->maskMode, fileNames, maskFile, error);
148  if (*error) break;
149 
150  // Get 'extNumOfImagingDataMask' extension number of IMAGING_DATA in mask file
151  extNumOfImagingDataMask = findImagingDataExtension (maskFile, MASK_IMAGING_DATA, error);
152  if (*error) break;
153 
154  cpl_msg_info(cpl_func,"Mask File is %s\n", maskFile);
155  fprintf( midiReportPtr, "Mask File is %s\n", maskFile);
156  }
157 
158  // Check if the file has data
159  if (localFormat->hasData)
160  {
161  // Check file Category and Type
162  if (strcmp (localFormat->obsTech, "INTERFEROMETRY") == 0)
163  {
164  // Check previous TYPE
165  if (obsTech != INTERF) newFile = 1;
166 
167  // Set the TYPE flag
168  obsTech = INTERF;
169 
170  // Check consistency
171  if ((formatInterf->numOfDetectorRegions != localFormat->numOfDetectorRegions) ||
172  (formatInterf->iXWidth != localFormat->iXWidth) || (formatInterf->iYWidth != localFormat->iYWidth))
173  {
174  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Expected format is incorrect");
175  *error = 1;
176  break;
177  }
178 
179  // The Interferometry files contribute to the computation of the visibilities
180  organiseFrgSP (newFile, extNumOfImagingDataFile, fileNames, extNumOfImagingDataMask,
181  maskFile, localFormat, formatInterf, compressedInterf, compressedPhotomA,
182  compressedPhotomB, error);
183 
184  if (*error) break;
185  newFile = 0; // So the next time through, will know not to start at frame 0
186  }
187  else if ((strcmp (localFormat->obsTech, "IMAGE,WINDOW,CHOPNOD") == 0) &&
188  (strcmp(localFormat->shutterId,"AOPEN") == 0))
189  {
190  if (diagnostic > 1) midiReportInfo (midiReportPtr,
191  routine, __FILE__, __LINE__, "The above Photometry file will not be processed");
192  obsTech = PHOTOMA;
193  }
194  else if ((strcmp (localFormat->obsTech, "IMAGE,WINDOW,CHOPNOD") == 0) &&
195  (strcmp(localFormat->shutterId,"BOPEN") == 0))
196  {
197  if (diagnostic > 1) midiReportInfo (midiReportPtr,
198  routine, __FILE__, __LINE__, "The above Photometry file will not be processed");
199  obsTech = PHOTOMB;
200  }
201  else
202  {
203  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Unknown Catg, Type or Tech");
204  *error = 1;
205  break;
206  }
207  }
208  else
209  {
210  sprintf (midiMessage, "No data tables in %s. Not processed", fileNames->inFitsName);
211  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
212  }
213  loopCount++;
214  }
215 
216  // Close files and release memory
217  fclose (inFitsBatchPtr);
218  free (cleanString);
219  free (stringTemp);
220  free (classification);
221  freeImageFormat (localFormat);
222  free (maskFile);
223 
224  // Check if reduction has been carried out
225  cpl_msg_info(cpl_func,"\nCompression status \n");
226  cpl_msg_info(cpl_func,"------------------ \n");
227  if (compressedInterf->exists)cpl_msg_info(cpl_func,"Created Compressed Interferometry Data\n");
228  if (compressedPhotomA->exists)cpl_msg_info(cpl_func,"Created Compressed Photometry A Data\n");
229  if (compressedPhotomB->exists)cpl_msg_info(cpl_func,"Created Compressed Photometry B Data\n");
230  cpl_msg_info(cpl_func,"\n");
231 
232  fprintf (midiReportPtr, "\nCompression status \n");
233  fprintf (midiReportPtr, "------------------ \n");
234  if (compressedInterf->exists) fprintf (midiReportPtr, "Created Compressed Interferometry Data\n");
235  if (compressedPhotomA->exists) fprintf (midiReportPtr, "Created Compressed Photometry A Data\n");
236  if (compressedPhotomB->exists) fprintf (midiReportPtr, "Created Compressed Photometry B Data\n");
237  fprintf (midiReportPtr, "\n");
238 
239  if (!(compressedInterf->exists) ||
240  !(compressedPhotomA->exists) || !(compressedPhotomB->exists) || *error)
241  {
242  *error = 1;
243  sprintf (midiMessage,
244  "Cannot continue. Need the following compressed data: Interf, PhotomA, PhotomB");
245  midiReportWarning (midiReportPtr,
246  routine, __FILE__, __LINE__, midiMessage);
247 
248  return;
249  }
250 
251  // Display images
252  if (diagnostic > 1 && plotFile)
253  {
254  fileName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
255  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
256 
257  for (R = 0; R < formatInterf->numOfRegionsToProcess; R++)
258  {
259  sprintf (fileName, "3dInterfDATA%d", R+2);
260  sprintf (title, "Interferometry DATA %d (Masked and Sky removed)", R+2);
261  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
262  compressedInterf->image[R], formatInterf->iXWidth, formatInterf->iYWidth, "lines", "3");
263  }
264 
265  for (R = 0; R < formatPhotomA->numOfRegionsToProcess; R++)
266  {
267  sprintf (fileName, "3dPhotomADATA%d", R+1);
268  sprintf (title, "Photometry A DATA %d (Masked and Sky removed)", R+1);
269  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
270  compressedPhotomA->image[R], formatPhotomA->iXWidth, formatPhotomA->iYWidth, "lines", "3");
271  }
272 
273  for (R = 0; R < formatPhotomB->numOfRegionsToProcess; R++)
274  {
275  sprintf (fileName, "3dPhotomBDATA%d", R+4);
276  sprintf (title, "Photometry B DATA %d (Masked and Sky removed)", R+4);
277  midiCreatePlotFile3D (fileName, title, "X", "Y", "Flux", 0,
278  compressedPhotomB->image[R], formatPhotomB->iXWidth, formatPhotomB->iYWidth, "lines", "3");
279  }
280 
281  free (fileName);
282  free (title);
283  }
284 
285  return;
286 }
287 /*****************************************************************************/
288 
289 
290 
291 
292 /******************************************************************************
293 * European Southern Observatory
294 * VLTI MIDI Data Reduction Software
295 *
296 * Module name: organiseFrgSP
297 * Input/Output: See function arguments to avoid duplication
298 * Description: Prepares SCI_PHOT files for compression
299 *
300 * History:
301 * 03-Nov-05 (csabet) Created
302 ******************************************************************************/
303 void organiseFrgSP (
304  int newFile, // In: Flag indicating continuation
305  int extNumOfImagingDataFile, // In: Extention number
306  MidiFiles *fileNames, // In: Pointer to the MIDI file structure
307  int extNumOfImagingDataMask, // In: Extention number
308  char *maskFile, // In: Mask file
309  ImageFormat *localFormat, // In: Local format size
310  ImageFormat *formatInterf, // In: Interf size parameters
311  CompressedData *compressedInterf, // Ou: Pointer to the compressed interferometry data structure
312  CompressedData *compressedPhotomA, // Ou: Pointer to the compressed photom A data structure
313  CompressedData *compressedPhotomB, // Ou: Pointer to the compressed photom B data structure
314  int *error) // Ou: Error status
315 {
316 
317  // Local Declarations
318  // ------------------
319  const char routine[] = "organiseFrgSP";
320  int frame0;
321 
322  // Algorithm
323  // ---------
324  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
325  if (diagnostic > 4) fprintf(midiReportPtr, "Invoking routine '%s' \n", routine);
326 
327  // Initialise
328  *error = 0;
329 
330  // Compress data
331  frame0 = compressFrgSP (fileNames->inFitsName, maskFile, extNumOfImagingDataFile,
332  extNumOfImagingDataMask, compressedInterf, compressedPhotomA, compressedPhotomB,
333  newFile, localFormat, formatInterf->numOfFrames, error);
334  if (*error) return;
335 
336  // Report compression status
337  if (diagnostic)
338  {
339  sprintf (midiMessage, "\nWrote %d frames into Interf and Photom, starting at frame %d\nThus now %d frames total,"
340  " expecting %d altogether. Error=%d \n\n", localFormat->numOfFrames, frame0,
341  localFormat->numOfFrames+frame0, formatInterf->numOfFrames, *error);
342  midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
343  }
344 
345  // Set the flags
346  if ((frame0 + localFormat->numOfFrames) == formatInterf->numOfFrames) // ALL have been read in
347  {
348  compressedInterf->exists = 1; // Mark it as completed
349  compressedPhotomA->exists = 1; // Mark it as completed
350  compressedPhotomB->exists = 1; // Mark it as completed
351 
352  // Now with data read in, display some of the parameters:
353  if (plotFile && diagnostic)
354  {
355  midiCreatePlotFile2D ("Time", "TIME", "Frame", "Time", 0, compressedInterf->time, 0,
356  formatInterf->numOfFrames, 1, 0);
357  midiCreatePlotFile2D ("BigDelayLine", "Net VLTI Delay Line Positions", "Frame",
358  "Net Position", 0, compressedInterf->bigDL, 0, formatInterf->numOfFrames, 1, 0);
359  midiCreatePlotFile2D ("PiezoDelayLine", "Net Piezo Delay Line Position", "Frame", "Net Delay", 0,
360  compressedInterf->localOPD, 0, formatInterf->numOfFrames, 1, 0);
361  }
362  }
363 
364  return;
365 }
366 /*****************************************************************************/
367 
368 
369 
370 /******************************************************************************
371 * European Southern Observatory
372 * VLTI MIDI Data Reduction Software
373 *
374 * Module name: compressFrgSP
375 * Input/Output: See function arguments to avoid duplication
376 * Description: This routine prepares the interferomerty data for dispersed
377 * processing. In DISPERSED mode iDispFringe points to a 3D structure.
378 * One dimension is time (frame), the second dimension is wave (produced
379 * by collapsing each X, Y image into one line) and the third dimension
380 * is the number of sub-regions. This routine is used for
381 * SCI_PHOT templates
382 *
383 * Note:
384 * DATA1 = PHOTOM A
385 * DATA2 = INTERF
386 * DATA3 = INTERF
387 * DATA4 = PHOTOM B
388 *
389 * All 4 data are chopped
390 *
391 * History:
392 * 07-Jul-05 (csabet) Adapted from HS. This module is far too big. Will modularise later
393 ******************************************************************************/
394 int compressFrgSP ( // Ou: Status. Returns number of first frame compressed; -1 if error
395  char *inFitsFile, // In: Name of the input FITS file
396  char *maskFile, // In: Name of the mask file
397  int extNumOfImagingDataFile,// In: Extension number of the IMAGING_DATA in input file
398  int extNumOfImagingDataMask,// In: Extension number of the IMAGING_DATA in mask file
399  CompressedData *compressedInterf, // In: Compressed Interf data
400  CompressedData *compressedPhotomA, // In: Compressed Photom A data
401  CompressedData *compressedPhotomB, // In: Compressed Photom B data
402  int newSet, // In: Flag indicating the arrival of a new set of FITS files
403  ImageFormat *localFormat, // In: Local split file format
404  int numOfFramesMax, // In: Maximum number of frames
405  int *error) // Ou: Error status
406 {
407 
408  // Local Declarations
409  // ------------------
410  const char routine[] = "compressFrgSP";
411  qfits_table *pTable = NULL, *pMask = NULL;
412  short int **inData, *inSteppingPhase;
413  float **inMask, accum, current,
414  *normalization; // Normalization factor. Becomes an array in X for the normalization of the
415  // compressedInterf data, valid for each subRegion then overwritten
416  char **inTARTYP = NULL, *tempStr, fitsColumnString[10], *dataName, *title=NULL, *fileString=NULL;
417  double *inTIME = NULL;
418  int frame0 = - 1; // Gets set to a valid number UNLESS routine terminates in error
419  double (*inLOCALOPD)[2] = NULL, (*inOPD)[2] = NULL;
420  static int aprioriSteppingPhase, iF, channelSelected = 0;
421  static double zeroTime;
422  int i, k, *foundData, foundSteppingPhase = 0, indexSteppingPhase, scalingOffset, *indexData,
423  *indexMask, maskWidthX, maskWidthY, maskSubWindow, F, X, Y, R,
424  *indexTARTYP, *foundTARTYP, startframe=0, frameOffset, indexOPD= -1, indexLOCALOPD= -1,
425  indexTIME= -1, maxstep,
426  tartypMult=2; // Signifies that there are 2 (instead of 1!) characters per tartyp value (only
427  // first is useful). In future, if the problem producing the MIDI files is solved,
428  // can be changed to 1 (only for newer files) or made variable (with some way to
429  // determine whether it is an "old" type file). Note: this problem really has
430  // nothing to do with Qfits, but with the (unintended) way the files are written.
431  int i2, fst, snd, found;
432 
433 
434  // Algorithm
435  // ---------
436  if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking routine '%s' \n", routine);
437  if (diagnostic > 4) fprintf (midiReportPtr, "Invoking routine '%s' \n", routine);
438 
439  // Reset status
440  *error = 0;
441  if (localFormat->numOfDetectorRegions != 4)
442  {
443  *error = 1;
444  sprintf (midiMessage, "Incorrect number of regions. Expected 4, found %d", localFormat->numOfDetectorRegions);
445  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
446  return (-1);
447  }
448 
449  // Allocate memory
450  inData = (short int **) calloc (localFormat->numOfDetectorRegions, sizeof (short int *));
451  inTARTYP = (char **) calloc (localFormat->numOfDetectorRegions, sizeof (char *));
452  inMask = (float **) calloc (localFormat->numOfDetectorRegions, sizeof (float *)); // Now has each OUTPUT region allocated
453  foundData = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
454  indexData = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
455  foundTARTYP = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
456  indexTARTYP = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
457  indexMask = (int *) calloc (localFormat->numOfDetectorRegions, sizeof (int));
458  normalization = (float *) calloc (localFormat->iXWidth, sizeof (float));
459  dataName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
460 
461  // Initialise the indices for a new set of FITS files. The
462  // files may be split the indices ensure continuity of split files
463  if (newSet)
464  {
465  aprioriSteppingPhase = 0;
466  iF = 0;
467  }
468 
469  // Make sure the sum of the split-files numOfFrames is not more than the allowed maximum
470  if ((iF + localFormat->numOfFrames) > numOfFramesMax)
471  localFormat->numOfFrames = numOfFramesMax - iF;
472 
473  // Open IMAGING_DATA = INDEX 2
474  pMask = qfits_table_open (maskFile, extNumOfImagingDataMask);
475  if (!pMask)
476  {
477  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load Mask's IMAGING_DATA");
478  free (inData);
479  free (inTARTYP);
480  free (inMask);
481  free (foundData);
482  free (indexData);
483  free (foundTARTYP);
484  free (indexTARTYP);
485  free (indexMask);
486  free (normalization);
487  free (dataName);
488  *error = 1;
489  return (-1);
490  }
491  pTable = qfits_table_open (inFitsFile, extNumOfImagingDataFile);
492  if (!pTable)
493  {
494  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load Data's IMAGING_DATA");
495  qfits_table_close (pMask);
496  free (inData);
497  free (inTARTYP);
498  free (inMask);
499  free (foundData);
500  free (indexData);
501  free (foundTARTYP);
502  free (indexTARTYP);
503  free (indexMask);
504  free (normalization);
505  free (dataName);
506  *error = 1;
507  return (-1);
508  }
509 
510  // Get data table information
511  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
512  {
513  foundData[R] = 0;
514  indexData[R] = 0;
515  }
516  for (i = 0; i < pTable->nc; i++)
517  {
518  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
519  {
520  sprintf (dataName, "DATA%d", R+1);
521  if (strcmp (pTable->col[i].tlabel, dataName) == 0)
522  {
523  foundData[R] = 1;
524  indexData[R] = i;
525  if (diagnostic)
526  {
527  cpl_msg_info(cpl_func,"Found 'DATA%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
528  fprintf(midiReportPtr, "Found 'DATA%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
529  }
530  }
531 
532  sprintf (dataName, "TARTYP%d", R+1);
533  if (strcmp (pTable->col[i].tlabel, dataName) == 0)
534  {
535  foundTARTYP[R] = 1;
536  indexTARTYP[R] = i;
537  if (diagnostic)
538  {
539  cpl_msg_info(cpl_func,"Found 'TARTYP%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
540  fprintf(midiReportPtr, "Found 'TARTYP%d' at column %d in data file %s \n", R+1, i+1, inFitsFile);
541  }
542  }
543  }
544  if (strcmp (pTable->col[i].tlabel, "STEPPING_PHASE") == 0)
545  {
546  foundSteppingPhase = 1;
547  indexSteppingPhase = i;
548  }
549  if (strcmp (pTable->col[i].tlabel, "OPD") == 0)
550  {
551  indexOPD = i;
552  }
553  if (strcmp (pTable->col[i].tlabel, "LOCALOPD") == 0)
554  {
555  indexLOCALOPD = i;
556  }
557  if (strcmp (pTable->col[i].tlabel, "TIME") == 0)
558  {
559  indexTIME = i;
560  }
561  }
562 
563  // Now issue warnings
564  if (foundSteppingPhase == 0)
565  {
566  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find STEPPING_PHASE in data FITS file");
567  *error = 1;
568  }
569  if (indexOPD < 0)
570  {
571  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for OPD in data FITS file");
572  *error = 1;
573  }
574  if (indexLOCALOPD < 0)
575  {
576  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for LOCALOPD in data FITS file");
577  *error = 1;
578  }
579  if (indexTIME < 0)
580  {
581  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find column for TIME in data FITS file");
582  *error = 1;
583  }
584  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
585  {
586  if (foundData[R] == 0)
587  {
588  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
589  "Cannot find requested DATA column in data FITS file");
590  *error = 1;
591  }
592  if (foundTARTYP[R] == 0)
593  {
594  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
595  "Cannot find requested TARTYP column in data FITS file");
596  *error = 1;
597  }
598  }
599 
600  // Get mask data table information
601  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
602  {
603  foundData[R] = 0;
604  indexMask[R] = 0;
605  }
606  for (i = 0; i < pMask->nc; i++)
607  {
608  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
609  {
610  sprintf (dataName, "DATA%d", R+1);
611  if (strcmp (pMask->col[i].tlabel, dataName) == 0)
612  {
613  foundData[R] = 1;
614  indexMask[R] = i;
615  if (diagnostic)
616  {
617  cpl_msg_info(cpl_func,"Found 'DATA%d' at column %d in mask file %s \n", R+1, i+1, maskFile);
618  fprintf(midiReportPtr, "Found 'DATA%d' at column %d in mask file %s \n", R+1, i+1, maskFile);
619  }
620  }
621  }
622  }
623 
624  // Now issue warnings
625  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
626  {
627  if (foundData[R] == 0)
628  {
629  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__,
630  "Cannot find requested DATA column in mask FITS file");
631  *error = 1;
632  }
633  }
634 
635  // Get subwindow format for the mask file
636  sprintf (fitsColumnString, "TDIM%d", indexMask[1]+1); // Point it to the column of a valid region
637  tempStr = qfits_query_ext (maskFile, fitsColumnString, extNumOfImagingDataMask);
638  sscanf (tempStr, "'(%d,%d) '", &maskWidthX, &maskWidthY);
639  maskSubWindow = maskWidthX * maskWidthY;
640  if (diagnostic)cpl_msg_info(cpl_func,"Mask sub-window size = %d\n", maskSubWindow);
641  if (diagnostic)cpl_msg_info(cpl_func,"Data sub-window size = %d\n", localFormat->subWindowSize);
642  fprintf (midiReportPtr, "Mask sub-window size = %d\n", maskSubWindow);
643  fprintf (midiReportPtr, "Data sub-window size = %d\n", localFormat->subWindowSize);
644  if (maskSubWindow != localFormat->subWindowSize)
645  {
646  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Mask has incompatible sub window size");
647  *error = 1;
648  }
649 
650  // Get MAXSTEP value
651  tempStr = qfits_query_ext (inFitsFile, "MAXSTEP", extNumOfImagingDataFile);
652  if (tempStr != NULL)
653  {
654  if (diagnostic)cpl_msg_info(cpl_func,"MAXSTEP = %s\n", tempStr);
655  fprintf( midiReportPtr, "MAXSTEP = %s\n", tempStr);
656  sscanf (tempStr, "%d", &maxstep);
657  }
658  else
659  {
660  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read MAXSTEP");
661  *error = 1;
662  }
663 
664  // Read column COL_STEPPING_PHASE and check compatibility
665  inSteppingPhase = (short int*)qfits_query_column (pTable, indexSteppingPhase, NULL);
666  if (inSteppingPhase[0] != ((aprioriSteppingPhase % localFormat->framesPerScan) + 1))
667  {
668  sprintf (midiMessage, "Incorrect Stepping Phase. Expected %d to %d. Instead found %d to %d",
669  aprioriSteppingPhase+1, maxstep, inSteppingPhase[0], maxstep-1);
670  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
671  *error = 1;
672  }
673 
674  // Release memory and exit this module if there has been a warning. No point going any further
675  if (*error)
676  {
677  qfits_table_close (pTable);
678  qfits_table_close (pMask);
679  free (inData);
680  free (inTARTYP);
681  free (inMask);
682  free (foundData);
683  free (indexData);
684  free (foundTARTYP);
685  free (indexTARTYP);
686  free (indexMask);
687  free (normalization);
688  free (inSteppingPhase);
689  free (dataName);
690  return (-1);
691  }
692 
693  // Read DATA and TARTYP for all regions from input files
694  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
695  {
696  inData[R] = (short int*) qfits_query_column (pTable, indexData[R], NULL);
697  inTARTYP[R] = (char *) qfits_query_column (pTable, indexTARTYP[R], NULL);
698 
699  found = 0;
700  for (F = 0; F < localFormat->numOfFrames; F++)
701  {
702  // Correct the initial anomalies
703  if (newSet)
704  {
705  if (inTARTYP[R][F*tartypMult] == 'U')
706  found = 1;
707  if (!found)
708  inTARTYP[R][F*tartypMult] = 'U';
709  }
710 
711  // Load data and correct tartype problem
712  if (R == 0) (compressedPhotomA->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
713  else if (R == 1) (compressedInterf->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
714  else if (R == 2)
715  {
716  if (inTARTYP[R][F*tartypMult] != inTARTYP[R-1][F*tartypMult])
717  {
718  // All channels and regions are affected
719  for (i = 0; i < localFormat->iXWidth; i++)
720  {
721  compressedInterf->rejectList[i][iF+F] |= BSL_TARTYPE_CROSS;
722  }
723  sprintf (midiMessage, "Incompatible Interferometry TARTYP2 & TARTYP3 at frame %d", iF+F);
724  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
725  }
726  }
727  else if (R == 3)
728  {
729  (compressedPhotomB->tarType)[iF+F] = inTARTYP[R][F*tartypMult];
730  if (inTARTYP[R][F*tartypMult] != inTARTYP[R-3][F*tartypMult])
731  {
732  // All channels and regions are affected
733  for (i = 0; i < localFormat->iXWidth; i++)
734  {
735  compressedPhotomA->rejectList[i][iF+F] |= BSL_TARTYPE_CROSS;
736  compressedPhotomB->rejectList[i][iF+F] |= BSL_TARTYPE_CROSS;
737  }
738  sprintf (midiMessage, "Incompatible Photometry TARTYP1 & TARTYP4 at frame %d", iF+F);
739  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
740  }
741  }
742  }
743  }
744 
745  // Get TIME
746  if(indexTIME >= 0) // Valid ...
747  {
748  startframe = 0;
749  inTIME = (double *) qfits_query_column (pTable, indexTIME, NULL);
750  // Stuff it into time, subtracting off the zero time
751  if (iF == 0) zeroTime = inTIME[0];
752 
753  // Fix a bug found in the data:
754  if (iF == 0)
755  {
756  for (startframe = 0; startframe < ARB_NUM_OF_FRAMES; startframe++)
757  {
758  if ((zeroTime = inTIME[startframe]) > 1.0) // Should be more like 53000 or something!
759  break;
760  }
761  }
762  if (startframe)
763  {
764  if (diagnostic)
765  {
766  cpl_msg_info(cpl_func,"\nLOOK: frames 0 - %d had ZERO for their time field!!\n", startframe-1);
767  fprintf(midiReportPtr, "\nLOOK: frames 0 - %d had ZERO for their time field!!\n", startframe-1);
768  }
769  }
770 
771  for (F = startframe; F < localFormat->numOfFrames; F++)
772  {
773  compressedInterf->time[iF+F] = (float)(inTIME[F] - zeroTime);
774  compressedPhotomA->time[iF+F] = (float)(inTIME[F] - zeroTime);
775  compressedPhotomB->time[iF+F] = (float)(inTIME[F] - zeroTime);
776  if (isnan (compressedInterf->time[iF+F]) ||
777  isnan (compressedPhotomA->time[iF+F]) ||
778  isnan (compressedPhotomB->time[iF+F]))
779  {
780  // All channels and regions are affected
781  for (i = 0; i < localFormat->iXWidth; i++)
782  {
783  compressedInterf->rejectList[i][iF+F] |= BSL_TIME_ERROR;
784  compressedPhotomA->rejectList[i][iF+F] |= BSL_TIME_ERROR;
785  compressedPhotomB->rejectList[i][iF+F] |= BSL_TIME_ERROR;
786  }
787  sprintf (midiMessage, "inTIME has an INVALID value at frame %d", iF+F);
788  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
789  }
790  }
791  }
792 
793  // Get LOCALOPD
794  if (indexLOCALOPD >= 0) // Valid ...
795  {
796  inLOCALOPD = (double (*)[2]) qfits_query_column (pTable, indexLOCALOPD, NULL);
797  // Add up the 2 delay lines, and stuff it into localOPD:
798  for (F = 0; F < localFormat->numOfFrames; F++)
799  {
800  compressedInterf->localOPD[iF+F] = (float)(inLOCALOPD[F][0] + inLOCALOPD[F][1]);
801  compressedPhotomA->localOPD[iF+F] = (float)(inLOCALOPD[F][0] + inLOCALOPD[F][1]);
802  compressedPhotomB->localOPD[iF+F] = (float)(inLOCALOPD[F][0] + inLOCALOPD[F][1]);
803  if (isnan (compressedInterf->localOPD[iF+F]) ||
804  isnan (compressedPhotomA->localOPD[iF+F]) ||
805  isnan (compressedPhotomB->localOPD[iF+F]))
806  {
807  // All channels and regions are affected
808  for (i = 0; i < localFormat->iXWidth; i++)
809  {
810  compressedInterf->rejectList[i][iF+F] |= BSL_LOCALOPD_ERROR;
811  compressedPhotomA->rejectList[i][iF+F] |= BSL_LOCALOPD_ERROR;
812  compressedPhotomB->rejectList[i][iF+F] |= BSL_LOCALOPD_ERROR;
813  }
814  sprintf (midiMessage, "localOPD has an INVALID value at frame %d", iF+F);
815  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
816  }
817  }
818  }
819 
820  // Get OPD
821  if (indexOPD >= 0) // Valid ...
822  {
823  inOPD = (double (*)[2]) qfits_query_column (pTable, indexOPD, NULL);
824  // Add up the 2 delay lines, and stuff it into bigDL:
825  for (F = 0; F < localFormat->numOfFrames; F++)
826  {
827  (compressedInterf->bigDL)[iF+F] = (float)(inOPD[F][0] + inOPD[F][1]);
828  (compressedPhotomA->bigDL)[iF+F] = (float)(inOPD[F][0] + inOPD[F][1]);
829  (compressedPhotomB->bigDL)[iF+F] = (float)(inOPD[F][0] + inOPD[F][1]);
830  if (isnan (compressedInterf->bigDL[iF+F]) ||
831  isnan (compressedPhotomA->bigDL[iF+F]) ||
832  isnan (compressedPhotomB->bigDL[iF+F]))
833  {
834  // All channels and regions are affected
835  for (i = 0; i < localFormat->iXWidth; i++)
836  {
837  compressedInterf->rejectList[i][iF+F] |= BSL_OPD_ERROR;
838  compressedPhotomA->rejectList[i][iF+F] |= BSL_OPD_ERROR;
839  compressedPhotomB->rejectList[i][iF+F] |= BSL_OPD_ERROR;
840  }
841  sprintf (midiMessage, "bigDL has an INVALID value at frame %d", iF+F);
842  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
843  }
844  }
845  }
846 
847  // Read all subregions from mask file
848  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
849  {
850  inMask[R] = (float*) qfits_query_column (pMask, indexMask[R], NULL);
851 
852  // Display mask data
853  if (diagnostic > 1 && plotFile && newSet)
854  {
855  fileString = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
856  title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
857  sprintf (fileString, "3dMaskDATA%d", R+1);
858  sprintf (title, "3D Mask DATA %d", R+1);
859  midiCreatePlotFile3D (fileString, title, "X", "Y", "Flux", 0,
860  inMask[R], localFormat->iXWidth, localFormat->iYWidth, "lines", "3");
861  free (fileString);
862  free (title);
863  }
864  }
865 
866  // Select valid channels
867  if (newSet || !channelSelected)
868  {
869  selectChannels (0, localFormat->numOfDetectorRegions, localFormat, inMask);
870  channelSelected = 1;
871  }
872 
873  // Get the scaling offset
874  for (i = 14; i < 25; i++)
875  {
876  sprintf (dataName, "TZERO%d", i);
877  tempStr = qfits_query_ext (inFitsFile, dataName, extNumOfImagingDataFile);
878  if (tempStr != NULL)
879  {
880  if (diagnostic)cpl_msg_info(cpl_func,"Scaling Offset = %s\n", tempStr);
881  if (diagnostic) fprintf (midiReportPtr, "Scaling Offset = %s\n", tempStr);
882  sscanf (tempStr, "%d", &scalingOffset);
883  break;
884  }
885  }
886  if (tempStr == NULL)
887  {
888  scalingOffset = 0;
889  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Scaling Offset. It is set to 0");
890  }
891 
892  // ACTUAL COMPRESSION OF DATA HERE
893  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
894  {
895  // Determine the normalization for each X (DISPERSED mode)
896  for (X = 0; X < localFormat->iXWidth; X++)
897  {
898  accum = 0.0F;
899  for (Y = 0; Y < localFormat->iYWidth; Y++)
900  accum += inMask[R][Y * localFormat->iXWidth + X];
901 
902  if (accum > 0.0)
903  normalization[X] = 1.F/accum;
904  else
905  normalization[X] = 1.F;
906  }
907 
908  for (F = 0; F < localFormat->numOfFrames; F++)
909  {
910  frameOffset = F * localFormat->subWindowSize;
911  for (X = 0; X < localFormat->iXWidth; X++)
912  {
913  accum = 0.0F;
914  for (Y = 0; Y < localFormat->iYWidth; Y++)
915  {
916  k = Y * localFormat->iXWidth + X;
917  i = frameOffset + k;
918 
919  // If data is bad reject it
920  if (isnan (inData[R][i]))
921  {
922  compressedInterf->rejectList[X][iF+F] |= BSL_DATA_ERROR;
923  compressedPhotomA->rejectList[X][iF+F] |= BSL_DATA_ERROR;
924  compressedPhotomB->rejectList[X][iF+F] |= BSL_DATA_ERROR;
925  sprintf (midiMessage, "inData has an INVALID value at frame %d", (iF+F));
926  midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
927  }
928  else
929  {
930  current = (inData[R][i] + scalingOffset) * inMask[R][k];
931  accum += current;
932  }
933  }
934 
935  // Accumulate further
936  if (R == 0)
937  {
938  (((compressedPhotomA->iDispFringe)[R])[X])[iF+F] =
939  accum * normalization[X];
940  (compressedPhotomA->iFringe1)[iF+F] += accum;
941  }
942  else if (R == 1)
943  {
944  (((compressedInterf->iDispFringe)[R-1])[X])[iF+F] =
945  accum * normalization[X];
946  (compressedInterf->iFringe1)[iF+F] += accum;
947  (compressedInterf->iFringe)[iF+F] += accum;
948  }
949  else if (R == 2)
950  {
951  (((compressedInterf->iDispFringe)[R-1])[X])[iF+F] =
952  accum * normalization[X];
953  (compressedInterf->iFringe2)[iF+F] += accum;
954  (compressedInterf->iFringe)[iF+F] -= accum;
955  }
956  else if (R == 3)
957  {
958  (((compressedPhotomB->iDispFringe)[R-3])[X])[iF+F] =
959  accum * normalization[X];
960  (compressedPhotomB->iFringe1)[iF+F] += accum;
961  }
962  }
963  }
964 
965  // Diagnostic
966  if (diagnostic > 1)
967  {
968  for (F = 0; F < localFormat->numOfFrames-maxstep; F++)
969  {
970  fst = F * localFormat->subWindowSize;
971  snd = (F+maxstep) * localFormat->subWindowSize;
972  if (compressedInterf->tarType[F] != 'U')
973  {
974  for (Y = 0; Y < localFormat->iYWidth; Y++)
975  {
976  for (X = 0; X < localFormat->iXWidth; X++)
977  {
978  k = X * localFormat->iYWidth + Y;
979  i = fst + k;
980  i2 = snd + k;
981  if ((compressedInterf->tarType[F] == 'T') && (compressedInterf->tarType[F+maxstep] == 'S'))
982  current = (inData[R][i] + scalingOffset) - (inData[R][i2] + scalingOffset);
983  else if ((compressedInterf->tarType[F] == 'S') && (compressedInterf->tarType[F+maxstep] == 'T'))
984  current = (inData[R][i2] + scalingOffset) - (inData[R][i] + scalingOffset);
985 
986  // Load
987  if (R == 0)
988  compressedPhotomA->image[R][k] += (current * inMask[R][k]);
989  if (R == 1 || R == 2)
990  compressedInterf->image[R-1][k] += (current * inMask[R][k]);
991  if (R == 3)
992  compressedPhotomB->image[R-3][k] += (current * inMask[R][k]);
993  }
994  }
995  }
996  }
997  F += maxstep;
998  }
999  }
1000 
1001  // Save local parameters and indices
1002  aprioriSteppingPhase = inSteppingPhase[F-1]; // F-1 because of the above loop
1003  if (aprioriSteppingPhase == localFormat->framesPerScan) aprioriSteppingPhase = 0;
1004 
1005  frame0 = iF; // To output on return, below
1006  iF += localFormat->numOfFrames; // Set it correctly for the next call to this function
1007 
1008  // Clean up now:
1009  for (R = 0; R < localFormat->numOfDetectorRegions; R++)
1010  {
1011  free (inData[R]);
1012  free (inTARTYP[R]);
1013  free (inMask[R]);
1014  }
1015  if (pMask) qfits_table_close (pMask);
1016  if (pTable) qfits_table_close (pTable);
1017  if (inLOCALOPD) free(inLOCALOPD);
1018  if (inOPD) free(inOPD);
1019  if (inTIME) free(inTIME);
1020  free (inData);
1021  free (inTARTYP);
1022  free (inMask);
1023  free (inSteppingPhase);
1024  free (foundData);
1025  free (indexData);
1026  free (foundTARTYP);
1027  free (indexTARTYP);
1028  free (indexMask);
1029  free (normalization);
1030  free (dataName);
1031 
1032  return (frame0);
1033 }
1034 /*****************************************************************************/
1035