VIRCAM Pipeline  1.3.4
imcore_conf.c
1 /* $Id: imcore_conf.c,v 1.25 2012-03-06 10:02:03 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2012-03-06 10:02:03 $
24  * $Revision: 1.25 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <math.h>
33 
34 #include <cpl.h>
35 #include "../vircam_fits.h"
36 #include "../vircam_pfits.h"
37 #include "../vircam_wcsutils.h"
38 
39 #include "ap.h"
40 #include "util.h"
41 #include "imcore.h"
42 #include "floatmath.h"
43 #include "imcore_version.h"
44 
45 #define FATAL_ERR(_a) {freetable(tab); cpl_msg_error(fctid,_a); tidy(); return(VIR_FATAL);}
46 
47 #define NW 5
48 
49 static float *smoothed = NULL;
50 static float *smoothedc = NULL;
51 static unsigned char *mflag = NULL;
52 static float *indata = NULL;
53 static int *confdata = NULL;
54 static float *confsqrt = NULL;
55 static ap_t ap;
56 static int freeconf = 0;
57 
58 static float weights[NW*NW];
59 static float weightc[NW*NW];
60 static long nx;
61 static long ny;
62 
63 static void crweights(float);
64 static void convolve(int);
65 static void tidy(void);
66 
69 /*---------------------------------------------------------------------------*/
144 /*---------------------------------------------------------------------------*/
145 
146 extern int imcore_conf(vir_fits *infile, vir_fits *conf, int ipix,
147  float threshold, int icrowd, float rcore, int nbsize,
148  int cattyp, float filtfwhm, vir_tfits **outcat) {
149 
150  int i,retval,mulpix,j,nw2,status;
151  float fconst,nullval,skymed,skysig,thresh,xintmin,offset;
152  float *current,isatbc,isat,junk,*currentc;
153  long npix,nxc,nyc,npts;
154  cpl_image *map,*cmap;
155  cpl_propertylist *plist,*extra;
156  char card[64];
157  const char *fctid = "imcore_conf";
158 
159  /* Initialise output */
160 
161  *outcat = NULL;
162 
163  /* Useful constants */
164 
165  fconst = CPL_MATH_LOG2E;
166  nullval = 0.0;
167  cattype = cattyp;
168  nobjects = 0;
169 
170  /* Open input image */
171 
172  map = vircam_fits_get_image(infile);
173  if ((indata = cpl_image_get_data_float(map)) == NULL)
174  FATAL_ERR("Error getting image data");
175  nx = (long)cpl_image_get_size_x(map);
176  ny = (long)cpl_image_get_size_y(map);
177  npts = nx*ny;
178  if (vircam_pfits_get_gain(vircam_fits_get_ehu(infile),&gain) != VIR_OK)
179  gain = 5.0;
180 
181  /* Open the associated confidence map, if it exists */
182 
183  if (conf != NULL) {
184  cmap = vircam_fits_get_image(conf);
185  if ((confdata = cpl_image_get_data(cmap)) == NULL)
186  FATAL_ERR("Error getting confidence map data");
187  nxc = (long)cpl_image_get_size_x(cmap);
188  nyc = (long)cpl_image_get_size_y(cmap);
189  if ((nx != nxc) || (ny != nyc))
190  FATAL_ERR("Input image and confidence dimensions don't match");
191  freeconf = 0;
192  } else {
193  confdata = cpl_malloc(npts*sizeof(*confdata));
194  for (i = 0; i < npts; i++)
195  confdata[i] = 100;
196  freeconf = 1;
197  cmap = NULL;
198  }
199  confsqrt = cpl_malloc(npts*sizeof(*confsqrt));
200  for (i = 0; i < npts; i++)
201  confsqrt[i] = sqrt(0.01*(float)confdata[i]);
202 
203  /* Get mflag array for flagging saturated pixels */
204 
205  npix = nx*ny;
206  mflag = cpl_calloc(npix,sizeof(*mflag));
207 
208  /* Open the ap structure and define some stuff in it */
209 
210  ap.lsiz = nx;
211  ap.csiz = ny;
212  ap.inframe = map;
213  ap.conframe = cmap;
214  ap.xtnum = vircam_fits_get_nexten(infile);
215  apinit(&ap);
216  ap.indata = indata;
217  ap.confdata = confdata;
218  ap.multiply = 1;
219  ap.ipnop = ipix;
220  ap.mflag = mflag;
221  ap.rcore = rcore;
222  ap.filtfwhm = filtfwhm;
223  ap.icrowd = icrowd;
224  ap.fconst = fconst;
225 
226  /* Open the output catalogue FITS table */
227 
228  tabinit(&ap);
229 
230  /* Set up the data flags */
231 
232  for (i = 0; i < npix; i++)
233  if (confdata[i] == 0)
234  mflag[i] = MF_ZEROCONF;
235  else if (indata[i] < STUPID_VALUE)
236  mflag[i] = MF_STUPID_VALUE;
237  else
238  mflag[i] = MF_CLEANPIX;
239 
240  /* Compute a saturation level before background correction. */
241 
242  retval = imcore_backstats(&ap,nullval,1,&skymed,&skysig,&isatbc);
243  if (retval != VIR_OK)
244  FATAL_ERR("Error calculating saturation level");
245 
246  /* Flag up regions where the value is above the saturation level
247  determined above. */
248 
249  for (i = 0; i < npix ; i++)
250  if (mflag[i] == MF_CLEANPIX && indata[i] > isatbc)
251  mflag[i] = MF_SATURATED;
252 
253  /* Compute the background variation and remove it from the data*/
254 
255  retval = imcore_background(&ap,nbsize,nullval);
256  if (retval != VIR_OK)
257  FATAL_ERR("Error calculating background");
258 
259  /* Compute a saturation level after background correction. */
260 
261  retval = imcore_backstats(&ap,nullval,1,&skymed,&skysig,&isat);
262  if (retval != VIR_OK)
263  FATAL_ERR("Error calculating saturation");
264 
265 
266  /* Compute background statistics */
267 
268  retval = imcore_backstats(&ap,nullval,0,&skymed,&skysig,&junk);
269  if (retval != VIR_OK)
270  FATAL_ERR("Error calculating background stats");
271 
272  /* Get the propertly list for the input file and add some info*/
273 
274  plist = vircam_fits_get_ehu(infile);
275  cpl_propertylist_update_float(plist,"ESO DRS SKYLEVEL",skymed);
276  cpl_propertylist_set_comment(plist,"ESO DRS SKYLEVEL",
277  "[adu] Median sky brightness");
278  cpl_propertylist_update_float(plist,"ESO DRS SKYNOISE",skysig);
279  cpl_propertylist_set_comment(plist,"ESO DRS SKYNOISE",
280  "[adu] Pixel noise at sky level");
281 
282  /* Take mean sky level out of data. Uncomment the bit below if
283  we start working out the covariance in dribbled data. See cirdr
284  version. */
285 
286  for (i = 0; i < nx*ny; i++) {
287  indata[i] -= skymed;
288  /* if (indata[i]*confsqrt[i] > 3.0*skysig && */
289  /* mflag[i] != MF_SATURATED && mflag[i] != MF_STUPID_VALUE) */
290  /* mflag[i] = MF_3SIG; */
291  }
292 
293  /* Work out isophotal detection threshold levels */
294 
295  thresh = threshold*skysig;
296 
297  /* Minimum intensity for consideration */
298 
299  xintmin = 1.5*thresh*((float)ipix);
300 
301  /* Minimum size for considering multiple images */
302 
303  mulpix = MAX(8,2*ipix);
304 
305  /* Actual areal profile levels: T, 2T, 4T, 8T,...but written wrt T
306  i.e. threshold as a power of 2 */
307 
308  offset = logf(thresh)*fconst;
309 
310  /* Get a bit of workspace for buffers */
311 
312  smoothed = cpl_malloc(nx*sizeof(*smoothed));
313  smoothedc = cpl_malloc(nx*sizeof(*smoothedc));
314 
315  /* Define a few things more things in ap structure */
316 
317  ap.mulpix = mulpix;
318  ap.areal_offset = offset;
319  ap.thresh = thresh;
320  ap.xintmin = xintmin;
321  ap.sigma = skysig;
322  ap.background = skymed;
323  ap.saturation = (float)isat;
324 
325  /* Set the weights */
326 
327  crweights(filtfwhm);
328  nw2 = NW/2;
329 
330  /* Right, now for the extraction loop. Begin by defining a group of
331  three rows of data and confidence */
332 
333  for (j = nw2; j < ny-nw2; j++) {
334  current = indata + j*nx;
335  currentc = confsqrt + j*nx;
336  convolve(j);
337 
338  /* Do the detection now */
339 
340  apline(&ap,current,currentc,smoothed,smoothedc,j,NULL);
341 
342  /* Make sure we're not overruning the stacks */
343 
344  if (ap.ibstack > (ap.maxbl - ap.lsiz))
345  apfu(&ap);
346  if (ap.ipstack > (ap.maxpa*3/4))
347  for (i = 0; i < ap.maxpa*3/8; i++)
348  apfu(&ap);
349 
350  /* See if there are any images to terminate */
351 
352  if (ap.ipstack > 1)
353  terminate(&ap);
354  }
355 
356  /* Post process. First truncate the cpl_table to the correct size and then
357  work out an estimate of the seeing */
358 
359  cpl_table_set_size(tab,nobjects);
360  retval = do_seeing(&ap);
361  if (retval != VIR_OK)
362  FATAL_ERR("Error working out seeing");
363  tabclose(&ap);
364 
365  /* Create a property list with extra parameters. First QC parameters */
366 
367  extra = cpl_propertylist_duplicate(vircam_fits_get_ehu(infile));
368  cpl_propertylist_update_float(extra,"ESO QC SATURATION",ap.saturation);
369  cpl_propertylist_set_comment(extra,"ESO QC SATURATION",
370  "[adu] Saturation level");
371  cpl_propertylist_update_float(extra,"ESO QC MEAN_SKY",ap.background);
372  cpl_propertylist_set_comment(extra,"ESO QC MEAN_SKY",
373  "[adu] Median sky brightness");
374  cpl_propertylist_update_float(extra,"ESO QC SKY_NOISE",ap.sigma);
375  cpl_propertylist_set_comment(extra,"ESO QC SKY_NOISE",
376  "[adu] Pixel noise at sky level");
377 
378  /* Now DRS parameters */
379 
380  cpl_propertylist_update_float(extra,"ESO DRS THRESHOL",ap.thresh);
381  cpl_propertylist_set_comment(extra,"ESO DRS THRESHOL",
382  "[adu] Isophotal analysis threshold");
383  cpl_propertylist_update_int(extra,"ESO DRS MINPIX",ap.ipnop);
384  cpl_propertylist_set_comment(extra,"ESO DRS MINPIX",
385  "[pixels] Minimum size for images");
386  cpl_propertylist_update_int(extra,"ESO DRS CROWDED",ap.icrowd);
387  cpl_propertylist_set_comment(extra,"ESO DRS CROWDED",
388  "Crowded field analysis flag");
389  cpl_propertylist_update_float(extra,"ESO DRS RCORE",ap.rcore);
390  cpl_propertylist_set_comment(extra,"ESO DRS RCORE",
391  "[pixels] Core radius for default profile fit");
392  cpl_propertylist_update_float(extra,"ESO DRS SEEING",ap.fwhm);
393  cpl_propertylist_set_comment(extra,"ESO DRS SEEING",
394  "[pixels] Average FWHM");
395  cpl_propertylist_update_float(extra,"ESO DRS FILTFWHM",ap.filtfwhm);
396  cpl_propertylist_set_comment(extra,"ESO DRS FILTFWHM",
397  "[pixels] FWHM of smoothing kernel");
398  cpl_propertylist_update_int(extra,"ESO DRS XCOL",imcore_xcol);
399  cpl_propertylist_set_comment(extra,"ESO DRS XCOL","Column for X position");
400  cpl_propertylist_update_int(extra,"ESO DRS YCOL",imcore_ycol);
401  cpl_propertylist_set_comment(extra,"ESO DRS YCOL","Column for Y position");
402  cpl_propertylist_update_int(extra,"ESO DRS NXOUT",nx);
403  cpl_propertylist_set_comment(extra,"ESO DRS NXOUT",
404  "X Dimension of input image");
405  cpl_propertylist_update_int(extra,"ESO DRS NYOUT",ny);
406  cpl_propertylist_set_comment(extra,"ESO DRS NYOUT",
407  "Y Dimension of input image");
408  snprintf(card,64,"IMCORE version: %s",imcore_version);
409  cpl_propertylist_append_string(extra,"HISTORY",card);
410 
411  /* Now wrap all this stuff up and send it back */
412 
413  plist = cpl_propertylist_duplicate(vircam_fits_get_phu(infile));
414  status = VIR_OK;
415  (void)vircam_tabwcs(extra,imcore_xcol,imcore_ycol,&status);
416  *outcat = vircam_tfits_wrap(tab,NULL,plist,extra);
417 
418  /* Tidy and exit */
419 
420  tidy();
421  return(VIR_OK);
422 }
423 
424 /*---------------------------------------------------------------------------*/
441 /*---------------------------------------------------------------------------*/
442 
443 static void crweights(float filtfwhm) {
444  int i,j,nw2,n;
445  double gsigsq,di,dj;
446  float renorm;
447 
448  /* Get the kernel size */
449 
450  nw2 = NW/2;
451 
452  /* Set the normalisation constants */
453 
454  gsigsq = 1.0/(2.0*pow(MAX(1.0,(double)filtfwhm)/2.35,2.0));
455  renorm = 0.0;
456 
457  /* Now work out the weights */
458 
459  n = -1;
460  for (i = -nw2; i <= nw2; i++) {
461  di = (double)i;
462  di *= gsigsq*di;
463  for (j = -nw2; j <= nw2; j++) {
464  dj = (double)j;
465  dj *= gsigsq*dj;
466  n++;
467  weights[n] = (float)exp(-(di + dj));
468  renorm += weights[n];
469  }
470  }
471 
472  /* Now normalise the weights */
473 
474  n = -1;
475  for (i = -nw2; i <= nw2; i++) {
476  for (j = -nw2; j <= nw2; j++) {
477  n++;
478  weights[n] /= renorm;
479  /* weightc[n] = 0.01*weights[n]; */
480  weightc[n] = weights[n];
481  }
482  }
483 }
484 
485 /*---------------------------------------------------------------------------*/
503 /*---------------------------------------------------------------------------*/
504 
505 static void convolve(int ir) {
506  int i,nw2,ix,jx,jy,n;
507  float *idata,*cdata;
508 
509  /* Zero the summations */
510 
511  for (i = 0; i < nx; i++) {
512  smoothed[i] = 0.0;
513  smoothedc[i] = 0.0;
514  }
515 
516  /* Now big is the smoothing kernel? */
517 
518  nw2 = NW/2;
519 
520  /* Now loop for each column */
521 
522  for (ix = nw2; ix < nx-nw2; ix++) {
523  n = -1;
524  for (jy = ir-nw2; jy <= ir+nw2; jy++) {
525  idata = indata + jy*nx;
526  cdata = confsqrt + jy*nx;
527  for (jx = ix-nw2; jx <= ix+nw2; jx++) {
528  n++;
529  smoothed[ix] += weights[n]*idata[jx];
530  smoothedc[ix] += weightc[n]*idata[jx]*cdata[jx];
531  }
532  }
533  }
534 }
535 
536 static void tidy(void) {
537 
538  if (freeconf)
539  freespace(confdata);
540  freespace(confsqrt);
541  freespace(smoothed);
542  freespace(smoothedc);
543  freespace(mflag);
544  apclose(&ap);
545 }
546 
549 /*
550 
551 $Log: not supported by cvs2svn $
552 Revision 1.24 2010/09/10 11:23:09 jim
553 Fixed declaration of currentc
554 
555 Revision 1.23 2010/09/09 12:09:49 jim
556 Modified to use sqrt weighting with confidence maps. Also added docs
557 
558 Revision 1.22 2009/09/09 09:42:25 jim
559 modified to use CPL defined macros for constants
560 
561 Revision 1.21 2009/05/20 12:22:34 jim
562 Now adds ESO DRS NXOUT and NYOUT to header
563 
564 Revision 1.20 2009/01/30 08:26:23 jim
565 Now adds a global version number to header
566 
567 Revision 1.19 2009/01/23 12:24:33 jim
568 Fixed bugs in pixel flagging
569 
570 Revision 1.18 2008/10/13 08:10:56 jim
571 Fixed pixel masking scheme
572 
573 Revision 1.17 2007/05/03 11:15:34 jim
574 Fixed little problem with table wcs
575 
576 Revision 1.16 2007/05/02 09:11:35 jim
577 Modified to allow for inclusion of table WCS keywords into FITS header
578 
579 Revision 1.15 2007/03/01 12:38:26 jim
580 Small modifications after a bit of code checking
581 
582 Revision 1.14 2006/11/27 11:56:59 jim
583 Changed cpl_propertylist_append to cpl_propertylist_update calls
584 
585 Revision 1.13 2006/08/01 11:27:54 jim
586 Modifications to imcore background estimation and to add ability to
587 specify the smoothing kernel width
588 
589 Revision 1.12 2006/07/11 14:51:02 jim
590 Fixed small bug in the range of the main loop
591 
592 Revision 1.11 2006/07/03 09:33:19 jim
593 Fixed a few things to keep the compiler happy
594 
595 Revision 1.10 2006/06/30 21:31:09 jim
596 MOdifications to background routines and smoothing kernel
597 
598 Revision 1.9 2006/06/13 14:06:10 jim
599 Gets gain estimate from header
600 
601 Revision 1.8 2006/06/06 13:04:22 jim
602 Fixed apline so that it now takes confidence map info correctly
603 
604 Revision 1.7 2006/05/30 12:13:53 jim
605 Initialised nobjects
606 
607 Revision 1.6 2006/05/18 12:35:01 jim
608 Fixed bug for header writing
609 
610 Revision 1.5 2006/03/15 10:43:42 jim
611 Fixed a few things
612 
613 Revision 1.4 2006/03/01 10:31:29 jim
614 Now uses new vir_fits objects
615 
616 Revision 1.3 2005/09/22 08:40:42 jim
617 Fixed some bugs in imcore and added classification Removed some unnecessary
618 files
619 
620 Revision 1.2 2005/09/20 15:07:47 jim
621 Fixed a few bugs and added a few things
622 
623 Revision 1.1 2005/09/13 13:25:29 jim
624 Initial entry after modifications to make cpl compliant
625 
626 
627 */
int vircam_tabwcs(cpl_propertylist *p, int xcol, int ycol, int *status)
void terminate(ap_t *ap)
Check for objects that have terminated.
Definition: terminate.c:130
int imcore_backstats(ap_t *ap, float nullval, int satonly, float *skymed, float *skysig, float *sat)
Work out robust background estimate over a whole input image.
vir_tfits * vircam_tfits_wrap(cpl_table *tab, vir_tfits *model, cpl_propertylist *phu, cpl_propertylist *ehu)
Definition: vircam_tfits.c:703
int vircam_fits_get_nexten(vir_fits *p)
Definition: vircam_fits.c:382
cpl_image * vircam_fits_get_image(vir_fits *p)
Definition: vircam_fits.c:349
int vircam_pfits_get_gain(const cpl_propertylist *plist, float *gain)
Get the value of the detector gain.
Definition: vircam_pfits.c:709
int imcore_conf(vir_fits *infile, vir_fits *conf, int ipix, float threshold, int icrowd, float rcore, int nbsize, int cattyp, float filtfwhm, vir_tfits **outcat)
Do source extraction.
Definition: imcore_conf.c:146
void apclose(ap_t *ap)
Close ap structure.
Definition: apinit.c:185
void apline(ap_t *ap, float dat[], float conf[], float smoothed[], float smoothedc[], int j, unsigned char *bpm)
Detect objects on a line of data.
Definition: apline.c:75
cpl_propertylist * vircam_fits_get_phu(vir_fits *p)
Definition: vircam_fits.c:416
int tabclose(ap_t *ap)
Close the table structure.
Definition: create_table.c:234
void apfu(ap_t *ap)
Get rid of the largest contributor in an ap structure.
Definition: terminate.c:192
cpl_propertylist * vircam_fits_get_ehu(vir_fits *p)
Definition: vircam_fits.c:457
void apinit(ap_t *ap)
Initialise the ap structure.
Definition: apinit.c:64
int do_seeing(ap_t *ap)
Do seeing estimate.
Definition: create_table.c:120
void tabinit(ap_t *ap)
Initialise catalogues.
Definition: create_table.c:65
int imcore_background(ap_t *ap, int nbsize, float nullval)
Model and create background map.