00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027 #ifdef HAVE_CONFIG_H
00028 #include <config.h>
00029 #endif
00030
00031
00037
00040
00041
00042
00043 #include <math.h>
00044
00045 #include <xsh_drl.h>
00046 #include <xsh_data_pre.h>
00047 #include <xsh_dfs.h>
00048 #include <xsh_pfits.h>
00049 #include <xsh_error.h>
00050 #include <xsh_msg.h>
00051 #include <xsh_badpixelmap.h>
00052
00053 #include <cpl.h>
00054
00055
00056
00057
00058
00059
00060
00061
00062
00063
00064
00065
00066
00067 static void add_noisy_pixel_to_ascii_file( int x, int y )
00068 {
00069 static FILE *fout ;
00070 const char * fname = "noisy_pixels.dat" ;
00071
00072 if ( fout == NULL )
00073 fout = fopen( fname, "w" ) ;
00074 fprintf( fout, "%d %d\n", x, y ) ;
00075 }
00076
00089 static int
00090 flag_noisy_pixels (cpl_imagelist * raws,
00091 cpl_image * bpmap,
00092 cpl_image * noisymap,
00093 int nx, int ny,
00094 xsh_clipping_param * noise_clipping,xsh_instrument* inst)
00095 {
00096 int ix, iy, i;
00097 cpl_image **pimg = NULL;
00098 int nimg, nbad = 0, npix = 0;
00099 double medStackv = 0.,
00100 aveStackv = 0.,
00101 sumStackv = 0., errorStackv = 0.;
00102 double *Stackv = NULL,
00104 *pStackv = NULL;
00105 double *stack = NULL;
00106 double *bisStackv = NULL,
00107 *pbis = NULL;
00108 double diff = 0.0, diff2 = 0.0;
00109 double min = 999999., max = -9999999.;
00110 double maxdelta = 0.0;
00111 cpl_binary *bpmap_mask = NULL;
00112 int idpix = 0;
00113 float ** pixdata = NULL;
00114 cpl_binary **pixmask = NULL;
00115 int nhot = 0;
00116 int totpix = nx * ny;
00117 nimg = cpl_imagelist_get_size(raws);
00118
00119 XSH_MALLOC(pimg, cpl_image*, nimg);
00120
00121
00122 for( i = 0; i < nimg; i++) {
00123 *(pimg + i) = cpl_imagelist_get (raws, i);
00124 }
00125
00126 XSH_MALLOC(Stackv, double, nx * ny);
00127
00128 pStackv = Stackv;
00129
00130 xsh_msg_dbg_low( "Rejected in bpmap: %" CPL_SIZE_FORMAT "", cpl_image_count_rejected( bpmap ));
00131
00132
00133 XSH_MALLOC(stack, double, nimg);
00134
00135
00136 bpmap_mask = cpl_mask_get_data(cpl_image_get_bpm(bpmap));
00137 assure( bpmap_mask != NULL, cpl_error_get_code(),
00138 "Cant get bpmap Mask" );
00139
00140 XSH_MALLOC(bisStackv, double, totpix);
00141 pbis = bisStackv;
00142
00143
00144 {
00145 int j;
00146 XSH_MALLOC(pixdata, float *, nimg);
00147 XSH_MALLOC (pixmask,cpl_binary *,nimg);
00148
00149 for (j = 0; j < nimg; j++) {
00150 *(pixdata + j) = cpl_image_get_data_float(*(pimg + j));
00151 *(pixmask + j) = cpl_mask_get_data(cpl_image_get_bpm(*(pimg + j)));
00152 }
00153 }
00154 idpix = 0;
00155
00156 for (iy = 1; iy <= ny; iy++) {
00157 for (ix = 1; ix <= nx; ix++) {
00158 int j, count = 0;
00159 double sum = 0.;
00160
00161
00162 if ((*(bpmap_mask + idpix) & inst->decode_bp) > 0) {
00163 nbad++;
00164 xsh_msg_dbg_medium("Pixel %d,%d globally bad", ix, iy);
00165 *pStackv = 0.;
00166 } else {
00167 for (j = 0; j < nimg; j++) {
00168 int rej;
00169 double fval = *(*(pixdata + j) + idpix);
00170 rej = (*(*(pixmask + j) + idpix ) & inst->decode_bp);
00171
00172 if (rej > 0) {
00173
00174 xsh_msg_dbg_medium("Dont use pixel %d,%d [%d]", ix, iy, j);
00175 continue;
00176 }
00177 sum += fval;
00178
00179 *(stack + count) = fval;
00180 count++;
00181 }
00182 }
00183
00184 if (count > 1) {
00185
00186 double avg = sum / (double) count;
00187 double dif, dif2 = 0., sigma = 0.;
00188 int k;
00189
00190 xsh_msg_dbg_medium(
00191 " [%d,%d] Count=%d - sum = %lf, avg = %lf", ix, iy, count, sum, avg);
00192
00193 for (k = 0; k < count; k++) {
00194 xsh_msg_dbg_medium( " stack[%d] = %lf", k, *(stack+k));
00195 dif = *(stack + k) - avg;
00196 dif2 += dif * dif;
00197 }
00198 sigma = sqrt(dif2 / (double) (count - 1));
00199 *pStackv = sigma;
00200 *pbis++ = sigma;
00201 sumStackv += sigma;
00202 xsh_msg_dbg_medium(
00203 "++ dif2 = %lf, count= %d, sigma=%lf, Stackv=%lf, sum=%lf", dif2, count, sigma, *pStackv, sumStackv);
00204 if ((npix % 500) == 0)
00205 xsh_msg_dbg_medium(
00206 "+++++ (%d,%d) sumStackv = %lf", ix, iy, sumStackv);
00207 npix++;
00208 } else {
00209 xsh_msg_dbg_medium( "Not enough good pixels (%d)", count);
00210 *pStackv = 0.;
00211 }
00212 pStackv++;
00213 idpix++;
00214 }
00215 }
00216 xsh_msg_dbg_medium( "+++++ sumStackv = %lf", sumStackv);
00217
00218
00219
00220
00221 medStackv = xsh_tools_get_median_double(bisStackv, npix);
00222
00223
00224 aveStackv = sumStackv / (double) npix;
00225 xsh_msg_dbg_low(
00226 "--- Npix=%d - sumStackv=%lf - aveStackv=%lf", npix, sumStackv, aveStackv);
00227
00228
00229
00230 for (pStackv = bisStackv, i = 0; i < npix; i++, pStackv++) {
00231 diff = *pStackv - aveStackv;
00232 if (diff < min)
00233 min = diff;
00234 if (diff > max)
00235 max = diff;
00236 diff2 += diff * diff;
00237 }
00238 cpl_free(bisStackv);
00239 bisStackv = NULL;
00240
00241 xsh_msg_dbg_low("Diff Min: %lf, Max: %lf", min, max);
00242 errorStackv = sqrt(diff2 / (double) (npix - 1));
00243 xsh_msg_dbg_low( "errorStackv=%lf", errorStackv);
00244
00245
00246 maxdelta = noise_clipping->sigma * errorStackv;
00247
00248 xsh_msg_dbg_low(
00249 "Npix:%d/%d, sum: %lf, med:%lf, avg:%lf, err:%lf, delta: %lf", npix, nx * ny, sumStackv, medStackv, aveStackv, errorStackv, maxdelta);
00250
00251 pStackv = Stackv;
00252 idpix = 0;
00253
00254 for (iy = 1; iy <= ny; iy++) {
00255 for (ix = 1; ix <= nx; ix++) {
00256 if ( (*(bpmap_mask + idpix) & inst->decode_bp) == 0 ) {
00257 double delta = *pStackv - medStackv;
00258 if (fabs(delta) > maxdelta) {
00259 nhot++;
00260
00261 xsh_bpmap_set_bad_pixel(bpmap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
00262 if (xsh_debug_level_get() >= XSH_DEBUG_LEVEL_LOW)
00263 add_noisy_pixel_to_ascii_file(ix, iy);
00264
00265 xsh_bpmap_set_bad_pixel(noisymap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
00266
00267 }
00268
00269 }
00270 pStackv++;
00271 idpix++;
00272 }
00273 }
00274 xsh_msg( "Found %d Electronic Pickup Noise Hot Pixels", nhot );
00275
00276 cleanup:
00277 XSH_FREE(stack);
00278 XSH_FREE(Stackv);
00279 XSH_FREE(bisStackv);
00280 XSH_FREE(pimg);
00281 XSH_FREE(pixdata);
00282 XSH_FREE(pixmask);
00283 return nhot;
00284 }
00285
00286 static void
00287 set_pickup_noise_pixels_qc (cpl_propertylist * header, int nbad,
00288 xsh_instrument * instrument)
00289 {
00290 xsh_pfits_set_qc( header, (void *)&nbad,
00291 XSH_QC_BP_MAP_PICKUP_NOISE_PIX, instrument ) ;
00292 }
00293
00304 cpl_frame *
00305 xsh_compute_noise_map (cpl_imagelist * dataList,
00306 cpl_frame * medFrame,
00307 xsh_clipping_param * noise_clipping,
00308 xsh_instrument* instr,
00309 cpl_frame ** noisyFrame
00310 )
00311 {
00312 int nx, ny, nframes = 0;
00313 cpl_image *resBpMap = NULL;
00314 cpl_propertylist *bpmapHeader = NULL;
00315 cpl_frame *resFrame = NULL;
00316 int prevnbad;
00317 const char *bpmapFile = cpl_frame_get_filename (medFrame);
00318 xsh_pre *medPre = NULL;
00319 int iter = 0,curnbad = 0;
00320 char result_name[256] ;
00321 const char* tag = XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr);
00322 char noisy_name[256];
00323 cpl_image * noisyMap = NULL ;
00324 cpl_propertylist * noisymapHeader = NULL ;
00325
00326 int binx=0;
00327 int biny=0;
00328 cpl_image* ima_aux=NULL;
00329
00330
00331
00332
00333
00334
00335
00336 cpl_msg_indent_more ();
00337 xsh_msg ("*** Removing Noisy Pixels (%s)",
00338 xsh_instrument_arm_tostring(instr));
00339
00340
00341 xsh_msg ("Bpmap file = \"%s\"", bpmapFile);
00342
00343
00344 medPre = xsh_pre_load (medFrame,instr);
00345 XSH_ASSURE_NOT_NULL(medPre);
00346
00347 resBpMap = xsh_pre_get_qual (medPre);
00348 bpmapHeader = medPre->qual_header;
00349
00350 nx = medPre->nx;
00351 ny = medPre->ny;
00352
00353 check( noisyMap = cpl_image_new( nx, ny, CPL_TYPE_INT ) ) ;
00354 check( noisymapHeader = cpl_propertylist_duplicate( bpmapHeader ) ) ;
00355 xsh_msg( " Image size: %d,%d", nx, ny ) ;
00356
00357 xsh_set_image_cpl_bpmap (resBpMap, resBpMap, instr->decode_bp);
00358 prevnbad = cpl_image_count_rejected (resBpMap);
00359
00360 nframes = cpl_imagelist_get_size (dataList);
00361 xsh_msg ("%d dark images in image list", nframes);
00362
00363
00364 for (iter = 0; iter < noise_clipping->niter; iter++) {
00365 int nhot=0 ;
00366
00367 xsh_msg (">>>> Iteration Nb %d/%d", iter+1, noise_clipping->niter);
00368 cpl_msg_indent_more ();
00369
00370 if(nframes>1) {
00371
00372 nhot = flag_noisy_pixels (dataList, resBpMap, noisyMap,
00373 nx, ny, noise_clipping,instr);
00374 }
00375 cpl_msg_indent_less ();
00376 if ( nhot == 0 ) break ;
00377 }
00378
00379
00380 curnbad = cpl_image_count_rejected (resBpMap);
00381 xsh_msg ("End of noisy pixels, total bad pixels: %d", curnbad);
00382
00383 xsh_msg (" Nb of noisy pixels: %d", curnbad - prevnbad);
00384
00385
00386
00387
00388
00389
00390 strcpy(result_name,bpmapFile) ;
00391 xsh_msg ("save frame %s\n", result_name);
00392
00393
00394
00395
00396
00397
00398 check(set_pickup_noise_pixels_qc (medPre->data_header, curnbad, instr ));
00399
00400 check(xsh_pfits_set_qc_nhpix( medPre->data_header, curnbad - prevnbad )) ;
00401 if ( xsh_instrument_get_arm(instr) == XSH_ARM_NIR ) {
00402 sprintf(noisy_name,"%s.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr));
00403 } else {
00404 check(binx=xsh_pfits_get_binx(medPre->data_header));
00405 check(biny=xsh_pfits_get_biny(medPre->data_header));
00406 sprintf(noisy_name,"%s_%dx%d.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,
00407 instr),
00408 binx,biny);
00409 }
00410 ima_aux=cpl_image_cast(medPre->data,CPL_TYPE_FLOAT);
00411 check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
00412 medPre->data_header, CPL_IO_DEFAULT));
00413
00414 xsh_free_image(&ima_aux);
00415
00416 xsh_pfits_set_extname (medPre->errs_header, "ERRS");
00417 ima_aux=cpl_image_cast(medPre->errs,CPL_TYPE_FLOAT);
00418 check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
00419 medPre->errs_header, CPL_IO_EXTEND));
00420 xsh_free_image(&ima_aux);
00421
00422
00423 xsh_pfits_set_extname (bpmapHeader, "QUAL");
00424 check(cpl_image_save (resBpMap, result_name, XSH_PRE_DATA_BPP, bpmapHeader,
00425 CPL_IO_EXTEND));
00426
00427
00428 check(resFrame=xsh_frame_product(result_name,tag,
00429 CPL_FRAME_TYPE_IMAGE,
00430 CPL_FRAME_GROUP_PRODUCT,
00431 CPL_FRAME_LEVEL_FINAL));
00432
00433
00434
00435 set_pickup_noise_pixels_qc (noisymapHeader, curnbad, instr );
00436 xsh_pfits_set_qc_nhpix( noisymapHeader, (curnbad - prevnbad) ) ;
00437 cpl_image_save (noisyMap, noisy_name, XSH_PRE_DATA_BPP,
00438 noisymapHeader,
00439 CPL_IO_DEFAULT );
00440 xsh_add_temporary_file(noisy_name);
00441
00442 check(*noisyFrame=xsh_frame_product(noisy_name,tag,
00443 CPL_FRAME_TYPE_IMAGE,
00444 CPL_FRAME_GROUP_PRODUCT,
00445 CPL_FRAME_LEVEL_TEMPORARY));
00446
00447
00448 cleanup:
00449
00450 xsh_free_propertylist(&noisymapHeader);
00451 xsh_free_image(&noisyMap);
00452 xsh_free_image(&ima_aux);
00453 xsh_pre_free( &medPre ) ;
00454
00455 return resFrame ;
00456 }
00457
00458
00459