diff options
author | Thomas White <taw@physics.org> | 2022-08-18 14:20:16 +0200 |
---|---|---|
committer | Thomas White <taw@physics.org> | 2022-08-18 15:57:46 +0200 |
commit | eaf9e6250654e65daabbe98b6b4d5b5f8cd6924f (patch) | |
tree | ecaa9a88d1bfe1b03d36961d58f26f1dc2ab7c1d /libcrystfel | |
parent | e8847558c3831e309d0325382589f34b273e96c8 (diff) |
Restructure image data array creation
This adds a central procedure (image_create_dp_bad_sat) to create all
the arrays. Then it's up to the loading procedure to put the values
into the arrays.
This also makes the loading procedures responsible for marking NaN/inf
pixels in the bad pixel map. This avoids an additional sweep through
the image data, and makes it possible to skip the NaN/inf check
altogether if the image data comes, as it often does, in format which
can't represent NaN/inf anyway.
Finally, it removes quite a lot of duplicated code.
Diffstat (limited to 'libcrystfel')
-rw-r--r-- | libcrystfel/src/image-cbf.c | 68 | ||||
-rw-r--r-- | libcrystfel/src/image-hdf5.c | 63 | ||||
-rw-r--r-- | libcrystfel/src/image-msgpack.c | 37 | ||||
-rw-r--r-- | libcrystfel/src/image-seedee.c | 23 | ||||
-rw-r--r-- | libcrystfel/src/image.c | 144 | ||||
-rw-r--r-- | libcrystfel/src/image.h | 6 | ||||
-rw-r--r-- | libcrystfel/src/stream.c | 2 |
7 files changed, 154 insertions, 189 deletions
diff --git a/libcrystfel/src/image-cbf.c b/libcrystfel/src/image-cbf.c index 51003934..ecbc3209 100644 --- a/libcrystfel/src/image-cbf.c +++ b/libcrystfel/src/image-cbf.c @@ -572,55 +572,47 @@ static int unpack_panels(struct image *image, const DataTemplate *dtempl, float *data, int data_width, int data_height) { - int pi; - - image->dp = malloc(dtempl->n_panels * sizeof(float *)); - if ( image->dp == NULL ) { - ERROR("Failed to allocate panels.\n"); - return 1; - } - - for ( pi=0; pi<dtempl->n_panels; pi++ ) { - - struct panel_template *p; - int fs, ss; - int p_w, p_h; - - p = &dtempl->panels[pi]; - p_w = p->orig_max_fs - p->orig_min_fs + 1; - p_h = p->orig_max_ss - p->orig_min_ss + 1; - image->dp[pi] = malloc(p_w*p_h*sizeof(float)); - if ( image->dp[pi] == NULL ) { - ERROR("Failed to allocate panel\n"); - return 1; - } + int pi; + + for ( pi=0; pi<dtempl->n_panels; pi++ ) { + + struct panel_template *p; + int fs, ss; + int p_w, p_h; - if ( (p->orig_min_fs + p_w > data_width) - || (p->orig_min_ss + p_h > data_height) ) - { - ERROR("Panel %s is outside range of data in CBF file\n", - p->name); - return 1; + p = &dtempl->panels[pi]; + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; + + if ( (p->orig_min_fs + p_w > data_width) + || (p->orig_min_ss + p_h > data_height) ) + { + ERROR("Panel %s is outside range of data in CBF file\n", + p->name); + return 1; } for ( ss=0; ss<p_h; ss++ ) { for ( fs=0; fs<p_w; fs++ ) { - int idx; - int cfs, css; + int idx; + int cfs, css; - cfs = fs+p->orig_min_fs; - css = ss+p->orig_min_ss; - idx = cfs + css*data_width; + cfs = fs+p->orig_min_fs; + css = ss+p->orig_min_ss; + idx = cfs + css*data_width; - image->dp[pi][fs+p_w*ss] = data[idx]; + image->dp[pi][fs+p_w*ss] = data[idx]; + if ( !isfinite(data[idx]) ) { + image->bad[pi][fs+p_w*ss] = 1; + } - } - } + } + } - } + } - return 0; + return 0; } diff --git a/libcrystfel/src/image-hdf5.c b/libcrystfel/src/image-hdf5.c index 529420fb..d615df88 100644 --- a/libcrystfel/src/image-hdf5.c +++ b/libcrystfel/src/image-hdf5.c @@ -365,7 +365,7 @@ static void close_hdf5(hid_t fh) static int load_hdf5_hyperslab(struct panel_template *p, hid_t fh, const char *event, - void **pdata, + void *data, hid_t el_type, size_t el_size, int skip_placeholders_ok, const char *path_spec) @@ -381,7 +381,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, hid_t dataspace, memspace; hsize_t dims[2]; char *panel_full_path; - void *data; int ndims; int dim; int *dim_vals; @@ -505,14 +504,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, dims[1] = p->orig_max_fs - p->orig_min_fs + 1; memspace = H5Screate_simple(2, dims, NULL); - data = malloc(dims[0]*dims[1]*el_size); - if ( data == NULL ) { - ERROR("Failed to allocate panel %s\n", p->name); - free(f_offset); - free(f_count); - return 1; - } - profile_start("H5Dread"); r = H5Dread(dh, el_type, memspace, dataspace, H5P_DEFAULT, data); profile_end("H5Dread"); @@ -528,7 +519,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, free(f_offset); free(f_count); - *pdata = data; return 0; } @@ -585,19 +575,10 @@ int image_hdf5_read(struct image *image, int i; hid_t fh; - image->dp = malloc(dtempl->n_panels*sizeof(float *)); - if ( image->dp == NULL ) { - ERROR("Failed to allocate data array.\n"); - return 1; - } - if ( image->ev == NULL ) { image->ev = "//"; } - /* Set all pointers to NULL for easier clean-up */ - for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL; - profile_start("open-hdf5"); fh = open_hdf5(image); profile_end("open-hdf5"); @@ -607,9 +588,11 @@ int image_hdf5_read(struct image *image, } for ( i=0; i<dtempl->n_panels; i++ ) { + long int j; + struct panel_template *p = &dtempl->panels[i]; profile_start("load-hdf5-hyperslab"); - if ( load_hdf5_hyperslab(&dtempl->panels[i], fh, - image->ev, (void *)&image->dp[i], + if ( load_hdf5_hyperslab(p, fh, + image->ev, image->dp[i], H5T_NATIVE_FLOAT, sizeof(float), 0, dtempl->panels[i].data) ) @@ -620,6 +603,13 @@ int image_hdf5_read(struct image *image, return 1; } profile_end("load-hdf5-hyperslab"); + profile_start("nan-inf"); + for ( j=0; j<PANEL_WIDTH(p)*PANEL_HEIGHT(p); j++ ) { + if ( !isfinite(image->dp[i][j]) ) { + image->bad[i][j] = 1; + } + } + profile_end("nan-inf"); } close_hdf5(fh); @@ -627,29 +617,28 @@ int image_hdf5_read(struct image *image, } -float *image_hdf5_read_satmap(struct panel_template *p, - const char *filename, - const char *event, - const char *map_location) +int image_hdf5_read_satmap(struct panel_template *p, + const char *filename, + const char *event, + const char *map_location, + float *map_data) { - float *map = NULL; hid_t fh; fh = open_hdf5_file(filename); - if ( fh < 0 ) return NULL; + if ( fh < 0 ) return 1; if ( load_hdf5_hyperslab(p, fh, event, - (void *)&map, H5T_NATIVE_FLOAT, + map_data, H5T_NATIVE_FLOAT, sizeof(float), 1, map_location) ) { ERROR("Failed to load saturation map data\n"); - free(map); - return NULL; + return 1; } close_hdf5(fh); - return map; + return 0; } @@ -668,10 +657,16 @@ int image_hdf5_read_mask(struct panel_template *p, p_h = p->orig_max_ss - p->orig_min_ss + 1; fh = open_hdf5_file(filename); - if ( fh < 0 ) return 1; + if ( fh < 0 ) { + ERROR("Failed to open mask '%s'\n", filename); + return 1; + } + + mask = malloc(p_w*p_h*sizeof(int)); + if ( mask == NULL ) return 1; if ( load_hdf5_hyperslab(p, fh, event, - (void *)&mask, H5T_NATIVE_INT, + mask, H5T_NATIVE_INT, sizeof(int), 1, mask_location) ) { ERROR("Failed to load mask data\n"); diff --git a/libcrystfel/src/image-msgpack.c b/libcrystfel/src/image-msgpack.c index 11876b7c..f6d6f2dd 100644 --- a/libcrystfel/src/image-msgpack.c +++ b/libcrystfel/src/image-msgpack.c @@ -318,7 +318,7 @@ int image_msgpack_read_header_to_cache(struct image *image, static int load_msgpack_data(struct panel_template *p, msgpack_object *map_obj, - float **pdata) + float *data, int *bad) { msgpack_object *obj; msgpack_object *type_obj; @@ -326,7 +326,6 @@ static int load_msgpack_data(struct panel_template *p, msgpack_object *data_obj; char *dtype; int data_size_fs, data_size_ss; - void *data = NULL; obj = find_msgpack_kv(map_obj, p->data); if ( obj == NULL ) { @@ -398,45 +397,36 @@ static int load_msgpack_data(struct panel_template *p, if ( strcmp(dtype, "<i4") == 0 ) { int fs, ss; - float *fdata; int32_t *in_data = (int32_t *)data_obj->via.bin.ptr; - fdata = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float)); - if ( fdata == NULL ) return 1; - for ( ss=0; ss<PANEL_HEIGHT(p); ss++ ) { for ( fs=0; fs<PANEL_WIDTH(p); fs++ ) { size_t idx = fs+p->orig_min_fs + (ss+p->orig_min_ss)*data_size_fs; - fdata[fs+ss*PANEL_WIDTH(p)] = in_data[idx]; + data[fs+ss*PANEL_WIDTH(p)] = in_data[idx]; + /* Integer data -> no need to check NaN/inf */ } } - data = fdata; - } else if ( strcmp(dtype, "<f4") == 0 ) { int fs, ss; - float *fdata; float *in_data = (float *)data_obj->via.bin.ptr; - fdata = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float)); - if ( fdata == NULL ) return 1; - for ( ss=0; ss<PANEL_HEIGHT(p); ss++ ) { for ( fs=0; fs<PANEL_WIDTH(p); fs++ ) { size_t idx = fs+p->orig_min_fs + (ss+p->orig_min_ss)*data_size_fs; - fdata[fs+ss*PANEL_WIDTH(p)] = in_data[idx]; + data[fs+ss*PANEL_WIDTH(p)] = in_data[idx]; + if ( !isfinite(in_data[idx]) ) { + bad[idx] = 1; + } } } - data = fdata; - } else { ERROR("Unrecognised data type '%s'\n", dtype); } free(dtype); - *pdata = data; return 0; } @@ -476,18 +466,9 @@ int image_msgpack_read(struct image *image, return 1; } - image->dp = malloc(dtempl->n_panels*sizeof(float *)); - if ( image->dp == NULL ) { - ERROR("Failed to allocate data array.\n"); - msgpack_unpacked_destroy(&unpacked); - return 1; - } - - /* Set all pointers to NULL for easier clean-up */ - for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL; - for ( i=0; i<dtempl->n_panels; i++ ) { - if ( load_msgpack_data(&dtempl->panels[i], obj, &image->dp[i]) ) + if ( load_msgpack_data(&dtempl->panels[i], obj, + image->dp[i], image->bad[i]) ) { ERROR("Failed to load data for panel '%s'\n", dtempl->panels[i].name); diff --git a/libcrystfel/src/image-seedee.c b/libcrystfel/src/image-seedee.c index 537fc9e3..60153591 100644 --- a/libcrystfel/src/image-seedee.c +++ b/libcrystfel/src/image-seedee.c @@ -51,10 +51,9 @@ static int load_seedee_data(struct panel_template *p, struct SeedeeNDArray *array, - float **pdata) + float *data, int *bad) { int data_size_fs, data_size_ss; - float *data = NULL; data_size_ss = array->shape[0]; data_size_fs = array->shape[1]; @@ -78,22 +77,20 @@ static int load_seedee_data(struct panel_template *p, int fs, ss; uint16_t *in_data = (uint16_t *)array->data; - data = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float)); - if ( data == NULL ) return 1; - for ( ss=0; ss<PANEL_HEIGHT(p); ss++ ) { for ( fs=0; fs<PANEL_WIDTH(p); fs++ ) { size_t idx = fs+p->orig_min_fs + (ss+p->orig_min_ss)*data_size_fs; data[fs+ss*PANEL_WIDTH(p)] = in_data[idx]; } } - *pdata = data; } else { ERROR("Unrecognised data type %c%i%c\n", array->datatype, array->itemsize, array->byteorder); return 1; } + /* Note: when adding a new data type, don't forget that you are + * responsible for marking NaN/inf pixels as bad. */ return 0; } @@ -161,20 +158,10 @@ int image_seedee_read(struct image *image, return 1; } - image->dp = malloc(dtempl->n_panels*sizeof(float *)); - if ( image->dp == NULL ) { - ERROR("Failed to allocate data array.\n"); - free(array.data); - free(array.shape); - return 1; - } - - /* Set all pointers to NULL for easier clean-up */ - for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL; - profile_start("seedee-panel"); for ( i=0; i<dtempl->n_panels; i++ ) { - if ( load_seedee_data(&dtempl->panels[i], &array, &image->dp[i]) ) + if ( load_seedee_data(&dtempl->panels[i], &array, + image->dp[i], image->bad[i]) ) { ERROR("Failed to load data for panel '%s'\n", dtempl->panels[i].name); diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c index 5d67ae2d..3593f52b 100644 --- a/libcrystfel/src/image.c +++ b/libcrystfel/src/image.c @@ -716,54 +716,75 @@ int image_set_zero_data(struct image *image, { int pi; - image->dp = malloc(dtempl->n_panels*sizeof(float *)); - if ( image->dp == NULL ) return 1; - for ( pi=0; pi<dtempl->n_panels; pi++ ) { - struct panel_template *p; - int p_w, p_h; - + long int i; p = &dtempl->panels[pi]; - p_w = p->orig_max_fs - p->orig_min_fs + 1; - p_h = p->orig_max_ss - p->orig_min_ss + 1; - - image->dp[pi] = calloc(p_w*p_h, sizeof(float)); - if ( image->dp[pi] == NULL ) return 1; - + for ( i=0; i<PANEL_WIDTH(p)*PANEL_HEIGHT(p); i++ ) { + image->dp[pi][i] = 0.0; + } } return 0; } -int image_set_zero_mask(struct image *image, - const DataTemplate *dtempl) +int image_create_dp_bad_sat(struct image *image, + const DataTemplate *dtempl) { - int pi; + int i; + + image->dp = malloc(dtempl->n_panels*sizeof(float *)); + if ( image->dp == NULL ) { + ERROR("Failed to allocate data array.\n"); + return 1; + } image->bad = malloc(dtempl->n_panels*sizeof(int *)); + if ( image->bad == NULL ) { + ERROR("Failed to allocate bad pixel mask\n"); + free(image->dp); + return 1; + } + image->sat = malloc(dtempl->n_panels*sizeof(float *)); - if ( (image->bad == NULL) || (image->sat == NULL) ) return 1; + if ( image->sat == NULL ) { + ERROR("Failed to allocate saturation map\n"); + free(image->dp); + free(image->bad); + return 1; + } - for ( pi=0; pi<dtempl->n_panels; pi++ ) { + /* Set all pointers to NULL for easier clean-up */ + for ( i=0; i<dtempl->n_panels; i++ ) { + image->dp[i] = NULL; + image->bad[i] = NULL; + image->sat[i] = NULL; + } - struct panel_template *p; - int p_w, p_h; - long int i; + for ( i=0; i<dtempl->n_panels; i++ ) { - p = &dtempl->panels[pi]; - p_w = p->orig_max_fs - p->orig_min_fs + 1; - p_h = p->orig_max_ss - p->orig_min_ss + 1; + size_t nel = PANEL_WIDTH(&dtempl->panels[i]) * PANEL_HEIGHT(&dtempl->panels[i]); - image->bad[pi] = calloc(p_w*p_h, sizeof(int)); - image->sat[pi] = calloc(p_w*p_h, sizeof(float)); - if ( image->bad[pi] == NULL ) return 1; - if ( image->sat[pi] == NULL ) return 1; + image->dp[i] = malloc(nel*sizeof(float)); + image->bad[i] = calloc(nel, sizeof(int)); + image->sat[i] = malloc(nel*sizeof(float)); - for ( i=0; i<p_w*p_h; i++ ) { - image->sat[pi][i] = INFINITY; + if ( (image->dp[i] == NULL) + || (image->bad[i] == NULL) + || (image->sat[i] == NULL) ) { + ERROR("Failed to allocate panel data arrays\n"); + for ( i=0; i<dtempl->n_panels; i++ ) { + free(image->dp[i]); + free(image->bad[i]); + free(image->sat[i]); + } + free(image->dp); + free(image->bad); + free(image->sat); + return 1; } + } return 0; @@ -882,16 +903,6 @@ static void mark_flagged_pixels_equal(float *dp, int *bad, } -static void mark_flagged_pixels_naninf(float *dp, int *bad, - long int n) -{ - long int i; - for ( i=0; i<n; i++ ) { - if ( !isfinite(dp[i]) ) bad[i] = 1; - } -} - - static void mark_flagged_pixels(struct panel_template *p, float *dp, int *bad) { @@ -903,10 +914,6 @@ static void mark_flagged_pixels(struct panel_template *p, p_h = p->orig_max_ss - p->orig_min_ss + 1; n = p_w * p_h; - profile_start("nan-inf"); - mark_flagged_pixels_naninf(dp, bad, n); - profile_end("nan-inf"); - profile_start("flag-values"); for ( i=0; i<MAX_FLAG_VALUES; i++ ) { @@ -1033,17 +1040,17 @@ static int load_mask(struct panel_template *p, { if ( is_hdf5_file(mask_fn) ) { #ifdef HAVE_HDF5 - image_hdf5_read_mask(p, mask_fn, ev, bad, mask_location, - mask_good, mask_bad); + return image_hdf5_read_mask(p, mask_fn, ev, bad, mask_location, + mask_good, mask_bad); #endif } else if ( is_cbf_file(mask_fn) ) { - image_cbf_read_mask(p, mask_fn, ev, 0, bad, - mask_good, mask_bad); + return image_cbf_read_mask(p, mask_fn, ev, 0, bad, + mask_good, mask_bad); } else if ( is_cbfgz_file(mask_fn) ) { - image_cbf_read_mask(p, mask_fn, ev, 1, bad, - mask_good, mask_bad); + return image_cbf_read_mask(p, mask_fn, ev, 1, bad, + mask_good, mask_bad); } else { ERROR("Unrecognised mask file type (%s)\n", mask_fn); @@ -1083,11 +1090,9 @@ static int create_badmap(struct image *image, { int i; - image->bad = malloc(dtempl->n_panels * sizeof(int *)); - if ( image->bad == NULL ) { - ERROR("Failed to allocate bad pixel mask\n"); - return 1; - } + /* The bad pixel map array is already created (see image_create_dp_bad_sat), + * and a preliminary mask (with NaN/inf pixels marked) has already been + * created when the image data was loaded. */ for ( i=0; i<dtempl->n_panels; i++ ) { @@ -1097,12 +1102,6 @@ static int create_badmap(struct image *image, p_w = p->orig_max_fs - p->orig_min_fs + 1; p_h = p->orig_max_ss - p->orig_min_ss + 1; - image->bad[i] = calloc(p_w*p_h, sizeof(int)); - if ( image->bad[i] == NULL ) { - ERROR("Failed to allocate bad pixel mask\n"); - return 1; - } - /* Panel marked as bad? */ if ( p->bad ) { profile_start("whole-panel"); @@ -1148,10 +1147,16 @@ static int create_badmap(struct image *image, mask_fn = p->masks[j].filename; } - load_mask(p, mask_fn, image->ev, image->bad[i], - p->masks[j].data_location, - p->masks[j].good_bits, - p->masks[j].bad_bits); + if ( load_mask(p, mask_fn, image->ev, image->bad[i], + p->masks[j].data_location, + p->masks[j].good_bits, + p->masks[j].bad_bits) ) + { + ERROR("Failed to load mask for %s\n", + p->name); + profile_end("load-masks"); + return 1; + } } profile_end("load-masks"); @@ -1226,9 +1231,7 @@ static int create_satmap(struct image *image, if ( is_hdf5_file(map_fn) ) { #ifdef HAVE_HDF5 - image->sat[i] = image_hdf5_read_satmap(p, map_fn, - image->ev, - p->satmap); + image_hdf5_read_satmap(p, map_fn, image->ev, p->satmap); #endif } else { @@ -1276,6 +1279,11 @@ struct image *image_create_for_simulation(const DataTemplate *dtempl) return NULL; } + if ( image_create_dp_bad_sat(image, dtempl) ) { + image_free(image); + return NULL; + } + if ( image_set_zero_data(image, dtempl) ) { image_free(image); return NULL; @@ -1311,6 +1319,8 @@ static int do_image_read(struct image *image, const DataTemplate *dtempl, int i; int r; + if ( image_create_dp_bad_sat(image, dtempl) ) return 1; + /* Load the image data */ if ( !no_image_data ) { int r; diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h index 654bf3b1..3f1ed219 100644 --- a/libcrystfel/src/image.h +++ b/libcrystfel/src/image.h @@ -260,10 +260,10 @@ extern ImageFeatureList *image_read_peaks(const DataTemplate *dtempl, extern char **image_expand_frames(const DataTemplate *dtempl, const char *filename, int *nframes); -extern int image_set_zero_data(struct image *image, - const DataTemplate *dtempl); +extern int image_create_dp_bad_sat(struct image *image, + const DataTemplate *dtempl); -extern int image_set_zero_mask(struct image *image, +extern int image_set_zero_data(struct image *image, const DataTemplate *dtempl); extern int image_write(const struct image *image, diff --git a/libcrystfel/src/stream.c b/libcrystfel/src/stream.c index 67297e49..708e681c 100644 --- a/libcrystfel/src/stream.c +++ b/libcrystfel/src/stream.c @@ -900,8 +900,8 @@ struct image *stream_read_chunk(Stream *st, StreamFlags srf) image_free(image); return NULL; } + image_create_dp_bad_sat(image, st->dtempl_read); image_set_zero_data(image, st->dtempl_read); - image_set_zero_mask(image, st->dtempl_read); } image->spectrum = spectrum_generate_gaussian(image->lambda, image->bw); |