aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThomas White <taw@physics.org>2022-08-18 14:20:16 +0200
committerThomas White <taw@physics.org>2022-08-18 15:57:46 +0200
commiteaf9e6250654e65daabbe98b6b4d5b5f8cd6924f (patch)
treeecaa9a88d1bfe1b03d36961d58f26f1dc2ab7c1d
parente8847558c3831e309d0325382589f34b273e96c8 (diff)
Restructure image data array creation
This adds a central procedure (image_create_dp_bad_sat) to create all the arrays. Then it's up to the loading procedure to put the values into the arrays. This also makes the loading procedures responsible for marking NaN/inf pixels in the bad pixel map. This avoids an additional sweep through the image data, and makes it possible to skip the NaN/inf check altogether if the image data comes, as it often does, in format which can't represent NaN/inf anyway. Finally, it removes quite a lot of duplicated code.
-rw-r--r--libcrystfel/src/image-cbf.c68
-rw-r--r--libcrystfel/src/image-hdf5.c63
-rw-r--r--libcrystfel/src/image-msgpack.c37
-rw-r--r--libcrystfel/src/image-seedee.c23
-rw-r--r--libcrystfel/src/image.c144
-rw-r--r--libcrystfel/src/image.h6
-rw-r--r--libcrystfel/src/stream.c2
7 files changed, 154 insertions, 189 deletions
diff --git a/libcrystfel/src/image-cbf.c b/libcrystfel/src/image-cbf.c
index 51003934..ecbc3209 100644
--- a/libcrystfel/src/image-cbf.c
+++ b/libcrystfel/src/image-cbf.c
@@ -572,55 +572,47 @@ static int unpack_panels(struct image *image,
const DataTemplate *dtempl,
float *data, int data_width, int data_height)
{
- int pi;
-
- image->dp = malloc(dtempl->n_panels * sizeof(float *));
- if ( image->dp == NULL ) {
- ERROR("Failed to allocate panels.\n");
- return 1;
- }
-
- for ( pi=0; pi<dtempl->n_panels; pi++ ) {
-
- struct panel_template *p;
- int fs, ss;
- int p_w, p_h;
-
- p = &dtempl->panels[pi];
- p_w = p->orig_max_fs - p->orig_min_fs + 1;
- p_h = p->orig_max_ss - p->orig_min_ss + 1;
- image->dp[pi] = malloc(p_w*p_h*sizeof(float));
- if ( image->dp[pi] == NULL ) {
- ERROR("Failed to allocate panel\n");
- return 1;
- }
+ int pi;
+
+ for ( pi=0; pi<dtempl->n_panels; pi++ ) {
+
+ struct panel_template *p;
+ int fs, ss;
+ int p_w, p_h;
- if ( (p->orig_min_fs + p_w > data_width)
- || (p->orig_min_ss + p_h > data_height) )
- {
- ERROR("Panel %s is outside range of data in CBF file\n",
- p->name);
- return 1;
+ p = &dtempl->panels[pi];
+ p_w = p->orig_max_fs - p->orig_min_fs + 1;
+ p_h = p->orig_max_ss - p->orig_min_ss + 1;
+
+ if ( (p->orig_min_fs + p_w > data_width)
+ || (p->orig_min_ss + p_h > data_height) )
+ {
+ ERROR("Panel %s is outside range of data in CBF file\n",
+ p->name);
+ return 1;
}
for ( ss=0; ss<p_h; ss++ ) {
for ( fs=0; fs<p_w; fs++ ) {
- int idx;
- int cfs, css;
+ int idx;
+ int cfs, css;
- cfs = fs+p->orig_min_fs;
- css = ss+p->orig_min_ss;
- idx = cfs + css*data_width;
+ cfs = fs+p->orig_min_fs;
+ css = ss+p->orig_min_ss;
+ idx = cfs + css*data_width;
- image->dp[pi][fs+p_w*ss] = data[idx];
+ image->dp[pi][fs+p_w*ss] = data[idx];
+ if ( !isfinite(data[idx]) ) {
+ image->bad[pi][fs+p_w*ss] = 1;
+ }
- }
- }
+ }
+ }
- }
+ }
- return 0;
+ return 0;
}
diff --git a/libcrystfel/src/image-hdf5.c b/libcrystfel/src/image-hdf5.c
index 529420fb..d615df88 100644
--- a/libcrystfel/src/image-hdf5.c
+++ b/libcrystfel/src/image-hdf5.c
@@ -365,7 +365,7 @@ static void close_hdf5(hid_t fh)
static int load_hdf5_hyperslab(struct panel_template *p,
hid_t fh,
const char *event,
- void **pdata,
+ void *data,
hid_t el_type, size_t el_size,
int skip_placeholders_ok,
const char *path_spec)
@@ -381,7 +381,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
hid_t dataspace, memspace;
hsize_t dims[2];
char *panel_full_path;
- void *data;
int ndims;
int dim;
int *dim_vals;
@@ -505,14 +504,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
dims[1] = p->orig_max_fs - p->orig_min_fs + 1;
memspace = H5Screate_simple(2, dims, NULL);
- data = malloc(dims[0]*dims[1]*el_size);
- if ( data == NULL ) {
- ERROR("Failed to allocate panel %s\n", p->name);
- free(f_offset);
- free(f_count);
- return 1;
- }
-
profile_start("H5Dread");
r = H5Dread(dh, el_type, memspace, dataspace, H5P_DEFAULT, data);
profile_end("H5Dread");
@@ -528,7 +519,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
free(f_offset);
free(f_count);
- *pdata = data;
return 0;
}
@@ -585,19 +575,10 @@ int image_hdf5_read(struct image *image,
int i;
hid_t fh;
- image->dp = malloc(dtempl->n_panels*sizeof(float *));
- if ( image->dp == NULL ) {
- ERROR("Failed to allocate data array.\n");
- return 1;
- }
-
if ( image->ev == NULL ) {
image->ev = "//";
}
- /* Set all pointers to NULL for easier clean-up */
- for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL;
-
profile_start("open-hdf5");
fh = open_hdf5(image);
profile_end("open-hdf5");
@@ -607,9 +588,11 @@ int image_hdf5_read(struct image *image,
}
for ( i=0; i<dtempl->n_panels; i++ ) {
+ long int j;
+ struct panel_template *p = &dtempl->panels[i];
profile_start("load-hdf5-hyperslab");
- if ( load_hdf5_hyperslab(&dtempl->panels[i], fh,
- image->ev, (void *)&image->dp[i],
+ if ( load_hdf5_hyperslab(p, fh,
+ image->ev, image->dp[i],
H5T_NATIVE_FLOAT,
sizeof(float), 0,
dtempl->panels[i].data) )
@@ -620,6 +603,13 @@ int image_hdf5_read(struct image *image,
return 1;
}
profile_end("load-hdf5-hyperslab");
+ profile_start("nan-inf");
+ for ( j=0; j<PANEL_WIDTH(p)*PANEL_HEIGHT(p); j++ ) {
+ if ( !isfinite(image->dp[i][j]) ) {
+ image->bad[i][j] = 1;
+ }
+ }
+ profile_end("nan-inf");
}
close_hdf5(fh);
@@ -627,29 +617,28 @@ int image_hdf5_read(struct image *image,
}
-float *image_hdf5_read_satmap(struct panel_template *p,
- const char *filename,
- const char *event,
- const char *map_location)
+int image_hdf5_read_satmap(struct panel_template *p,
+ const char *filename,
+ const char *event,
+ const char *map_location,
+ float *map_data)
{
- float *map = NULL;
hid_t fh;
fh = open_hdf5_file(filename);
- if ( fh < 0 ) return NULL;
+ if ( fh < 0 ) return 1;
if ( load_hdf5_hyperslab(p, fh, event,
- (void *)&map, H5T_NATIVE_FLOAT,
+ map_data, H5T_NATIVE_FLOAT,
sizeof(float), 1, map_location) )
{
ERROR("Failed to load saturation map data\n");
- free(map);
- return NULL;
+ return 1;
}
close_hdf5(fh);
- return map;
+ return 0;
}
@@ -668,10 +657,16 @@ int image_hdf5_read_mask(struct panel_template *p,
p_h = p->orig_max_ss - p->orig_min_ss + 1;
fh = open_hdf5_file(filename);
- if ( fh < 0 ) return 1;
+ if ( fh < 0 ) {
+ ERROR("Failed to open mask '%s'\n", filename);
+ return 1;
+ }
+
+ mask = malloc(p_w*p_h*sizeof(int));
+ if ( mask == NULL ) return 1;
if ( load_hdf5_hyperslab(p, fh, event,
- (void *)&mask, H5T_NATIVE_INT,
+ mask, H5T_NATIVE_INT,
sizeof(int), 1, mask_location) )
{
ERROR("Failed to load mask data\n");
diff --git a/libcrystfel/src/image-msgpack.c b/libcrystfel/src/image-msgpack.c
index 11876b7c..f6d6f2dd 100644
--- a/libcrystfel/src/image-msgpack.c
+++ b/libcrystfel/src/image-msgpack.c
@@ -318,7 +318,7 @@ int image_msgpack_read_header_to_cache(struct image *image,
static int load_msgpack_data(struct panel_template *p,
msgpack_object *map_obj,
- float **pdata)
+ float *data, int *bad)
{
msgpack_object *obj;
msgpack_object *type_obj;
@@ -326,7 +326,6 @@ static int load_msgpack_data(struct panel_template *p,
msgpack_object *data_obj;
char *dtype;
int data_size_fs, data_size_ss;
- void *data = NULL;
obj = find_msgpack_kv(map_obj, p->data);
if ( obj == NULL ) {
@@ -398,45 +397,36 @@ static int load_msgpack_data(struct panel_template *p,
if ( strcmp(dtype, "<i4") == 0 ) {
int fs, ss;
- float *fdata;
int32_t *in_data = (int32_t *)data_obj->via.bin.ptr;
- fdata = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float));
- if ( fdata == NULL ) return 1;
-
for ( ss=0; ss<PANEL_HEIGHT(p); ss++ ) {
for ( fs=0; fs<PANEL_WIDTH(p); fs++ ) {
size_t idx = fs+p->orig_min_fs + (ss+p->orig_min_ss)*data_size_fs;
- fdata[fs+ss*PANEL_WIDTH(p)] = in_data[idx];
+ data[fs+ss*PANEL_WIDTH(p)] = in_data[idx];
+ /* Integer data -> no need to check NaN/inf */
}
}
- data = fdata;
-
} else if ( strcmp(dtype, "<f4") == 0 ) {
int fs, ss;
- float *fdata;
float *in_data = (float *)data_obj->via.bin.ptr;
- fdata = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float));
- if ( fdata == NULL ) return 1;
-
for ( ss=0; ss<PANEL_HEIGHT(p); ss++ ) {
for ( fs=0; fs<PANEL_WIDTH(p); fs++ ) {
size_t idx = fs+p->orig_min_fs + (ss+p->orig_min_ss)*data_size_fs;
- fdata[fs+ss*PANEL_WIDTH(p)] = in_data[idx];
+ data[fs+ss*PANEL_WIDTH(p)] = in_data[idx];
+ if ( !isfinite(in_data[idx]) ) {
+ bad[idx] = 1;
+ }
}
}
- data = fdata;
-
} else {
ERROR("Unrecognised data type '%s'\n", dtype);
}
free(dtype);
- *pdata = data;
return 0;
}
@@ -476,18 +466,9 @@ int image_msgpack_read(struct image *image,
return 1;
}
- image->dp = malloc(dtempl->n_panels*sizeof(float *));
- if ( image->dp == NULL ) {
- ERROR("Failed to allocate data array.\n");
- msgpack_unpacked_destroy(&unpacked);
- return 1;
- }
-
- /* Set all pointers to NULL for easier clean-up */
- for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL;
-
for ( i=0; i<dtempl->n_panels; i++ ) {
- if ( load_msgpack_data(&dtempl->panels[i], obj, &image->dp[i]) )
+ if ( load_msgpack_data(&dtempl->panels[i], obj,
+ image->dp[i], image->bad[i]) )
{
ERROR("Failed to load data for panel '%s'\n",
dtempl->panels[i].name);
diff --git a/libcrystfel/src/image-seedee.c b/libcrystfel/src/image-seedee.c
index 537fc9e3..60153591 100644
--- a/libcrystfel/src/image-seedee.c
+++ b/libcrystfel/src/image-seedee.c
@@ -51,10 +51,9 @@
static int load_seedee_data(struct panel_template *p,
struct SeedeeNDArray *array,
- float **pdata)
+ float *data, int *bad)
{
int data_size_fs, data_size_ss;
- float *data = NULL;
data_size_ss = array->shape[0];
data_size_fs = array->shape[1];
@@ -78,22 +77,20 @@ static int load_seedee_data(struct panel_template *p,
int fs, ss;
uint16_t *in_data = (uint16_t *)array->data;
- data = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float));
- if ( data == NULL ) return 1;
-
for ( ss=0; ss<PANEL_HEIGHT(p); ss++ ) {
for ( fs=0; fs<PANEL_WIDTH(p); fs++ ) {
size_t idx = fs+p->orig_min_fs + (ss+p->orig_min_ss)*data_size_fs;
data[fs+ss*PANEL_WIDTH(p)] = in_data[idx];
}
}
- *pdata = data;
} else {
ERROR("Unrecognised data type %c%i%c\n",
array->datatype, array->itemsize, array->byteorder);
return 1;
}
+ /* Note: when adding a new data type, don't forget that you are
+ * responsible for marking NaN/inf pixels as bad. */
return 0;
}
@@ -161,20 +158,10 @@ int image_seedee_read(struct image *image,
return 1;
}
- image->dp = malloc(dtempl->n_panels*sizeof(float *));
- if ( image->dp == NULL ) {
- ERROR("Failed to allocate data array.\n");
- free(array.data);
- free(array.shape);
- return 1;
- }
-
- /* Set all pointers to NULL for easier clean-up */
- for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL;
-
profile_start("seedee-panel");
for ( i=0; i<dtempl->n_panels; i++ ) {
- if ( load_seedee_data(&dtempl->panels[i], &array, &image->dp[i]) )
+ if ( load_seedee_data(&dtempl->panels[i], &array,
+ image->dp[i], image->bad[i]) )
{
ERROR("Failed to load data for panel '%s'\n",
dtempl->panels[i].name);
diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c
index 5d67ae2d..3593f52b 100644
--- a/libcrystfel/src/image.c
+++ b/libcrystfel/src/image.c
@@ -716,54 +716,75 @@ int image_set_zero_data(struct image *image,
{
int pi;
- image->dp = malloc(dtempl->n_panels*sizeof(float *));
- if ( image->dp == NULL ) return 1;
-
for ( pi=0; pi<dtempl->n_panels; pi++ ) {
-
struct panel_template *p;
- int p_w, p_h;
-
+ long int i;
p = &dtempl->panels[pi];
- p_w = p->orig_max_fs - p->orig_min_fs + 1;
- p_h = p->orig_max_ss - p->orig_min_ss + 1;
-
- image->dp[pi] = calloc(p_w*p_h, sizeof(float));
- if ( image->dp[pi] == NULL ) return 1;
-
+ for ( i=0; i<PANEL_WIDTH(p)*PANEL_HEIGHT(p); i++ ) {
+ image->dp[pi][i] = 0.0;
+ }
}
return 0;
}
-int image_set_zero_mask(struct image *image,
- const DataTemplate *dtempl)
+int image_create_dp_bad_sat(struct image *image,
+ const DataTemplate *dtempl)
{
- int pi;
+ int i;
+
+ image->dp = malloc(dtempl->n_panels*sizeof(float *));
+ if ( image->dp == NULL ) {
+ ERROR("Failed to allocate data array.\n");
+ return 1;
+ }
image->bad = malloc(dtempl->n_panels*sizeof(int *));
+ if ( image->bad == NULL ) {
+ ERROR("Failed to allocate bad pixel mask\n");
+ free(image->dp);
+ return 1;
+ }
+
image->sat = malloc(dtempl->n_panels*sizeof(float *));
- if ( (image->bad == NULL) || (image->sat == NULL) ) return 1;
+ if ( image->sat == NULL ) {
+ ERROR("Failed to allocate saturation map\n");
+ free(image->dp);
+ free(image->bad);
+ return 1;
+ }
- for ( pi=0; pi<dtempl->n_panels; pi++ ) {
+ /* Set all pointers to NULL for easier clean-up */
+ for ( i=0; i<dtempl->n_panels; i++ ) {
+ image->dp[i] = NULL;
+ image->bad[i] = NULL;
+ image->sat[i] = NULL;
+ }
- struct panel_template *p;
- int p_w, p_h;
- long int i;
+ for ( i=0; i<dtempl->n_panels; i++ ) {
- p = &dtempl->panels[pi];
- p_w = p->orig_max_fs - p->orig_min_fs + 1;
- p_h = p->orig_max_ss - p->orig_min_ss + 1;
+ size_t nel = PANEL_WIDTH(&dtempl->panels[i]) * PANEL_HEIGHT(&dtempl->panels[i]);
- image->bad[pi] = calloc(p_w*p_h, sizeof(int));
- image->sat[pi] = calloc(p_w*p_h, sizeof(float));
- if ( image->bad[pi] == NULL ) return 1;
- if ( image->sat[pi] == NULL ) return 1;
+ image->dp[i] = malloc(nel*sizeof(float));
+ image->bad[i] = calloc(nel, sizeof(int));
+ image->sat[i] = malloc(nel*sizeof(float));
- for ( i=0; i<p_w*p_h; i++ ) {
- image->sat[pi][i] = INFINITY;
+ if ( (image->dp[i] == NULL)
+ || (image->bad[i] == NULL)
+ || (image->sat[i] == NULL) ) {
+ ERROR("Failed to allocate panel data arrays\n");
+ for ( i=0; i<dtempl->n_panels; i++ ) {
+ free(image->dp[i]);
+ free(image->bad[i]);
+ free(image->sat[i]);
+ }
+ free(image->dp);
+ free(image->bad);
+ free(image->sat);
+ return 1;
}
+
}
return 0;
@@ -882,16 +903,6 @@ static void mark_flagged_pixels_equal(float *dp, int *bad,
}
-static void mark_flagged_pixels_naninf(float *dp, int *bad,
- long int n)
-{
- long int i;
- for ( i=0; i<n; i++ ) {
- if ( !isfinite(dp[i]) ) bad[i] = 1;
- }
-}
-
-
static void mark_flagged_pixels(struct panel_template *p,
float *dp, int *bad)
{
@@ -903,10 +914,6 @@ static void mark_flagged_pixels(struct panel_template *p,
p_h = p->orig_max_ss - p->orig_min_ss + 1;
n = p_w * p_h;
- profile_start("nan-inf");
- mark_flagged_pixels_naninf(dp, bad, n);
- profile_end("nan-inf");
-
profile_start("flag-values");
for ( i=0; i<MAX_FLAG_VALUES; i++ ) {
@@ -1033,17 +1040,17 @@ static int load_mask(struct panel_template *p,
{
if ( is_hdf5_file(mask_fn) ) {
#ifdef HAVE_HDF5
- image_hdf5_read_mask(p, mask_fn, ev, bad, mask_location,
- mask_good, mask_bad);
+ return image_hdf5_read_mask(p, mask_fn, ev, bad, mask_location,
+ mask_good, mask_bad);
#endif
} else if ( is_cbf_file(mask_fn) ) {
- image_cbf_read_mask(p, mask_fn, ev, 0, bad,
- mask_good, mask_bad);
+ return image_cbf_read_mask(p, mask_fn, ev, 0, bad,
+ mask_good, mask_bad);
} else if ( is_cbfgz_file(mask_fn) ) {
- image_cbf_read_mask(p, mask_fn, ev, 1, bad,
- mask_good, mask_bad);
+ return image_cbf_read_mask(p, mask_fn, ev, 1, bad,
+ mask_good, mask_bad);
} else {
ERROR("Unrecognised mask file type (%s)\n", mask_fn);
@@ -1083,11 +1090,9 @@ static int create_badmap(struct image *image,
{
int i;
- image->bad = malloc(dtempl->n_panels * sizeof(int *));
- if ( image->bad == NULL ) {
- ERROR("Failed to allocate bad pixel mask\n");
- return 1;
- }
+ /* The bad pixel map array is already created (see image_create_dp_bad_sat),
+ * and a preliminary mask (with NaN/inf pixels marked) has already been
+ * created when the image data was loaded. */
for ( i=0; i<dtempl->n_panels; i++ ) {
@@ -1097,12 +1102,6 @@ static int create_badmap(struct image *image,
p_w = p->orig_max_fs - p->orig_min_fs + 1;
p_h = p->orig_max_ss - p->orig_min_ss + 1;
- image->bad[i] = calloc(p_w*p_h, sizeof(int));
- if ( image->bad[i] == NULL ) {
- ERROR("Failed to allocate bad pixel mask\n");
- return 1;
- }
-
/* Panel marked as bad? */
if ( p->bad ) {
profile_start("whole-panel");
@@ -1148,10 +1147,16 @@ static int create_badmap(struct image *image,
mask_fn = p->masks[j].filename;
}
- load_mask(p, mask_fn, image->ev, image->bad[i],
- p->masks[j].data_location,
- p->masks[j].good_bits,
- p->masks[j].bad_bits);
+ if ( load_mask(p, mask_fn, image->ev, image->bad[i],
+ p->masks[j].data_location,
+ p->masks[j].good_bits,
+ p->masks[j].bad_bits) )
+ {
+ ERROR("Failed to load mask for %s\n",
+ p->name);
+ profile_end("load-masks");
+ return 1;
+ }
}
profile_end("load-masks");
@@ -1226,9 +1231,7 @@ static int create_satmap(struct image *image,
if ( is_hdf5_file(map_fn) ) {
#ifdef HAVE_HDF5
- image->sat[i] = image_hdf5_read_satmap(p, map_fn,
- image->ev,
- p->satmap);
+ image_hdf5_read_satmap(p, map_fn, image->ev, p->satmap);
#endif
} else {
@@ -1276,6 +1279,11 @@ struct image *image_create_for_simulation(const DataTemplate *dtempl)
return NULL;
}
+ if ( image_create_dp_bad_sat(image, dtempl) ) {
+ image_free(image);
+ return NULL;
+ }
+
if ( image_set_zero_data(image, dtempl) ) {
image_free(image);
return NULL;
@@ -1311,6 +1319,8 @@ static int do_image_read(struct image *image, const DataTemplate *dtempl,
int i;
int r;
+ if ( image_create_dp_bad_sat(image, dtempl) ) return 1;
+
/* Load the image data */
if ( !no_image_data ) {
int r;
diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h
index 654bf3b1..3f1ed219 100644
--- a/libcrystfel/src/image.h
+++ b/libcrystfel/src/image.h
@@ -260,10 +260,10 @@ extern ImageFeatureList *image_read_peaks(const DataTemplate *dtempl,
extern char **image_expand_frames(const DataTemplate *dtempl,
const char *filename, int *nframes);
-extern int image_set_zero_data(struct image *image,
- const DataTemplate *dtempl);
+extern int image_create_dp_bad_sat(struct image *image,
+ const DataTemplate *dtempl);
-extern int image_set_zero_mask(struct image *image,
+extern int image_set_zero_data(struct image *image,
const DataTemplate *dtempl);
extern int image_write(const struct image *image,
diff --git a/libcrystfel/src/stream.c b/libcrystfel/src/stream.c
index 67297e49..708e681c 100644
--- a/libcrystfel/src/stream.c
+++ b/libcrystfel/src/stream.c
@@ -900,8 +900,8 @@ struct image *stream_read_chunk(Stream *st, StreamFlags srf)
image_free(image);
return NULL;
}
+ image_create_dp_bad_sat(image, st->dtempl_read);
image_set_zero_data(image, st->dtempl_read);
- image_set_zero_mask(image, st->dtempl_read);
}
image->spectrum = spectrum_generate_gaussian(image->lambda,
image->bw);