aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThomas White <taw@physics.org>2021-09-14 12:30:37 +0200
committerThomas White <taw@physics.org>2021-09-17 11:09:02 +0200
commit3d55c5c01aaeedfdcb16ee079080b26a3e8c6830 (patch)
tree43b5d43d87b6ef1d95413b123c14db242857e99b
parent6482185648e5f9d202a40f1bada7972454f7283b (diff)
Handle in-memory HDF5s
-rw-r--r--CMakeLists.txt2
-rw-r--r--libcrystfel/CMakeLists.txt4
-rw-r--r--libcrystfel/src/image-hdf5.c102
3 files changed, 77 insertions, 31 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78bcce10..1f69bed2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -11,7 +11,7 @@ include(GNUInstallDirs)
# Set CMAKE_MODULE_PATH to assist cmake in finding our own package definitions
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")
-find_package(HDF5 REQUIRED COMPONENTS C)
+find_package(HDF5 REQUIRED COMPONENTS C HL)
find_package(Threads REQUIRED)
find_package(GSL REQUIRED)
find_package(OpenCL)
diff --git a/libcrystfel/CMakeLists.txt b/libcrystfel/CMakeLists.txt
index 1c8ef887..77290677 100644
--- a/libcrystfel/CMakeLists.txt
+++ b/libcrystfel/CMakeLists.txt
@@ -138,8 +138,8 @@ target_include_directories(${PROJECT_NAME} INTERFACE ${PROJECT_SOURCE_DIR}/src)
include_directories(${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(${PROJECT_NAME} PRIVATE ${HDF5_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
-target_link_libraries(${PROJECT_NAME} PRIVATE util ${HDF5_C_LIBRARIES} ${ZLIB_LIBRARIES}
- Threads::Threads GSL::gsl m)
+target_link_libraries(${PROJECT_NAME} PRIVATE util ${HDF5_C_LIBRARIES} ${HDF5_C_HL_LIBRARIES}
+ ${ZLIB_LIBRARIES} Threads::Threads GSL::gsl m)
if (XGANDALF_FOUND)
target_include_directories(${PROJECT_NAME} PRIVATE ${XGANDALF_INCLUDE_DIRS})
diff --git a/libcrystfel/src/image-hdf5.c b/libcrystfel/src/image-hdf5.c
index 7898d06b..ab8d131d 100644
--- a/libcrystfel/src/image-hdf5.c
+++ b/libcrystfel/src/image-hdf5.c
@@ -34,6 +34,7 @@
#include <math.h>
#include <stdio.h>
#include <hdf5.h>
+#include <hdf5_hl.h>
#include <unistd.h>
#include "image.h"
@@ -355,14 +356,13 @@ static int total_dimensions(const struct panel_template *p)
static int load_hdf5_hyperslab(struct panel_template *p,
- const char *filename,
+ hid_t fh,
const char *event,
void **pdata,
hid_t el_type, size_t el_size,
int skip_placeholders_ok,
const char *path_spec)
{
- hid_t fh;
int total_dt_dims;
int plh_dt_dims;
int dt_dims[MAX_DIMS];
@@ -381,33 +381,19 @@ static int load_hdf5_hyperslab(struct panel_template *p,
int n_dim_vals;
int pl_pos;
- if ( access(filename, R_OK) == -1 ) {
- ERROR("File does not exist or cannot be read: %s\n",
- filename);
- return 1;
- }
-
- fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);
- if ( fh < 0 ) {
- ERROR("Couldn't open file (hyperslab): %s\n", filename);
- return 1;
- }
-
panel_full_path = substitute_path(event, path_spec,
skip_placeholders_ok);
if ( panel_full_path == NULL ) {
ERROR("Invalid path substitution: '%s' '%s'\n",
event, path_spec);
- close_hdf5(fh);
return 1;
}
dh = H5Dopen2(fh, panel_full_path, H5P_DEFAULT);
if ( dh < 0 ) {
- ERROR("Cannot open data for panel %s (%s) in file %s\n",
- p->name, panel_full_path, filename);
+ ERROR("Cannot open data for panel %s (%s)\n",
+ p->name, panel_full_path);
free(panel_full_path);
- close_hdf5(fh);
return 1;
}
@@ -420,7 +406,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
if ( ndims < 0 ) {
ERROR("Failed to get number of dimensions for panel %s\n",
p->name);
- close_hdf5(fh);
return 1;
}
@@ -441,7 +426,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
"panel %s (%i, but expected %i or %i)\n",
p->name, ndims, total_dt_dims,
total_dt_dims - plh_dt_dims);
- close_hdf5(fh);
return 1;
}
} else {
@@ -456,7 +440,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
f_count = malloc(ndims*sizeof(hsize_t));
if ( (f_offset == NULL) || (f_count == NULL ) ) {
ERROR("Failed to allocate offset or count.\n");
- close_hdf5(fh);
return 1;
}
@@ -505,7 +488,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
p->name);
free(f_offset);
free(f_count);
- close_hdf5(fh);
return 1;
}
@@ -518,7 +500,6 @@ static int load_hdf5_hyperslab(struct panel_template *p,
ERROR("Failed to allocate panel %s\n", p->name);
free(f_offset);
free(f_count);
- close_hdf5(fh);
return 1;
}
@@ -529,23 +510,68 @@ static int load_hdf5_hyperslab(struct panel_template *p,
free(f_offset);
free(f_count);
free(data);
- close_hdf5(fh);
return 1;
}
free(f_offset);
free(f_count);
- close_hdf5(fh);
*pdata = data;
return 0;
}
+static hid_t open_hdf5_file(const char *filename)
+{
+ hid_t fh;
+
+ if ( access(filename, R_OK) == -1 ) {
+ ERROR("File does not exist or cannot be read: %s\n",
+ filename);
+ return -1;
+ }
+
+ fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);
+ if ( fh < 0 ) {
+ ERROR("Couldn't open HDF5 file: %s\n", filename);
+ return -1;
+ }
+
+ return fh;
+}
+
+
+static hid_t open_hdf5(struct image *image)
+{
+ if ( image->data_block == NULL ) {
+
+ return open_hdf5_file(image->filename);
+
+ } else {
+
+ hid_t fh;
+
+ fh = H5LTopen_file_image(image->data_block,
+ image->data_block_size,
+ H5LT_FILE_IMAGE_DONT_COPY
+ | H5LT_FILE_IMAGE_DONT_RELEASE);
+
+ if ( fh < 0 ) {
+ ERROR("Couldn't open HDF5 image (%p %lli)\n",
+ image->data_block, image->data_block_size);
+ return -1;
+ }
+
+ return fh;
+ }
+}
+
+
int image_hdf5_read(struct image *image,
const DataTemplate *dtempl)
{
int i;
+ hid_t fh;
image->dp = malloc(dtempl->n_panels*sizeof(float *));
if ( image->dp == NULL ) {
@@ -560,18 +586,26 @@ int image_hdf5_read(struct image *image,
/* Set all pointers to NULL for easier clean-up */
for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL;
+ fh = open_hdf5(image);
+ if ( fh < 0 ) {
+ ERROR("Failed to open file\n");
+ return 1;
+ }
+
for ( i=0; i<dtempl->n_panels; i++ ) {
- if ( load_hdf5_hyperslab(&dtempl->panels[i], image->filename,
+ if ( load_hdf5_hyperslab(&dtempl->panels[i], fh,
image->ev, (void *)&image->dp[i],
H5T_NATIVE_FLOAT,
sizeof(float), 0,
dtempl->panels[i].data) )
{
ERROR("Failed to load panel data\n");
+ close_hdf5(fh);
return 1;
}
}
+ close_hdf5(fh);
return 0;
}
@@ -582,8 +616,12 @@ float *image_hdf5_read_satmap(struct panel_template *p,
const char *map_location)
{
float *map = NULL;
+ hid_t fh;
+
+ fh = open_hdf5_file(filename);
+ if ( fh < 0 ) return NULL;
- if ( load_hdf5_hyperslab(p, filename, event,
+ if ( load_hdf5_hyperslab(p, fh, event,
(void *)&map, H5T_NATIVE_FLOAT,
sizeof(float), 1, map_location) )
{
@@ -592,6 +630,8 @@ float *image_hdf5_read_satmap(struct panel_template *p,
return NULL;
}
+ close_hdf5(fh);
+
return map;
}
@@ -605,11 +645,15 @@ int image_hdf5_read_mask(struct panel_template *p,
int p_w, p_h;
int *mask = NULL;
long unsigned int j;
+ hid_t fh;
p_w = p->orig_max_fs - p->orig_min_fs + 1;
p_h = p->orig_max_ss - p->orig_min_ss + 1;
- if ( load_hdf5_hyperslab(p, filename, event,
+ fh = open_hdf5_file(filename);
+ if ( fh < 0 ) return 1;
+
+ if ( load_hdf5_hyperslab(p, fh, event,
(void *)&mask, H5T_NATIVE_INT,
sizeof(int), 1, mask_location) )
{
@@ -618,6 +662,8 @@ int image_hdf5_read_mask(struct panel_template *p,
return 1;
}
+ close_hdf5(fh);
+
for ( j=0; j<p_w*p_h; j++ ) {
/* Bad if it's missing any of the "good" bits */