From 3d55c5c01aaeedfdcb16ee079080b26a3e8c6830 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Tue, 14 Sep 2021 12:30:37 +0200 Subject: Handle in-memory HDF5s --- CMakeLists.txt | 2 +- libcrystfel/CMakeLists.txt | 4 +- libcrystfel/src/image-hdf5.c | 102 +++++++++++++++++++++++++++++++------------ 3 files changed, 77 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78bcce10..1f69bed2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ include(GNUInstallDirs) # Set CMAKE_MODULE_PATH to assist cmake in finding our own package definitions set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake") -find_package(HDF5 REQUIRED COMPONENTS C) +find_package(HDF5 REQUIRED COMPONENTS C HL) find_package(Threads REQUIRED) find_package(GSL REQUIRED) find_package(OpenCL) diff --git a/libcrystfel/CMakeLists.txt b/libcrystfel/CMakeLists.txt index 1c8ef887..77290677 100644 --- a/libcrystfel/CMakeLists.txt +++ b/libcrystfel/CMakeLists.txt @@ -138,8 +138,8 @@ target_include_directories(${PROJECT_NAME} INTERFACE ${PROJECT_SOURCE_DIR}/src) include_directories(${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(${PROJECT_NAME} PRIVATE ${HDF5_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS}) -target_link_libraries(${PROJECT_NAME} PRIVATE util ${HDF5_C_LIBRARIES} ${ZLIB_LIBRARIES} - Threads::Threads GSL::gsl m) +target_link_libraries(${PROJECT_NAME} PRIVATE util ${HDF5_C_LIBRARIES} ${HDF5_C_HL_LIBRARIES} + ${ZLIB_LIBRARIES} Threads::Threads GSL::gsl m) if (XGANDALF_FOUND) target_include_directories(${PROJECT_NAME} PRIVATE ${XGANDALF_INCLUDE_DIRS}) diff --git a/libcrystfel/src/image-hdf5.c b/libcrystfel/src/image-hdf5.c index 7898d06b..ab8d131d 100644 --- a/libcrystfel/src/image-hdf5.c +++ b/libcrystfel/src/image-hdf5.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include "image.h" @@ -355,14 +356,13 @@ static int total_dimensions(const struct panel_template *p) static int load_hdf5_hyperslab(struct panel_template *p, - const char *filename, + hid_t fh, const char *event, void **pdata, hid_t el_type, size_t el_size, int skip_placeholders_ok, const char *path_spec) { - hid_t fh; int total_dt_dims; int plh_dt_dims; int dt_dims[MAX_DIMS]; @@ -381,33 +381,19 @@ static int load_hdf5_hyperslab(struct panel_template *p, int n_dim_vals; int pl_pos; - if ( access(filename, R_OK) == -1 ) { - ERROR("File does not exist or cannot be read: %s\n", - filename); - return 1; - } - - fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); - if ( fh < 0 ) { - ERROR("Couldn't open file (hyperslab): %s\n", filename); - return 1; - } - panel_full_path = substitute_path(event, path_spec, skip_placeholders_ok); if ( panel_full_path == NULL ) { ERROR("Invalid path substitution: '%s' '%s'\n", event, path_spec); - close_hdf5(fh); return 1; } dh = H5Dopen2(fh, panel_full_path, H5P_DEFAULT); if ( dh < 0 ) { - ERROR("Cannot open data for panel %s (%s) in file %s\n", - p->name, panel_full_path, filename); + ERROR("Cannot open data for panel %s (%s)\n", + p->name, panel_full_path); free(panel_full_path); - close_hdf5(fh); return 1; } @@ -420,7 +406,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, if ( ndims < 0 ) { ERROR("Failed to get number of dimensions for panel %s\n", p->name); - close_hdf5(fh); return 1; } @@ -441,7 +426,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, "panel %s (%i, but expected %i or %i)\n", p->name, ndims, total_dt_dims, total_dt_dims - plh_dt_dims); - close_hdf5(fh); return 1; } } else { @@ -456,7 +440,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, f_count = malloc(ndims*sizeof(hsize_t)); if ( (f_offset == NULL) || (f_count == NULL ) ) { ERROR("Failed to allocate offset or count.\n"); - close_hdf5(fh); return 1; } @@ -505,7 +488,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, p->name); free(f_offset); free(f_count); - close_hdf5(fh); return 1; } @@ -518,7 +500,6 @@ static int load_hdf5_hyperslab(struct panel_template *p, ERROR("Failed to allocate panel %s\n", p->name); free(f_offset); free(f_count); - close_hdf5(fh); return 1; } @@ -529,23 +510,68 @@ static int load_hdf5_hyperslab(struct panel_template *p, free(f_offset); free(f_count); free(data); - close_hdf5(fh); return 1; } free(f_offset); free(f_count); - close_hdf5(fh); *pdata = data; return 0; } +static hid_t open_hdf5_file(const char *filename) +{ + hid_t fh; + + if ( access(filename, R_OK) == -1 ) { + ERROR("File does not exist or cannot be read: %s\n", + filename); + return -1; + } + + fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if ( fh < 0 ) { + ERROR("Couldn't open HDF5 file: %s\n", filename); + return -1; + } + + return fh; +} + + +static hid_t open_hdf5(struct image *image) +{ + if ( image->data_block == NULL ) { + + return open_hdf5_file(image->filename); + + } else { + + hid_t fh; + + fh = H5LTopen_file_image(image->data_block, + image->data_block_size, + H5LT_FILE_IMAGE_DONT_COPY + | H5LT_FILE_IMAGE_DONT_RELEASE); + + if ( fh < 0 ) { + ERROR("Couldn't open HDF5 image (%p %lli)\n", + image->data_block, image->data_block_size); + return -1; + } + + return fh; + } +} + + int image_hdf5_read(struct image *image, const DataTemplate *dtempl) { int i; + hid_t fh; image->dp = malloc(dtempl->n_panels*sizeof(float *)); if ( image->dp == NULL ) { @@ -560,18 +586,26 @@ int image_hdf5_read(struct image *image, /* Set all pointers to NULL for easier clean-up */ for ( i=0; in_panels; i++ ) image->dp[i] = NULL; + fh = open_hdf5(image); + if ( fh < 0 ) { + ERROR("Failed to open file\n"); + return 1; + } + for ( i=0; in_panels; i++ ) { - if ( load_hdf5_hyperslab(&dtempl->panels[i], image->filename, + if ( load_hdf5_hyperslab(&dtempl->panels[i], fh, image->ev, (void *)&image->dp[i], H5T_NATIVE_FLOAT, sizeof(float), 0, dtempl->panels[i].data) ) { ERROR("Failed to load panel data\n"); + close_hdf5(fh); return 1; } } + close_hdf5(fh); return 0; } @@ -582,8 +616,12 @@ float *image_hdf5_read_satmap(struct panel_template *p, const char *map_location) { float *map = NULL; + hid_t fh; + + fh = open_hdf5_file(filename); + if ( fh < 0 ) return NULL; - if ( load_hdf5_hyperslab(p, filename, event, + if ( load_hdf5_hyperslab(p, fh, event, (void *)&map, H5T_NATIVE_FLOAT, sizeof(float), 1, map_location) ) { @@ -592,6 +630,8 @@ float *image_hdf5_read_satmap(struct panel_template *p, return NULL; } + close_hdf5(fh); + return map; } @@ -605,11 +645,15 @@ int image_hdf5_read_mask(struct panel_template *p, int p_w, p_h; int *mask = NULL; long unsigned int j; + hid_t fh; p_w = p->orig_max_fs - p->orig_min_fs + 1; p_h = p->orig_max_ss - p->orig_min_ss + 1; - if ( load_hdf5_hyperslab(p, filename, event, + fh = open_hdf5_file(filename); + if ( fh < 0 ) return 1; + + if ( load_hdf5_hyperslab(p, fh, event, (void *)&mask, H5T_NATIVE_INT, sizeof(int), 1, mask_location) ) { @@ -618,6 +662,8 @@ int image_hdf5_read_mask(struct panel_template *p, return 1; } + close_hdf5(fh); + for ( j=0; j