Seedee deserialization

13 files changed, 277 insertions, 8 deletions

diff --git a/doc/man/indexamajig.1 b/doc/man/indexamajig.1
index 0660167e..fa108270 100644
--- a/doc/man/indexamajig.1
+++ b/doc/man/indexamajig.1
@@ -201,7 +201,7 @@ Authentication token, beamtime, data source and consumer group, respectively, fo
 .PD 0
 .IP \fB--data-format=\fIformat\fR
 .PD
-Specify the data format for data received over ZeroMQ or ASAP::O.  Possible values in this version are \fBmsgpack\fR and \fBhdf5\fR.
+Specify the data format for data received over ZeroMQ or ASAP::O.  Possible values in this version are \fBmsgpack\fR, \fBhdf5\fR and \fBseedee\fR.
 
 .PD 0
 .IP \fB--basename\fR
diff --git a/libcrystfel/CMakeLists.txt b/libcrystfel/CMakeLists.txt
index 3ca50b1f..ea475637 100644
--- a/libcrystfel/CMakeLists.txt
+++ b/libcrystfel/CMakeLists.txt
@@ -19,6 +19,7 @@ set(HAVE_FDIP ${FDIP_FOUND})
 set(HAVE_MSGPACK ${MSGPACK_FOUND})
 set(HAVE_LIBCCP4 ${LIBCCP4_FOUND})
 set(HAVE_ZLIB ${ZLIB_FOUND})
+set(HAVE_SEEDEE 0)
 
 # Recent enough version of zlib?
 set(CMAKE_REQUIRED_LIBRARIES "-lz")
@@ -62,6 +63,7 @@ set(LIBCRYSTFEL_SOURCES
     src/image-hdf5.c
     src/fom.c
     src/image-msgpack.c
+    src/image-seedee.c
     src/profile.c
     ${BISON_symopp_OUTPUTS}
     ${FLEX_symopl_OUTPUTS}
diff --git a/libcrystfel/libcrystfel-config.h.cmake.in b/libcrystfel/libcrystfel-config.h.cmake.in
index 830055cb..82d0611a 100644
--- a/libcrystfel/libcrystfel-config.h.cmake.in
+++ b/libcrystfel/libcrystfel-config.h.cmake.in
@@ -12,6 +12,7 @@
 #cmakedefine HAVE_MSGPACK
 #cmakedefine HAVE_CLOCK_GETTIME
 #cmakedefine HAVE_HDF5
+#cmakedefine HAVE_SEEDEE
 
 #cmakedefine HAVE_FORKPTY_PTY_H
 #cmakedefine HAVE_FORKPTY_UTIL_H
diff --git a/libcrystfel/libcrystfel-config.h.meson.in b/libcrystfel/libcrystfel-config.h.meson.in
index 7d43147d..302fae0b 100644
--- a/libcrystfel/libcrystfel-config.h.meson.in
+++ b/libcrystfel/libcrystfel-config.h.meson.in
@@ -10,6 +10,7 @@
 #mesondefine HAVE_MSGPACK
 #mesondefine HAVE_CLOCK_GETTIME
 #mesondefine HAVE_HDF5
+#mesondefine HAVE_SEEDEE
 
 #mesondefine HAVE_FORKPTY_PTY_H
 #mesondefine HAVE_FORKPTY_UTIL_H
diff --git a/libcrystfel/meson.build b/libcrystfel/meson.build
index b5005aa5..41cfbaa8 100644
--- a/libcrystfel/meson.build
+++ b/libcrystfel/meson.build
@@ -25,6 +25,14 @@ if fftwdep.found()
   conf_data.set10('HAVE_FFTW', true)
 endif
 
+seedeedep = dependency('seedee', required: false)
+cjsondep = dependency('cjson',
+                      required: true,
+                      fallback: ['cjson', 'libcjson_dep'])
+if cjsondep.found() and seedeedep.found()
+  conf_data.set10('HAVE_SEEDEE', 1)
+endif
+
 xgandalfdep = dependency('xgandalf',
                          required: false,
                          fallback: ['xgandalf', 'xgandalf_dep'])
@@ -125,6 +133,7 @@ libcrystfel_sources = ['src/image.c',
                        'src/image-cbf.c',
                        'src/image-hdf5.c',
                        'src/image-msgpack.c',
+                       'src/image-seedee.c',
                        'src/indexers/dirax.c',
                        'src/indexers/felix.c',
                        'src/indexers/mosflm.c',
@@ -150,7 +159,7 @@ libcrystfel = library('crystfel', [libcrystfel_sources, libcrystfel_versionc],
                       dependencies: [mdep, utildep, fftwdep, gsldep, zlibdep,
                                      hdf5dep, pthreaddep,
                                      xgandalfdep, pinkindexerdep, fdipdep,
-                                     ccp4dep, msgpackdep],
+                                     ccp4dep, msgpackdep, seedeedep, cjsondep],
                       install: true)
 
 libcrystfeldep = declare_dependency(include_directories: libcrystfel_includes,
diff --git a/libcrystfel/src/image-seedee.c b/libcrystfel/src/image-seedee.c
new file mode 100644
index 00000000..1965aefa
--- /dev/null
+++ b/libcrystfel/src/image-seedee.c
@@ -0,0 +1,195 @@
+/*
+ * image-seedee.c
+ *
+ * Image loading with Seedee
+ *
+ * Copyright © 2017-2022 Deutsches Elektronen-Synchrotron DESY,
+ *                       a research centre of the Helmholtz Association.
+ *
+ * Authors:
+ *   2018-2022 Thomas White <taw@physics.org>
+ *   2014      Valerio Mariani
+ *   2017      Stijn de Graaf
+ *
+ * This file is part of CrystFEL.
+ *
+ * CrystFEL is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * CrystFEL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with CrystFEL.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <libcrystfel-config.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include <image.h>
+#include <utils.h>
+#include <profile.h>
+
+#include "datatemplate_priv.h"
+
+
+#if defined(HAVE_SEEDEE)
+
+#include <seedee/seedee.h>
+#include <cjson/cJSON.h>
+
+
+static int load_seedee_data(struct panel_template *p,
+                            struct SeedeeNDArray *array,
+                            float **pdata)
+{
+	int data_size_fs, data_size_ss;
+	float *data = NULL;
+
+	data_size_ss = array->shape[0];
+	data_size_fs = array->shape[1];
+
+	if ( (p->orig_min_fs + PANEL_WIDTH(p) > data_size_fs)
+	  || (p->orig_min_ss + PANEL_HEIGHT(p) > data_size_ss) )
+	{
+		ERROR("Data for panel %s (%i x %i + %i + %i) is outside data "
+		      "array bounds (%i x %i)\n",
+		      p->name,
+		      PANEL_WIDTH(p), PANEL_HEIGHT(p),
+		      p->orig_min_fs, p->orig_min_ss,
+		      data_size_fs, data_size_ss);
+		return 1;
+	}
+
+	if ( (array->datatype == 'u')
+	  && (array->itemsize == 2)
+	  && (array->byteorder == '<') )
+	{
+		int fs, ss;
+		uint16_t *in_data = (uint16_t *)array->data;
+
+		data = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float));
+		if ( data == NULL ) return 1;
+
+		for ( ss=0; ss<PANEL_HEIGHT(p); ss++ ) {
+			for ( fs=0; fs<PANEL_WIDTH(p); fs++ ) {
+				size_t idx = fs+p->orig_min_fs + (ss+p->orig_min_ss)*data_size_fs;
+				data[fs+ss*PANEL_WIDTH(p)] = in_data[idx];
+			}
+		}
+		*pdata = data;
+
+	} else {
+		ERROR("Unrecognised data type %c%i%c\n",
+		      array->datatype, array->itemsize, array->byteorder);
+		return 1;
+	}
+
+	return 0;
+}
+
+
+/* Read the image data from 'data_block' into 'image', according to 'dtempl' */
+int image_seedee_read(struct image *image,
+                      DataTemplate *dtempl,
+                      void *data_block,
+                      size_t data_block_size,
+                      char *meta_data)
+{
+	struct SeedeeNDArray array;
+	int r;
+	bool zero_copy;
+	int i;
+	cJSON *json;
+	cJSON *data_format_str;
+
+	json = cJSON_Parse(meta_data);
+	if ( json == NULL ) {
+		ERROR("Failed to parse JSON\n");
+		return 1;
+	}
+
+	data_format_str = cJSON_GetObjectItemCaseSensitive(json, "_data_format");
+	if ( !cJSON_IsString(data_format_str) ) {
+		ERROR("_data_format isn't a string");
+		cJSON_Delete(json);
+		return 1;
+	}
+
+	profile_start("seedee-get-size");
+	array.size = seedee_get_data_size(data_format_str->valuestring,
+	                                  data_block, data_block_size,
+	                                  &zero_copy, &array);
+	profile_end("seedee-get-size");
+	array.data = malloc(array.size);
+	array.shape = malloc(array.ndims*sizeof(int));
+	if ( (array.data == NULL) || (array.shape == NULL) ) {
+		cJSON_Delete(json);
+		return 1;
+	}
+
+	if ( array.ndims != 2 ) {
+		ERROR("Seedee data has unexpected number of dimensions "
+		      "(%i, expected 2)\n", array.ndims);
+		return 1;
+	}
+
+	profile_start("seedee-deserialize");
+	r = seedee_deserialize_ndarray(data_format_str->valuestring,
+	                               data_block, data_block_size,
+	                               0, &array);
+	profile_end("seedee-deserialize");
+	cJSON_Delete(json);
+	if ( r < 0 ) {
+		ERROR("Seedee deserialiation failed.\n");
+		return 1;
+	}
+
+	image->dp = malloc(dtempl->n_panels*sizeof(float *));
+	if ( image->dp == NULL ) {
+		ERROR("Failed to allocate data array.\n");
+		return 1;
+	}
+
+	/* Set all pointers to NULL for easier clean-up */
+	for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL;
+
+	profile_start("seedee-panel");
+	for ( i=0; i<dtempl->n_panels; i++ ) {
+		if ( load_seedee_data(&dtempl->panels[i], &array, &image->dp[i]) )
+		{
+			ERROR("Failed to load data for panel '%s'\n",
+			      dtempl->panels[i].name);
+			profile_end("seedee-panel");
+			return 1;
+		}
+	}
+	profile_end("seedee-panel");
+
+	return 0;
+}
+
+
+#else /* defined(HAVE_SEEDEE) */
+
+int image_seedee_read(struct image *image,
+                      const DataTemplate *dtempl,
+                      void *data,
+                      size_t data_size,
+                      char *meta_data)
+{
+	ERROR("Seedee is not supported in this installation (read).\n");
+	return 1;
+}
+
+#endif /* defined(HAVE_SEEDEE) */
diff --git a/libcrystfel/src/image-seedee.h b/libcrystfel/src/image-seedee.h
new file mode 100644
index 00000000..9432e74b
--- /dev/null
+++ b/libcrystfel/src/image-seedee.h
@@ -0,0 +1,41 @@
+/*
+ * image-seedee.h
+ *
+ * Image loading, SeeDee parts
+ *
+ * Copyright © 2012-2022 Deutsches Elektronen-Synchrotron DESY,
+ *                       a research centre of the Helmholtz Association.
+ *
+ * Authors:
+ *   2020-2022 Thomas White <taw@physics.org>
+ *
+ * This file is part of CrystFEL.
+ *
+ * CrystFEL is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * CrystFEL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with CrystFEL.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef IMAGE_SEEDEE_H
+#define IMAGE_SEEDEE_H
+
+#include "datatemplate.h"
+
+
+extern int image_seedee_read(struct image *image,
+                             const DataTemplate *dtempl,
+                             void *data,
+                             size_t data_size,
+                             char *meta_data);
+
+#endif	/* IMAGE_SEEDEE_H */
diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c
index 10c05525..f9c75b19 100644
--- a/libcrystfel/src/image.c
+++ b/libcrystfel/src/image.c
@@ -42,6 +42,7 @@
 #include "image-hdf5.h"
 #include "image-cbf.h"
 #include "image-msgpack.h"
+#include "image-seedee.h"
 #include "profile.h"
 
 #include "datatemplate.h"
@@ -803,6 +804,11 @@ static int image_read_image_data(struct image *image,
 		return image_msgpack_read(image, dtempl, image->data_block,
 		                          image->data_block_size);
 
+		case DATA_SOURCE_TYPE_SEEDEE:
+		return image_seedee_read(image, dtempl, image->data_block,
+		                         image->data_block_size,
+		                         image->meta_data);
+
 		default:
 		ERROR("Unrecognised file type %i (image_read_image_data)\n",
 		      image->data_source_type);
diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h
index 6c7e2a50..654bf3b1 100644
--- a/libcrystfel/src/image.h
+++ b/libcrystfel/src/image.h
@@ -103,7 +103,8 @@ typedef enum
 	DATA_SOURCE_TYPE_HDF5,
 	DATA_SOURCE_TYPE_CBF,
 	DATA_SOURCE_TYPE_CBFGZ,
-	DATA_SOURCE_TYPE_MSGPACK
+	DATA_SOURCE_TYPE_MSGPACK,
+	DATA_SOURCE_TYPE_SEEDEE
 } DataSourceType;
 
 
diff --git a/src/im-asapo.h b/src/im-asapo.h
index 719b5bf1..5fd7665c 100644
--- a/src/im-asapo.h
+++ b/src/im-asapo.h
@@ -3,11 +3,11 @@
  *
  * ASAP::O data interface
  *
- * Copyright © 2021 Deutsches Elektronen-Synchrotron DESY,
- *                  a research centre of the Helmholtz Association.
+ * Copyright © 2021-2022 Deutsches Elektronen-Synchrotron DESY,
+ *                       a research centre of the Helmholtz Association.
  *
  * Authors:
- *   2021 Thomas White <taw@physics.org>
+ *   2021-2022 Thomas White <taw@physics.org>
  *
  * This file is part of CrystFEL.
  *
diff --git a/src/indexamajig.c b/src/indexamajig.c
index 8d383d8e..dbea7261 100644
--- a/src/indexamajig.c
+++ b/src/indexamajig.c
@@ -297,6 +297,7 @@ static DataSourceType parse_data_format(const char *str)
 {
 	if ( strcmp(str, "hdf5") == 0 ) return DATA_SOURCE_TYPE_HDF5;
 	if ( strcmp(str, "msgpack") == 0 ) return DATA_SOURCE_TYPE_MSGPACK;
+	if ( strcmp(str, "seedee") == 0 ) return DATA_SOURCE_TYPE_SEEDEE;
 	/* CBF and CBFGZ should be added here once image-cbf.c supports
 	 * in-memory access */
 	return DATA_SOURCE_TYPE_UNKNOWN;
diff --git a/src/process_image.h b/src/process_image.h
index 5e12ff29..e2f792a5 100644
--- a/src/process_image.h
+++ b/src/process_image.h
@@ -3,11 +3,11 @@
  *
  * The processing pipeline for one image
  *
- * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY,
+ * Copyright © 2012-2022 Deutsches Elektronen-Synchrotron DESY,
  *                       a research centre of the Helmholtz Association.
  *
  * Authors:
- *   2010-2021 Thomas White <taw@physics.org>
+ *   2010-2022 Thomas White <taw@physics.org>
  *   2014-2017 Valerio Mariani <valerio.mariani@desy.de>
  *   2017-2018 Yaroslav Gevorkov <yaroslav.gevorkov@desy.de>
  *
diff --git a/subprojects/cjson.wrap b/subprojects/cjson.wrap
new file mode 100644
index 00000000..dc10279a
--- /dev/null
+++ b/subprojects/cjson.wrap
@@ -0,0 +1,12 @@
+[wrap-file]
+directory = cJSON-1.7.15
+source_url = https://github.com/DaveGamble/cJSON/archive/refs/tags/v1.7.15.tar.gz
+source_filename = v1.7.15.tar.gz
+source_hash = 5308fd4bd90cef7aa060558514de6a1a4a0819974a26e6ed13973c5f624c24b2
+patch_filename = cjson_1.7.15-2_patch.zip
+patch_url = https://wrapdb.mesonbuild.com/v2/cjson_1.7.15-2/get_patch
+patch_hash = d83b4bc0ca94e392c62c8c6c7839392f382d66a84974f5e10611074836ef1777
+
+[provide]
+libcjson = libcjson_dep
+