From 6462dabdb52c14437f4e72de0a09d5efb04285f6 Mon Sep 17 00:00:00 2001
From: Thomas White <taw@physics.org>
Date: Tue, 16 Apr 2019 15:56:50 +0200
Subject: Recognise (if not handle) all CBF formats

---
 libcrystfel/src/image.c | 267 +++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 250 insertions(+), 17 deletions(-)

diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c
index d2d68a62..5cc073d8 100644
--- a/libcrystfel/src/image.c
+++ b/libcrystfel/src/image.c
@@ -558,6 +558,9 @@ static void add_out(float val, float *data_out, int nmemb_out,
 }
 
 
+/* Reverses byte offset compression and converts to single precision float.
+ * Note that this compression scheme specifies the data format of the input
+ * data, therefore the X-Binary-Element-Type is completely ignored. */
 static void decode_cbf_byte_offset(float *data_out, int nmemb_out,
                                    const int8_t *data_in, const size_t n)
 {
@@ -616,6 +619,152 @@ static int binary_start(char *data)
 }
 
 
+enum cbf_data_conversion
+{
+	CBF_NO_CONVERSION,
+	CBF_BYTE_OFFSET,
+	CBF_PACKED,
+	CBF_CANONICAL
+};
+
+enum cbf_data_type
+{
+	CBF_NO_TYPE,
+	CBF_ELEMENT_U8,
+	CBF_ELEMENT_S8,
+	CBF_ELEMENT_U16,
+	CBF_ELEMENT_S16,
+	CBF_ELEMENT_U32,
+	CBF_ELEMENT_S32,
+	CBF_ELEMENT_F32,
+	CBF_ELEMENT_F64,
+};
+
+
+static enum cbf_data_type parse_element_type(const char *t)
+{
+	if ( strstr(t, "signed 8-bit integer") != NULL )
+	{
+		return CBF_ELEMENT_S8;
+	}
+
+	if ( strstr(t, "unsigned 8-bit integer") != NULL )
+	{
+		return CBF_ELEMENT_U8;
+	}
+
+	if ( strstr(t, "signed 16-bit integer") != NULL )
+	{
+		return CBF_ELEMENT_S16;
+	}
+
+	if ( strstr(t, "unsigned 16-bit integer") != NULL )
+	{
+		return CBF_ELEMENT_U16;
+	}
+
+	if ( strstr(t, "signed 32-bit integer") != NULL )
+	{
+		return CBF_ELEMENT_S32;
+	}
+
+	if ( strstr(t, "unsigned 32-bit integer") != NULL )
+	{
+		return CBF_ELEMENT_U32;
+	}
+
+	if ( strstr(t, "signed 32-bit real IEEE") != NULL )
+	{
+		return CBF_ELEMENT_F32;
+	}
+
+	if ( strstr(t, "signed 64-bit real IEEE") != NULL )
+	{
+		return CBF_ELEMENT_F64;
+	}
+
+	/* complex type is unsupported */
+
+	return CBF_NO_TYPE;
+}
+
+
+static size_t element_size(enum cbf_data_type t)
+{
+	switch ( t ) {
+		case CBF_ELEMENT_S8  : return 1;
+		case CBF_ELEMENT_U8  : return 1;
+		case CBF_ELEMENT_S16 : return 2;
+		case CBF_ELEMENT_U16 : return 2;
+		case CBF_ELEMENT_S32 : return 4;
+		case CBF_ELEMENT_U32 : return 4;
+		case CBF_ELEMENT_F32 : return 4;
+		case CBF_ELEMENT_F64 : return 8;
+		default : return 0;
+	}
+}
+
+
+
+static int convert_type(float *data_out, long nmemb_exp,
+                        enum cbf_data_type eltype,
+                        void *data_in, size_t data_in_len)
+{
+	long int i;
+	long int o = 0;
+	size_t elsize = element_size(eltype);
+
+	if ( elsize == 0 ) return 1;
+
+	if ( nmemb_exp * elsize > data_in_len ) {
+		ERROR("Not enough CBF data for image size/type!\n");
+		return 1;
+	}
+
+	for ( i=0; i<nmemb_exp; i++ ) {
+		switch ( eltype ) {
+
+			case CBF_ELEMENT_S8:
+			data_out[o++] = ((int8_t *)data_in)[i];
+			break;
+
+			case CBF_ELEMENT_U8:
+			data_out[o++] = ((uint8_t *)data_in)[i];
+			break;
+
+			case CBF_ELEMENT_S16:
+			data_out[o++] = ((int16_t *)data_in)[i];
+			break;
+
+			case CBF_ELEMENT_U16:
+			data_out[o++] = ((uint16_t *)data_in)[i];
+			break;
+
+			case CBF_ELEMENT_S32:
+			data_out[o++] = ((int32_t *)data_in)[i];
+			break;
+
+			case CBF_ELEMENT_U32:
+			data_out[o++] = ((uint32_t *)data_in)[i];
+			break;
+
+			case CBF_ELEMENT_F32:
+			data_out[o++] = ((float *)data_in)[i];
+			break;
+
+			case CBF_ELEMENT_F64:
+			data_out[o++] = ((double *)data_in)[i];
+			break;
+
+			case CBF_NO_TYPE:
+			break;
+		}
+	}
+
+	return 0;
+}
+
+
 static float *read_cbf_data(struct imagefile *f, int *w, int *h)
 {
 	FILE *fh;
@@ -623,6 +772,9 @@ static float *read_cbf_data(struct imagefile *f, int *w, int *h)
 	char *rval;
 	size_t data_compressed_len = 0;
 	float *data_out = NULL;
+	enum cbf_data_conversion data_conversion = CBF_NO_CONVERSION;
+	enum cbf_data_type data_type = CBF_ELEMENT_U32;  /* ITG (2006) 2.3.3.3 */
+	int in_binary_section = 0;
 
 	*w = 0;
 	*h = 0;
@@ -676,7 +828,9 @@ static float *read_cbf_data(struct imagefile *f, int *w, int *h)
 		return NULL;
 	}
 
-	/* .... read data .... */
+	/* This is really horrible, but there are at least three different types
+	 * of header mingled together (CIF, MIME, DECTRIS), so a real parser
+	 * would be very complicated and much more likely to have weird bugs. */
 	do {
 
 		char line[1024];
@@ -687,27 +841,74 @@ static float *read_cbf_data(struct imagefile *f, int *w, int *h)
 		if ( rval == NULL ) break;
 		chomp(line);
 
-		if ( strncmp(line, "X-Binary-Size: ", 15) == 0 ) {
-			data_compressed_len = atoi(line+15);
+		if ( strcmp(line, "--CIF-BINARY-FORMAT-SECTION--") == 0 ) {
+			in_binary_section = 1;
 		}
 
-		if ( strncmp(line, "X-Binary-Size-Fastest-Dimension: ", 33) == 0 ) {
-			*w = atoi(line+33);
+		if ( strcmp(line, "--CIF-BINARY-FORMAT-SECTION----") == 0 ) {
+			in_binary_section = 0;
 		}
 
-		if ( strncmp(line, "X-Binary-Size-Second-Dimension: ", 32) == 0 ) {
-			*h = atoi(line+32);
-		}
+		if ( in_binary_section ) {
+
+			if ( strncmp(line, "X-Binary-Size: ", 15) == 0 ) {
+				data_compressed_len = atoi(line+15);
+			}
+
+			if ( strncmp(line, "X-Binary-Element-Byte-Order: ", 29) == 0 ) {
+				const char *elbo = line+29;
+				if ( strcmp(elbo, "LITTLE_ENDIAN") != 0 ) {
+					ERROR("Unsupported endianness: %s\n", elbo);
+					free(buf);
+					fclose(fh);
+					return NULL;
+				}
+			}
+
+			/* Try to spot compression algorithm */
+			if ( strstr(line, "conversions=\"x-CBF_BYTE_OFFSET\"") != NULL ) {
+				data_conversion = CBF_BYTE_OFFSET;
+			} else if ( strstr(line, "conversions=\"x-CBF_CANONICAL\"") != NULL ) {
+				data_conversion = CBF_CANONICAL;
+			} else if ( strstr(line, "conversions=\"x-CBF_PACKED\"") != NULL ) {
+				data_conversion = CBF_PACKED;
+			} else if ( strstr(line, "conversions=") != NULL ) {
+				ERROR("Unrecognised CBF content conversion: %s\n", line);
+				free(buf);
+				fclose(fh);
+				return NULL;
+			}
+
+			/* Likewise, element type */
+			if ( strncmp(line, "X-Binary-Element-Type: ", 23) == 0 )
+			{
+				const char *eltype = (line+23);
+				data_type = parse_element_type(eltype);
+				if ( data_type == CBF_NO_TYPE ) {
+					ERROR("Unrecognised element type: %s\n",
+					      eltype);
+					free(buf);
+					fclose(fh);
+					return NULL;
+				}
+			}
+
+			if ( strncmp(line, "X-Binary-Size-Fastest-Dimension: ", 33) == 0 ) {
+				*w = atoi(line+33);
+			}
+
+			if ( strncmp(line, "X-Binary-Size-Second-Dimension: ", 32) == 0 ) {
+				*h = atoi(line+32);
+			}
 
-		if ( strncmp(line, "X-Binary-Element-Type: ", 23) == 0 ) {
-			STATUS("binary type: %s\n", line+23);
 		}
 
-		if ( binary_start(line) ) {
+		if ( in_binary_section && binary_start(line) ) {
 
 			size_t len_read;
 			int nmemb_exp;
-			int8_t *data_compressed;
+			void *data_compressed;
+			int r = 0;
 
 			if ( data_compressed_len == 0 ) {
 				ERROR("Found CBF data before X-Binary-Size!\n");
@@ -758,19 +959,51 @@ static float *read_cbf_data(struct imagefile *f, int *w, int *h)
 				return NULL;
 			}
 
-			decode_cbf_byte_offset(data_out, nmemb_exp,
-			                       data_compressed,
-			                       data_compressed_len);
+			switch ( data_conversion ) {
+
+				case CBF_NO_CONVERSION:
+				r = convert_type(data_out, nmemb_exp, data_type,
+				                 data_compressed,
+				                 data_compressed_len);
+				break;
+
+				case CBF_BYTE_OFFSET:
+				decode_cbf_byte_offset(data_out, nmemb_exp,
+				                       data_compressed,
+				                       data_compressed_len);
+				break;
+
+				case CBF_PACKED:
+				case CBF_CANONICAL:
+				ERROR("Don't yet know how to decompress "
+				      "CBF_PACKED or CBF_CANONICAL\n");
+				free(buf);
+				free(data_compressed);
+				fclose(fh);
+				return NULL;
+
+			}
 
 			free(data_compressed);
 
+			if ( r ) {
+				free(buf);
+				free(data_out);
+				fclose(fh);
+				return NULL;
+			}
+
+			free(buf);
+			fclose(fh);
+			return data_out;
+
 		}
 
 	} while ( rval != NULL );
 
+	ERROR("Reached end of CBF file before finding data.\n");
 	free(buf);  /* might be NULL */
-
-	return data_out;
+	return NULL;
 }
 
 
-- 
cgit v1.2.3