/* * index.c * * Perform indexing (somehow) * * Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Lorenzo Galli * * Authors: * 2010-2017 Thomas White * 2010-2011 Richard Kirian * 2012 Lorenzo Galli * 2013 Cornelius Gati * 2015 Kenneth Beyerlein * 2014 Takanori Nakane * * This file is part of CrystFEL. * * CrystFEL is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * CrystFEL is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with CrystFEL. If not, see . * */ #ifdef HAVE_CONFIG_H #include #endif #include #include #include #include #include #include #include "image.h" #include "utils.h" #include "peaks.h" #include "dirax.h" #include "asdf.h" #include "mosflm.h" #include "xds.h" #include "detector.h" #include "index.h" #include "geometry.h" #include "cell-utils.h" #include "felix.h" #include "predict-refine.h" #include "taketwo.h" #include "xgandalf.h" /** \file index.h */ struct _indexingprivate { IndexingFlags flags; UnitCell *target_cell; double tolerance[6]; struct taketwo_options *ttopts; struct xgandalf_options *xgandalf_opts; int n_methods; IndexingMethod *methods; void **engine_private; }; static const char *onoff(int a) { if ( a ) return "on"; return "off"; } /* Definition and function definition duplicated here (from im-sandbox.{c,h}) * because libcrystfel code cannot depend on core CrystFEL programs. * * Must match the value and definition in im-sandbox.h */ #define MAX_TASK_LEN (32) static void set_last_task(char *lt, const char *task) { if ( lt == NULL ) return; assert(strlen(task) < MAX_TASK_LEN-1); strcpy(lt, task); } static void show_indexing_flags(IndexingFlags flags) { char check[64]; assert( !((flags & INDEXING_CHECK_CELL_COMBINATIONS) && (flags & INDEXING_CHECK_CELL_AXES)) ); STATUS("Indexing parameters:\n"); strcpy(check, onoff(flags & (INDEXING_CHECK_CELL_COMBINATIONS | INDEXING_CHECK_CELL_AXES))); if ( flags & INDEXING_CHECK_CELL_AXES ) { strcat(check, " (axis permutations only)"); } if ( flags & INDEXING_CHECK_CELL_COMBINATIONS ) { strcat(check, " (axis combinations)"); } STATUS(" Check unit cell parameters: %s\n", check); STATUS(" Check peak alignment: %s\n", onoff(flags & INDEXING_CHECK_PEAKS)); STATUS(" Refine indexing solutions: %s\n", onoff(flags & INDEXING_REFINE)); STATUS(" Multi-lattice indexing (\"delete and retry\"): %s\n", onoff(flags & INDEXING_MULTI)); STATUS(" Retry indexing: %s\n", onoff(flags & INDEXING_RETRY)); } static int debug_index(struct image *image) { FILE *fh; Crystal *cr; UnitCell *cell; float asx, asy, asz, bsx, bsy, bsz, csx, csy, csz; fh = fopen("../../indexing.debug", "r"); if ( fh == NULL ) { ERROR("indexing.debug not found\n"); return 0; } if ( fscanf(fh, "%e %e %e", &asx, &asy, &asz) != 3 ) { ERROR("Failed to read a* from indexing.debug\n"); return 0; } if ( fscanf(fh, "%e %e %e", &bsx, &bsy, &bsz) != 3 ) { ERROR("Failed to read b* from indexing.debug\n"); return 0; } if ( fscanf(fh, "%e %e %e", &csx, &csy, &csz) != 3 ) { ERROR("Failed to read c* from indexing.debug\n"); return 0; } cr = crystal_new(); cell = cell_new(); cell_set_reciprocal(cell, asx, asy, asz, bsx, bsy, bsz, csx, csy, csz); crystal_set_cell(cr, cell); image_add_crystal(image, cr); return 1; } static char *base_indexer_str(IndexingMethod indm) { char *str; str = malloc(256); if ( str == NULL ) { ERROR("Failed to allocate string.\n"); return NULL; } str[0] = '\0'; switch ( indm & INDEXING_METHOD_MASK ) { case INDEXING_NONE : strcpy(str, "none"); return str; case INDEXING_DIRAX : strcpy(str, "dirax"); break; case INDEXING_ASDF : strcpy(str, "asdf"); break; case INDEXING_MOSFLM : strcpy(str, "mosflm"); break; case INDEXING_FELIX : strcpy(str, "felix"); break; case INDEXING_XDS : strcpy(str, "xds"); break; case INDEXING_TAKETWO : strcpy(str, "taketwo"); break; case INDEXING_XGANDALF: strcpy(str, "xgandalf"); break; case INDEXING_SIMULATION : strcpy(str, "simulation"); break; case INDEXING_DEBUG : strcpy(str, "debug"); break; default : strcpy(str, "(unknown)"); break; } return str; } static char *friendly_indexer_name(IndexingMethod m) { char *base = base_indexer_str(m & INDEXING_METHOD_MASK); if ( (m & INDEXING_USE_CELL_PARAMETERS) && (m & INDEXING_USE_LATTICE_TYPE) ) { strcat(base, " using cell parameters and Bravais lattice type " "as prior information"); } else if ( m & INDEXING_USE_CELL_PARAMETERS ) { strcat(base, " using cell parameters as prior information"); } else if ( m & INDEXING_USE_LATTICE_TYPE ) { strcat(base, " using Bravais lattice type as prior information"); } else { strcat(base, " - no prior information"); } return base; } static void *prepare_method(IndexingMethod *m, UnitCell *cell, struct xgandalf_options *xgandalf_opts, struct felix_options *felix_opts) { char *str; IndexingMethod in = *m; void *priv = NULL; switch ( *m & INDEXING_METHOD_MASK ) { case INDEXING_NONE : priv = "none"; break; case INDEXING_DIRAX : priv = dirax_prepare(m, cell); break; case INDEXING_ASDF : priv = asdf_prepare(m, cell); break; case INDEXING_MOSFLM : priv = mosflm_prepare(m, cell); break; case INDEXING_XDS : priv = xds_prepare(m, cell); break; case INDEXING_DEBUG : priv = (IndexingPrivate *)strdup("Hello!"); break; case INDEXING_FELIX : priv = felix_prepare(m, cell, felix_opts); break; case INDEXING_TAKETWO : priv = taketwo_prepare(m, cell); break; case INDEXING_XGANDALF : priv = xgandalf_prepare(m, cell, xgandalf_opts); break; default : ERROR("Don't know how to prepare indexing method %i\n", *m); break; } str = indexer_str(*m); if ( priv == NULL ) { ERROR("Failed to prepare indexing method %s\n", str); free(str); return NULL; } free(str); if ( in != *m ) { ERROR("Note: flags were altered to take into account " "the information provided and/or the limitations " "of the indexing method.\nPlease check the " "methods listed above carefully.\n"); } return priv; } IndexingPrivate *setup_indexing(const char *method_list, UnitCell *cell, struct detector *det, float *tols, IndexingFlags flags, struct taketwo_options *ttopts, struct xgandalf_options *xgandalf_opts, struct felix_options *felix_opts) { int i, n; char **method_strings; IndexingPrivate *ipriv; /* Parse indexing methods */ n = assplode(method_list, ",", &method_strings, ASSPLODE_NONE); IndexingMethod *methods = malloc(n * sizeof(IndexingMethod)); if ( methods == NULL ) { ERROR("Failed to allocate indexing method list\n"); return NULL; } for ( i=0; i no cell checking, no prior cell */ if ( !cell_has_parameters(cell) ) { int warn = 0; if ( (flags & INDEXING_CHECK_CELL_COMBINATIONS) || (flags & INDEXING_CHECK_CELL_AXES) ) { ERROR("WARNING: Forcing --no-check-cell because " "reference unit cell parameters were not " "given.\n"); flags &= ~INDEXING_CHECK_CELL_COMBINATIONS; flags &= ~INDEXING_CHECK_CELL_AXES; } for ( i=0; i no prior lattice type */ if ( cell == NULL ) { int warn = 0; for ( i=0; iengine_private = malloc((n+1) * sizeof(void *)); for ( i=0; iengine_private[i] = prepare_method(&methods[i], cell, xgandalf_opts, felix_opts); if ( ipriv->engine_private[i] == NULL ) return NULL; for ( j=0; jmethods = methods; ipriv->n_methods = n; ipriv->flags = flags; if ( cell != NULL ) { ipriv->target_cell = cell_new_from_cell(cell); } else { ipriv->target_cell = NULL; } for ( i=0; i<6; i++ ) ipriv->tolerance[i] = tols[i]; ipriv->ttopts = ttopts; ipriv->xgandalf_opts = xgandalf_opts; STATUS("List of indexing methods:\n"); for ( i=0; in_methods; n++ ) { switch ( ipriv->methods[n] & INDEXING_METHOD_MASK ) { case INDEXING_NONE : break; case INDEXING_DIRAX : dirax_cleanup(ipriv->engine_private[n]); break; case INDEXING_ASDF : asdf_cleanup(ipriv->engine_private[n]); break; case INDEXING_MOSFLM : mosflm_cleanup(ipriv->engine_private[n]); break; case INDEXING_XDS : xds_cleanup(ipriv->engine_private[n]); break; case INDEXING_FELIX : felix_cleanup(ipriv->engine_private[n]); break; case INDEXING_DEBUG : free(ipriv->engine_private[n]); break; case INDEXING_TAKETWO : taketwo_cleanup(ipriv->engine_private[n]); break; case INDEXING_XGANDALF : xgandalf_cleanup(ipriv->engine_private[n]); break; default : ERROR("Don't know how to clean up indexing method %i\n", ipriv->methods[n]); break; } } free(ipriv->methods); free(ipriv->engine_private); cell_free(ipriv->target_cell); free(ipriv); } void map_all_peaks(struct image *image) { int i, n; n = image_feature_count(image->features); /* Map positions to 3D */ for ( i=0; ifeatures, i); if ( f == NULL ) continue; r = get_q_for_panel(f->p, f->fs, f->ss, NULL, 1.0/image->lambda); f->rx = r.u; f->ry = r.v; f->rz = r.w; } } /* Return 0 for cell OK, 1 for cell incorrect */ static int check_cell(IndexingFlags flags, Crystal *cr, UnitCell *target, double *tolerance) { UnitCell *out; IntegerMatrix *im; RationalMatrix *rm; /* Check at all? */ if ( ! ((flags & INDEXING_CHECK_CELL_COMBINATIONS) || (flags & INDEXING_CHECK_CELL_AXES)) ) return 0; if ( compare_permuted_cell_parameters(crystal_get_cell(cr), target, tolerance, &im) ) { out = cell_transform_intmat(crystal_get_cell(cr), im); cell_free(crystal_get_cell(cr)); crystal_set_cell(cr, out); intmat_free(im); return 0; } if ( (flags & INDEXING_CHECK_CELL_COMBINATIONS ) && compare_reindexed_cell_parameters(crystal_get_cell(cr), target, tolerance, 0, &rm) ) { out = cell_transform_rational(crystal_get_cell(cr), rm); cell_free(crystal_get_cell(cr)); crystal_set_cell(cr, out); rtnl_mtx_free(rm); return 0; } return 1; } /* Return non-zero for "success" */ static int try_indexer(struct image *image, IndexingMethod indm, IndexingPrivate *ipriv, void *mpriv, char *last_task) { int i, r; int n_bad = 0; int n_before = image->n_crystals; switch ( indm & INDEXING_METHOD_MASK ) { case INDEXING_NONE : return 0; case INDEXING_DIRAX : set_last_task(last_task, "indexing:dirax"); r = run_dirax(image, mpriv); break; case INDEXING_ASDF : set_last_task(last_task, "indexing:asdf"); r = run_asdf(image, mpriv); break; case INDEXING_MOSFLM : set_last_task(last_task, "indexing:mosflm"); r = run_mosflm(image, mpriv); break; case INDEXING_XDS : set_last_task(last_task, "indexing:xds"); r = run_xds(image, mpriv); break; case INDEXING_DEBUG : set_last_task(last_task, "indexing:debug"); r = debug_index(image); break; case INDEXING_FELIX : set_last_task(last_task, "indexing:felix"); r = felix_index(image, mpriv); break; case INDEXING_TAKETWO : set_last_task(last_task, "indexing:taketwo"); r = taketwo_index(image, ipriv->ttopts, mpriv); break; case INDEXING_XGANDALF : set_last_task(last_task, "indexing:xgandalf"); r = run_xgandalf(image, mpriv); break; default : ERROR("Unrecognised indexing method: %i\n", indm); return 0; } set_last_task(last_task, "indexing:finalisation"); /* Stop a really difficult to debug situation in its tracks */ if ( image->n_crystals - n_before != r ) { ERROR("Whoops, indexer didn't return the right number " "of crystals!\n"); exit(1); } /* For all the crystals found this time ... */ for ( i=0; in_crystals - i - 1; /* ... starting at the end of the (complete) list ... */ Crystal *cr = image->crystals[this_crystal]; crystal_set_image(cr, image); crystal_set_profile_radius(cr, 0.02e9); crystal_set_mosaicity(cr, 0.0); assert( !((ipriv->flags & INDEXING_CHECK_CELL_COMBINATIONS) && (ipriv->flags & INDEXING_CHECK_CELL_AXES)) ); /* Pre-refinement unit cell check if requested */ if ( check_cell(ipriv->flags, cr, ipriv->target_cell, ipriv->tolerance) ) { crystal_set_user_flag(cr, 1); continue; } /* Prediction refinement if requested */ if ( ipriv->flags & INDEXING_REFINE ) { if ( refine_prediction(image, cr) ) { crystal_set_user_flag(cr, 1); continue; } } /* After refinement unit cell check if requested */ if ( check_cell(ipriv->flags, cr, ipriv->target_cell, ipriv->tolerance) ) { crystal_set_user_flag(cr, 1); continue; } /* Peak alignment check if requested */ if ( ipriv->flags & INDEXING_CHECK_PEAKS ) { if ( !peak_sanity_check(image, &cr, 1) ) { crystal_set_user_flag(cr, 1); continue; } } /* Don't do similarity check if this crystal is bad */ if ( crystal_get_user_flag(cr) ) continue; /* Check if cell is too similar to existing ones */ for ( j=0; jcrystals[j]; /* Don't do similarity check against bad crystals */ if ( crystal_get_user_flag(that_cr) ) continue; if ( compare_permuted_cell_parameters_and_orientation(crystal_get_cell(cr), crystal_get_cell(that_cr), 0.1, deg2rad(0.5), NULL) ) { crystal_set_user_flag(cr, 1); } } } n_bad = remove_flagged_crystals(image); assert(r >= n_bad); return r - n_bad; } static int delete_weakest_peaks(ImageFeatureList *peaks) { int i; int np, ndel, n; np = image_feature_count(peaks); n = 0; for ( i=0; in-ndel-1; i-- ) { image_remove_feature(peaks, i); } return 0; } static int delete_explained_peaks(struct image *image, Crystal *cr) { double ax, ay, az; double bx, by, bz; double cx, cy, cz; const double min_dist = 0.25; int i, nspots = 0, nindexed = 0; /* Round towards nearest */ fesetround(1); /* Cell basis vectors for this image */ cell_get_cartesian(crystal_get_cell(cr), &ax, &ay, &az, &bx, &by, &bz, &cx, &cy, &cz); /* Loop over peaks, checking proximity to nearest reflection */ for ( i=0; ifeatures); i++ ) { struct imagefeature *f; struct rvec q; double h, k, l, hd, kd, ld; double dsq; f = image_get_feature(image->features, i); if ( f == NULL ) continue; nspots++; /* Reciprocal space position of found peak */ q = get_q_for_panel(f->p, f->fs, f->ss, NULL, 1.0/image->lambda); /* Decimal and fractional Miller indices of nearest * reciprocal lattice point */ hd = q.u * ax + q.v * ay + q.w * az; kd = q.u * bx + q.v * by + q.w * bz; ld = q.u * cx + q.v * cy + q.w * cz; h = lrint(hd); k = lrint(kd); l = lrint(ld); /* Check distance */ dsq = pow(h-hd, 2.0) + pow(k-kd, 2.0) + pow(l-ld, 2.0); if ( sqrt(dsq) < min_dist ) { image_remove_feature(image->features, i); nindexed++; } } /* Return TRUE if not enough peaks to continue */ return (nspots - nindexed) < 5; } /* indm = the current indexing method * r = the result from try_indexer() on this method just now * image = the image structure * * Returns false for "try again", true for "no, stop now" */ static int finished_retry(IndexingMethod indm, IndexingFlags flags, int r, struct image *image) { if ( r == 0 ) { /* Indexing failed on the previous attempt. Maybe try again * after poking the peak list a bit */ if ( flags & INDEXING_RETRY ) { /* Retry with fewer peaks */ return delete_weakest_peaks(image->features); } else { /* Indexing failed, opted not to try again */ return 1; } } else { /* Indexing succeeded on previous attempt. Maybe try again * after deleting the explained peaks */ if ( flags & INDEXING_MULTI ) { /* Remove "used" spots and try for another lattice */ Crystal *cr; cr = image->crystals[image->n_crystals-1]; return delete_explained_peaks(image, cr); } else { return 1; } } } void index_pattern(struct image *image, IndexingPrivate *ipriv) { index_pattern_3(image, ipriv, NULL, NULL); } void index_pattern_2(struct image *image, IndexingPrivate *ipriv, int *ping) { index_pattern_3(image, ipriv, ping, NULL); } void index_pattern_3(struct image *image, IndexingPrivate *ipriv, int *ping, char *last_task) { int n = 0; ImageFeatureList *orig; if ( ipriv == NULL ) return; map_all_peaks(image); image->crystals = NULL; image->n_crystals = 0; orig = image->features; for ( n=0; nn_methods; n++ ) { int done = 0; int r; int ntry = 0; int success = 0; image->features = sort_peaks(orig); do { r = try_indexer(image, ipriv->methods[n], ipriv, ipriv->engine_private[n], last_task); success += r; ntry++; done = finished_retry(ipriv->methods[n], ipriv->flags, r, image); if ( ntry > 5 ) done = 1; if ( ping != NULL ) (*ping)++; } while ( !done ); image_feature_list_free(image->features); /* Stop now if the pattern is indexed (don't try again for more * crystals with a different indexing method) */ if ( success ) { image->n_indexing_tries = ntry; break; } } if ( n < ipriv->n_methods ) { image->indexed_by = ipriv->methods[n]; } else { image->indexed_by = INDEXING_NONE; } image->features = orig; } /* Set the indexer flags for "use no lattice type information" */ static IndexingMethod set_nolattice(IndexingMethod a) { return a & ~INDEXING_USE_LATTICE_TYPE; } /* Set the indexer flags for "use lattice type information" */ static IndexingMethod set_lattice(IndexingMethod a) { return a | INDEXING_USE_LATTICE_TYPE; } /* Set the indexer flags for "use no unit cell parameters" */ static IndexingMethod set_nocellparams(IndexingMethod a) { return a & ~INDEXING_USE_CELL_PARAMETERS; } /* Set the indexer flags for "use unit cell parameters" */ static IndexingMethod set_cellparams(IndexingMethod a) { return a | INDEXING_USE_CELL_PARAMETERS; } char *indexer_str(IndexingMethod indm) { char *str = base_indexer_str(indm); if ( (indm & INDEXING_METHOD_MASK) == INDEXING_SIMULATION ) return str; if ( (indm & INDEXING_METHOD_MASK) == INDEXING_NONE ) return str; if ( indm & INDEXING_USE_LATTICE_TYPE ) { strcat(str, "-latt"); } else { strcat(str, "-nolatt"); } if ( indm & INDEXING_USE_CELL_PARAMETERS ) { strcat(str, "-cell"); } else { strcat(str, "-nocell"); } return str; } static IndexingMethod warn_method(const char *str) { ERROR("Indexing method must contain exactly one engine name: '%s'\n", str); return INDEXING_ERROR; } IndexingMethod get_indm_from_string_2(const char *str, int *err) { int n, i; char **bits; IndexingMethod method = INDEXING_NONE; int have_method = 0; if ( err != NULL ) *err = 0; n = assplode(str, "-", &bits, ASSPLODE_NONE); for ( i=0; i