diff --git a/src/hyperion/device_setup_plans/read_hardware_for_setup.py b/src/hyperion/device_setup_plans/read_hardware_for_setup.py index 9c509e02e..ca9308394 100644 --- a/src/hyperion/device_setup_plans/read_hardware_for_setup.py +++ b/src/hyperion/device_setup_plans/read_hardware_for_setup.py @@ -24,7 +24,7 @@ def read_hardware_for_ispyb_pre_collection( ): LOGGER.info("Reading status of beamline for ispyb deposition, pre collection.") yield from bps.create( - name=CONST.PLAN.ISPYB_HARDWARE_READ + name=CONST.DESCRIPTORS.ISPYB_HARDWARE_READ ) # gives name to event *descriptor* document yield from bps.read(undulator.current_gap) yield from bps.read(synchrotron.synchrotron_mode) @@ -39,7 +39,7 @@ def read_hardware_for_ispyb_during_collection( attenuator: Attenuator, flux: Flux, dcm: DCM ): LOGGER.info("Reading status of beamline for ispyb deposition, during collection.") - yield from bps.create(name=CONST.PLAN.ISPYB_TRANSMISSION_FLUX_READ) + yield from bps.create(name=CONST.DESCRIPTORS.ISPYB_TRANSMISSION_FLUX_READ) yield from bps.read(attenuator.actual_transmission) yield from bps.read(flux.flux_reading) yield from bps.read(dcm.energy_in_kev) @@ -47,12 +47,12 @@ def read_hardware_for_ispyb_during_collection( def read_hardware_for_nexus_writer(detector: EigerDetector): - yield from bps.create(name=CONST.PLAN.NEXUS_READ) + yield from bps.create(name=CONST.DESCRIPTORS.NEXUS_READ) yield from bps.read(detector.bit_depth) yield from bps.save() def read_hardware_for_zocalo(detector: EigerDetector): - yield from bps.create(name=CONST.PLAN.ZOCALO_HW_READ) + yield from bps.create(name=CONST.DESCRIPTORS.ZOCALO_HW_READ) yield from bps.read(detector.odin.file_writer.id) yield from bps.save() diff --git a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py index 20e497d10..e24d57460 100755 --- a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py @@ -359,7 +359,6 @@ def flyscan_xray_centre( CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, "hyperion_internal_parameters": old_parameters.json(), "activate_callbacks": [ - "GridscanISPyBCallback", "GridscanNexusFileCallback", ], } diff --git a/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py b/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py index af24317b3..ed99c2636 100644 --- a/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py @@ -4,7 +4,6 @@ import json from typing import Any, Union -import numpy as np from blueapi.core import BlueskyContext, MsgGenerator from bluesky import plan_stubs as bps from bluesky import preprocessors as bpp @@ -49,8 +48,8 @@ from hyperion.external_interaction.callbacks.grid_detection_callback import ( GridDetectionCallback, ) -from hyperion.external_interaction.callbacks.oav_snapshot_callback import ( - OavSnapshotCallback, +from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( + ispyb_activation_wrapper, ) from hyperion.log import LOGGER from hyperion.parameters.gridscan import GridScanWithEdgeDetect @@ -134,6 +133,16 @@ def detect_grid_and_do_gridscan( composite: GridDetectThenXRayCentreComposite, parameters: GridScanWithEdgeDetectInternalParameters, oav_params: OAVParameters, +): + yield from ispyb_activation_wrapper( + _detect_grid_and_do_gridscan(composite, parameters, oav_params), parameters + ) + + +def _detect_grid_and_do_gridscan( + composite: GridDetectThenXRayCentreComposite, + parameters: GridScanWithEdgeDetectInternalParameters, + oav_params: OAVParameters, ): assert composite.aperture_scatterguard.aperture_positions is not None experiment_params: GridScanWithEdgeDetectParams = parameters.experiment_params @@ -143,7 +152,6 @@ def detect_grid_and_do_gridscan( f"{detector_params.prefix}_{detector_params.run_number}_{{angle}}" ) - oav_callback = OavSnapshotCallback() grid_params_callback = GridDetectionCallback( composite.oav.parameters, experiment_params.exposure_time, @@ -151,7 +159,7 @@ def detect_grid_and_do_gridscan( experiment_params.run_up_distance_mm, ) - @bpp.subs_decorator([oav_callback, grid_params_callback]) + @bpp.subs_decorator([grid_params_callback]) def run_grid_detection_plan( oav_params, snapshot_template, @@ -178,20 +186,6 @@ def run_grid_detection_plan( experiment_params.snapshot_dir, ) - # Hack because GDA only passes 3 values to ispyb - out_upper_left = np.array( - oav_callback.out_upper_left[0] + [oav_callback.out_upper_left[1][1]] - ) - - # Hack because the callback returns the list in inverted order - parameters.hyperion_params.ispyb_params.xtal_snapshots_omega_start = ( - oav_callback.snapshot_filenames[0][::-1] - ) - parameters.hyperion_params.ispyb_params.xtal_snapshots_omega_end = ( - oav_callback.snapshot_filenames[1][::-1] - ) - parameters.hyperion_params.ispyb_params.upper_left = out_upper_left - yield from bps.abs_set(composite.backlight, Backlight.OUT) LOGGER.info( diff --git a/src/hyperion/experiment_plans/oav_grid_detection_plan.py b/src/hyperion/experiment_plans/oav_grid_detection_plan.py index 43cb806cc..42e0747a4 100644 --- a/src/hyperion/experiment_plans/oav_grid_detection_plan.py +++ b/src/hyperion/experiment_plans/oav_grid_detection_plan.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Tuple import bluesky.plan_stubs as bps -import bluesky.preprocessors as bpp import numpy as np from blueapi.core import BlueskyContext from dodal.devices.backlight import Backlight @@ -57,7 +56,6 @@ def get_min_and_max_y_of_pin( return min_y, max_y -@bpp.run_decorator() def grid_detection_plan( composite: OavGridDetectionComposite, parameters: OAVParameters, @@ -159,7 +157,7 @@ def grid_detection_plan( yield from bps.abs_set(oav.snapshot.directory, snapshot_dir) yield from bps.trigger(oav.snapshot, wait=True) - yield from bps.create("snapshot_to_ispyb") + yield from bps.create(CONST.DESCRIPTORS.OAV_SNAPSHOT_TRIGGERED) yield from bps.read(oav.snapshot) yield from bps.read(smargon) diff --git a/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py index 2ac3080aa..9089a0429 100755 --- a/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py @@ -290,7 +290,6 @@ def panda_flyscan_xray_centre( CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, "hyperion_internal_parameters": parameters.json(), "activate_callbacks": [ - "GridscanISPyBCallback", "GridscanNexusFileCallback", ], } diff --git a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py index fb2c08c90..7fad38041 100644 --- a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py +++ b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py @@ -174,7 +174,7 @@ def robot_load(): yield from bps.wait("robot_load") - yield from bps.create(name=CONST.PLAN.ROBOT_LOAD) + yield from bps.create(name=CONST.DESCRIPTORS.ROBOT_LOAD) yield from bps.read(composite.robot.barcode) yield from bps.save() @@ -182,6 +182,8 @@ def robot_load(): yield from robot_load() + # XXX 1278 this effectively casts between unrelated types which doesn't have all + # attributes needed for downstream e.g. grid_width_microns params_json = json.loads(parameters.json()) pin_centre_params = PinCentreThenXrayCentreInternalParameters(**params_json) diff --git a/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py index 43b3835d0..73b9255a5 100644 --- a/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py @@ -1,11 +1,9 @@ from __future__ import annotations import re -from dataclasses import dataclass -from typing import Optional, Union +from typing import Optional from dodal.devices.detector import DetectorParams -from numpy import ndarray from hyperion.external_interaction.ispyb.data_model import ( DataCollectionGroupInfo, @@ -50,7 +48,7 @@ def populate_data_collection_position_info(ispyb_params): def populate_remaining_data_collection_info( - comment_constructor, + comment, data_collection_group_id, data_collection_info: DataCollectionInfo, detector_params, @@ -65,7 +63,7 @@ def populate_remaining_data_collection_info( data_collection_info.focal_spot_size_at_sampley = ispyb_params.focal_spot_size_y data_collection_info.beamsize_at_samplex = ispyb_params.beam_size_x data_collection_info.beamsize_at_sampley = ispyb_params.beam_size_y - data_collection_info.comments = comment_constructor() + data_collection_info.comments = comment data_collection_info.detector_distance = detector_params.detector_distance data_collection_info.exp_time = detector_params.exposure_time data_collection_info.imgdir = detector_params.directory @@ -123,10 +121,3 @@ def get_xtal_snapshots(ispyb_params): ISPYB_LOGGER.warning("No xtal snapshot paths sent to ISPyB!") xtal_snapshots = [] return xtal_snapshots + [None] * (3 - len(xtal_snapshots)) - - -@dataclass -class GridScanInfo: - upper_left: Union[list[int], ndarray] - y_steps: int - y_step_size: float diff --git a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py index a821cf03e..2e1e2a7a6 100644 --- a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -12,10 +12,16 @@ from hyperion.external_interaction.callbacks.plan_reactive_callback import ( PlanReactiveCallback, ) +from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( + construct_comment_for_gridscan, +) from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionGridInfo, + DataCollectionGroupInfo, DataCollectionInfo, ScanDataInfo, ) +from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, StoreInIspyb, @@ -48,7 +54,7 @@ def __init__( for self.ispyb_ids.""" ISPYB_LOGGER.debug("Initialising ISPyB callback") super().__init__(log=ISPYB_LOGGER, emit=emit) - self._event_driven_data_collection_info: Optional[DataCollectionInfo] = None + self._oav_snapshot_event_idx: int = 0 self._sample_barcode: Optional[str] = None self.params: GridscanInternalParameters | RotationInternalParameters | None = ( None @@ -70,7 +76,7 @@ def __init__( self.log = ISPYB_LOGGER def activity_gated_start(self, doc: RunStart): - self._event_driven_data_collection_info = DataCollectionInfo() + self._oav_snapshot_event_idx = 0 self._sample_barcode = None return self._tag_doc(doc) @@ -83,7 +89,7 @@ def activity_gated_event(self, doc: Event) -> Event: hyperion.log""" ISPYB_LOGGER.debug("ISPyB handler received event document.") assert self.ispyb is not None, "ISPyB deposition wasn't initialised!" - assert self.params is not None, "ISPyB handler didn't recieve parameters!" + assert self.params is not None, "ISPyB handler didn't receive parameters!" event_descriptor = self.descriptors.get(doc["descriptor"]) if event_descriptor is None: @@ -92,54 +98,118 @@ def activity_gated_event(self, doc: Event) -> Event: "has no corresponding descriptor record" ) return doc - if event_descriptor.get("name") == CONST.PLAN.ISPYB_HARDWARE_READ: - assert self._event_driven_data_collection_info - ISPYB_LOGGER.info("ISPyB handler received event from read hardware") - self._event_driven_data_collection_info.undulator_gap1 = doc["data"][ - "undulator_current_gap" - ] - assert isinstance( - synchrotron_mode := doc["data"]["synchrotron-synchrotron_mode"], - SynchrotronMode, - ) - self._event_driven_data_collection_info.synchrotron_mode = ( - synchrotron_mode.value - ) - self._event_driven_data_collection_info.slitgap_horizontal = doc["data"][ - "s4_slit_gaps_xgap" - ] - self._event_driven_data_collection_info.slitgap_vertical = doc["data"][ - "s4_slit_gaps_ygap" - ] - self._sample_barcode = doc["data"]["robot-barcode"] - - if event_descriptor.get("name") == CONST.PLAN.ISPYB_TRANSMISSION_FLUX_READ: - assert self._event_driven_data_collection_info - if transmission := doc["data"]["attenuator_actual_transmission"]: - # Ispyb wants the transmission in a percentage, we use fractions - self._event_driven_data_collection_info.transmission = ( - transmission * 100 + data_collection_group_info = None + match event_descriptor.get("name"): + case CONST.DESCRIPTORS.ISPYB_HARDWARE_READ: + data_collection_group_info, scan_data_infos = ( + self._handle_ispyb_hardware_read(doc) ) - self._event_driven_data_collection_info.flux = doc["data"][ - "flux_flux_reading" + case CONST.DESCRIPTORS.OAV_SNAPSHOT_TRIGGERED: + scan_data_infos = self._handle_oav_snapshot_triggered(doc) + case CONST.DESCRIPTORS.ISPYB_TRANSMISSION_FLUX_READ: + scan_data_infos = self._handle_ispyb_transmission_flux_read(doc) + case _: + return self._tag_doc(doc) + self.ispyb_ids = self.ispyb.update_deposition( + self.ispyb_ids, data_collection_group_info, scan_data_infos + ) + ISPYB_LOGGER.info(f"Recieved ISPYB IDs: {self.ispyb_ids}") + return self._tag_doc(doc) + + def _handle_ispyb_hardware_read( + self, doc + ) -> tuple[DataCollectionGroupInfo, Sequence[ScanDataInfo]]: + assert self.params, "Event handled before activity_gated_start received params" + ISPYB_LOGGER.info("ISPyB handler received event from read hardware") + assert isinstance( + synchrotron_mode := doc["data"]["synchrotron-synchrotron_mode"], + SynchrotronMode, + ) + hwscan_data_collection_info = DataCollectionInfo( + undulator_gap1=doc["data"]["undulator_current_gap"], + synchrotron_mode=synchrotron_mode.value, + slitgap_horizontal=doc["data"]["s4_slit_gaps_xgap"], + slitgap_vertical=doc["data"]["s4_slit_gaps_ygap"], + ) + self._sample_barcode = doc["data"]["robot-barcode"] + scan_data_infos = self.populate_info_for_update( + hwscan_data_collection_info, self.params + ) + ISPYB_LOGGER.info( + "Updating ispyb data collection and group after hardware read." + ) + data_collection_group_info = populate_data_collection_group( + self.ispyb.experiment_type, + self.params.hyperion_params.detector_params, + self.params.hyperion_params.ispyb_params, + self._sample_barcode, + ) + return data_collection_group_info, scan_data_infos + + def _handle_oav_snapshot_triggered(self, doc) -> Sequence[ScanDataInfo]: + assert self.ispyb_ids.data_collection_ids, "No current data collection" + assert self.params, "ISPyB handler didn't recieve parameters!" + data = doc["data"] + data_collection_id = None + data_collection_info = DataCollectionInfo( + xtal_snapshot1=data.get("oav_snapshot_last_path_full_overlay"), + xtal_snapshot2=data.get("oav_snapshot_last_path_outer"), + xtal_snapshot3=data.get("oav_snapshot_last_saved_path"), + n_images=( + data["oav_snapshot_num_boxes_x"] * data["oav_snapshot_num_boxes_y"] + ), + ) + data_collection_grid_info = DataCollectionGridInfo( + dx_in_mm=data["oav_snapshot_box_width"] + * self.params.hyperion_params.ispyb_params.microns_per_pixel_x + / 1000, + dy_in_mm=data["oav_snapshot_box_width"] + * self.params.hyperion_params.ispyb_params.microns_per_pixel_y + / 1000, + steps_x=data["oav_snapshot_num_boxes_x"], + steps_y=data["oav_snapshot_num_boxes_y"], + microns_per_pixel_x=self.params.hyperion_params.ispyb_params.microns_per_pixel_x, + microns_per_pixel_y=self.params.hyperion_params.ispyb_params.microns_per_pixel_y, + snapshot_offset_x_pixel=int(data["oav_snapshot_top_left_x"]), + snapshot_offset_y_pixel=int(data["oav_snapshot_top_left_y"]), + orientation=Orientation.HORIZONTAL, + snaked=True, + ) + data_collection_info.comments = construct_comment_for_gridscan( + self.params.hyperion_params.ispyb_params, data_collection_grid_info + ) + if len(self.ispyb_ids.data_collection_ids) > self._oav_snapshot_event_idx: + data_collection_id = self.ispyb_ids.data_collection_ids[ + self._oav_snapshot_event_idx ] - if doc["data"]["dcm_energy_in_kev"]: - energy_ev = doc["data"]["dcm_energy_in_kev"] * 1000 - self._event_driven_data_collection_info.wavelength = ( - convert_eV_to_angstrom(energy_ev) - ) - scan_data_infos = self.populate_info_for_update( - self._event_driven_data_collection_info, self.params - ) - ISPYB_LOGGER.info("Updating ispyb entry.") - self.ispyb_ids = self.update_deposition( - self.params, - scan_data_infos, - self._sample_barcode, - ) - ISPYB_LOGGER.info(f"Recieved ISPYB IDs: {self.ispyb_ids}") - return self._tag_doc(doc) + scan_data_info = ScanDataInfo( + data_collection_info=data_collection_info, + data_collection_id=data_collection_id, + data_collection_grid_info=data_collection_grid_info, + ) + ISPYB_LOGGER.info( + "Updating ispyb data collection and group after oav snapshot." + ) + self._oav_snapshot_event_idx += 1 + return [scan_data_info] + + def _handle_ispyb_transmission_flux_read(self, doc) -> Sequence[ScanDataInfo]: + assert self.params + hwscan_data_collection_info = DataCollectionInfo( + flux=doc["data"]["flux_flux_reading"] + ) + if transmission := doc["data"]["attenuator_actual_transmission"]: + # Ispyb wants the transmission in a percentage, we use fractions + hwscan_data_collection_info.transmission = transmission * 100 + if doc["data"]["dcm_energy_in_kev"]: + energy_ev = doc["data"]["dcm_energy_in_kev"] * 1000 + hwscan_data_collection_info.wavelength = convert_eV_to_angstrom(energy_ev) + scan_data_infos = self.populate_info_for_update( + hwscan_data_collection_info, self.params + ) + ISPYB_LOGGER.info("Updating ispyb data collection after flux read.") + return scan_data_infos def update_deposition( self, diff --git a/src/hyperion/external_interaction/callbacks/oav_snapshot_callback.py b/src/hyperion/external_interaction/callbacks/oav_snapshot_callback.py deleted file mode 100644 index 1786d17a3..000000000 --- a/src/hyperion/external_interaction/callbacks/oav_snapshot_callback.py +++ /dev/null @@ -1,24 +0,0 @@ -from bluesky.callbacks import CallbackBase - - -class OavSnapshotCallback(CallbackBase): - def __init__(self, *args) -> None: - super().__init__(*args) - self.snapshot_filenames: list = [] - self.out_upper_left: list = [] - - def event(self, doc): - data = doc.get("data") - - self.snapshot_filenames.append( - [ - data.get("oav_snapshot_last_saved_path"), - data.get("oav_snapshot_last_path_outer"), - data.get("oav_snapshot_last_path_full_overlay"), - ] - ) - - self.out_upper_left.append( - [data.get("oav_snapshot_top_left_x"), data.get("oav_snapshot_top_left_y")] - ) - return doc diff --git a/src/hyperion/external_interaction/callbacks/plan_reactive_callback.py b/src/hyperion/external_interaction/callbacks/plan_reactive_callback.py index 267757b68..77ef583dd 100644 --- a/src/hyperion/external_interaction/callbacks/plan_reactive_callback.py +++ b/src/hyperion/external_interaction/callbacks/plan_reactive_callback.py @@ -54,7 +54,7 @@ def _run_activity_gated(self, name: str, func, doc, override=False): def start(self, doc: RunStart) -> RunStart | None: callbacks_to_activate = doc.get("activate_callbacks") - if callbacks_to_activate: + if callbacks_to_activate and not self.active: activate = type(self).__name__ in callbacks_to_activate self.active = activate self.log.info( diff --git a/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py index 17c36dd0d..861cf3b92 100644 --- a/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py @@ -54,7 +54,10 @@ def activity_gated_descriptor(self, doc: EventDescriptor) -> EventDescriptor | N def activity_gated_event(self, doc: Event) -> Event | None: event_descriptor = self.descriptors.get(doc["descriptor"]) - if event_descriptor and event_descriptor.get("name") == CONST.PLAN.ROBOT_LOAD: + if ( + event_descriptor + and event_descriptor.get("name") == CONST.DESCRIPTORS.ROBOT_LOAD + ): assert ( self.action_id is not None ), "ISPyB Robot load callback event called unexpectedly" diff --git a/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py index 4988fc9c7..166558092 100644 --- a/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py @@ -1,7 +1,6 @@ from __future__ import annotations from collections.abc import Sequence -from dataclasses import asdict, replace from typing import TYPE_CHECKING, Any, Callable, cast from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( @@ -13,7 +12,6 @@ BaseISPyBCallback, ) from hyperion.external_interaction.callbacks.rotation.ispyb_mapping import ( - construct_comment_for_rotation_scan, populate_data_collection_info_for_rotation, ) from hyperion.external_interaction.ispyb.data_model import ( @@ -34,6 +32,8 @@ if TYPE_CHECKING: from event_model.documents import Event, RunStart, RunStop +COMMENT_FOR_ROTATION_SCAN = "Hyperion rotation scan" + class RotationISPyBCallback(BaseISPyBCallback): """Callback class to handle the deposition of experiment parameters into the ISPyB @@ -108,7 +108,7 @@ def activity_gated_start(self, doc: RunStart): self.params, ) data_collection_info = populate_remaining_data_collection_info( - construct_comment_for_rotation_scan, + COMMENT_FOR_ROTATION_SCAN, dcgid, data_collection_info, self.params.hyperion_params.detector_params, @@ -119,7 +119,7 @@ def activity_gated_start(self, doc: RunStart): data_collection_info=data_collection_info, ) self.ispyb_ids = self.ispyb.begin_deposition( - data_collection_group_info, scan_data_info + data_collection_group_info, [scan_data_info] ) ISPYB_LOGGER.info("ISPYB handler received start document.") if doc.get("subplan_name") == CONST.PLAN.ROTATION_MAIN: @@ -129,32 +129,17 @@ def activity_gated_start(self, doc: RunStart): def populate_info_for_update( self, event_sourced_data_collection_info: DataCollectionInfo, params ) -> Sequence[ScanDataInfo]: + assert ( + self.ispyb_ids.data_collection_ids + ), "Expect an existing DataCollection to update" params = cast(RotationInternalParameters, params) - initial_collection_info = populate_data_collection_info_for_rotation( - params.hyperion_params.ispyb_params, - params.hyperion_params.detector_params, - params, - ) - initial_collection_info = replace( - initial_collection_info, - **{ - k: v - for (k, v) in asdict(event_sourced_data_collection_info).items() - if v - }, - ) return [ ScanDataInfo( - data_collection_info=populate_remaining_data_collection_info( - construct_comment_for_rotation_scan, - self.ispyb_ids.data_collection_group_id, - initial_collection_info, - params.hyperion_params.detector_params, - params.hyperion_params.ispyb_params, - ), + data_collection_info=event_sourced_data_collection_info, data_collection_position_info=populate_data_collection_position_info( params.hyperion_params.ispyb_params ), + data_collection_id=self.ispyb_ids.data_collection_ids[0], ) ] diff --git a/src/hyperion/external_interaction/callbacks/rotation/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/rotation/ispyb_mapping.py index 735c843a1..78d70bf5f 100644 --- a/src/hyperion/external_interaction/callbacks/rotation/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/rotation/ispyb_mapping.py @@ -26,7 +26,3 @@ def populate_data_collection_info_for_rotation( info.xtal_snapshot3, ) = get_xtal_snapshots(ispyb_params) return info - - -def construct_comment_for_rotation_scan() -> str: - return "Hyperion rotation scan" diff --git a/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py b/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py index 391807876..de3a276b0 100644 --- a/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py +++ b/src/hyperion/external_interaction/callbacks/rotation/nexus_callback.py @@ -52,18 +52,22 @@ def activity_gated_event(self, doc: Event): "has no corresponding descriptor record" ) return doc - if event_descriptor.get("name") == CONST.PLAN.ISPYB_TRANSMISSION_FLUX_READ: + if ( + event_descriptor.get("name") + == CONST.DESCRIPTORS.ISPYB_TRANSMISSION_FLUX_READ + ): NEXUS_LOGGER.info(f"Nexus handler received event from read hardware {doc}") data = doc["data"] assert self.writer, "Nexus writer not initialised" - self.writer.beam, self.writer.attenuator = ( - create_beam_and_attenuator_parameters( - data["dcm_energy_in_kev"], - data["flux_flux_reading"], - data["attenuator_actual_transmission"], - ) + ( + self.writer.beam, + self.writer.attenuator, + ) = create_beam_and_attenuator_parameters( + data["dcm_energy_in_kev"], + data["flux_flux_reading"], + data["attenuator_actual_transmission"], ) - if event_descriptor.get("name") == CONST.PLAN.NEXUS_READ: + if event_descriptor.get("name") == CONST.DESCRIPTORS.NEXUS_READ: NEXUS_LOGGER.info(f"Nexus handler received event from read hardware {doc}") vds_data_type = vds_type_based_on_bit_depth(doc["data"]["eiger_bit_depth"]) assert self.writer is not None diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py index 764e7830f..98b8daa0c 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py @@ -1,15 +1,14 @@ from __future__ import annotations from collections.abc import Sequence -from dataclasses import asdict, replace from time import time -from typing import TYPE_CHECKING, Any, Callable, List, Optional, cast +from typing import TYPE_CHECKING, Any, Callable, List, cast import numpy as np +from bluesky import preprocessors as bpp from dodal.devices.zocalo.zocalo_results import ZOCALO_READING_PLAN_NAME from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( - GridScanInfo, populate_data_collection_group, populate_data_collection_position_info, populate_remaining_data_collection_info, @@ -18,8 +17,6 @@ BaseISPyBCallback, ) from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( - construct_comment_for_gridscan, - populate_data_collection_grid_info, populate_xy_data_collection_info, populate_xz_data_collection_info, ) @@ -35,6 +32,7 @@ ) from hyperion.log import ISPYB_LOGGER, set_dcgid_tag from hyperion.parameters.constants import CONST +from hyperion.parameters.gridscan import ThreeDGridScan from hyperion.parameters.plan_specific.gridscan_internal_params import ( GridscanInternalParameters, ) @@ -43,6 +41,21 @@ from event_model import Event, RunStart, RunStop +def ispyb_activation_wrapper(plan_generator, parameters): + return bpp.run_wrapper( + plan_generator, + md={ + "activate_callbacks": ["GridscanISPyBCallback"], + "subplan_name": CONST.PLAN.GRID_DETECT_AND_DO_GRIDSCAN, + "hyperion_internal_parameters": ( + parameters.old_parameters() + if isinstance(parameters, ThreeDGridScan) + else parameters + ).json(), + }, + ) + + class GridscanISPyBCallback(BaseISPyBCallback): """Callback class to handle the deposition of experiment parameters into the ISPyB database. Listens for 'event' and 'descriptor' documents. Creates the ISpyB entry on @@ -76,7 +89,7 @@ def is_3d_gridscan(self): def activity_gated_start(self, doc: RunStart): if doc.get("subplan_name") == CONST.PLAN.DO_FGS: self._start_of_fgs_uid = doc.get("uid") - if doc.get("subplan_name") == CONST.PLAN.GRIDSCAN_OUTER: + if doc.get("subplan_name") == CONST.PLAN.GRID_DETECT_AND_DO_GRIDSCAN: self.uid_to_finalize_on = doc.get("uid") ISPYB_LOGGER.info( "ISPyB callback recieved start document with experiment parameters and " @@ -97,35 +110,37 @@ def activity_gated_start(self, doc: RunStart): self.params.hyperion_params.detector_params, self.params.hyperion_params.ispyb_params, ) - grid_scan_info = GridScanInfo( - self.params.hyperion_params.ispyb_params.upper_left, - self.params.experiment_params.y_steps, - self.params.experiment_params.y_step_size, - ) - def constructor(): - return construct_comment_for_gridscan( - self.params, - self.params.hyperion_params.ispyb_params, - grid_scan_info, - ) - - scan_data_info = ScanDataInfo( - data_collection_info=populate_remaining_data_collection_info( - constructor, - None, - populate_xy_data_collection_info( - grid_scan_info, - self.params, - self.params.hyperion_params.ispyb_params, + scan_data_infos = [ + ScanDataInfo( + data_collection_info=populate_remaining_data_collection_info( + None, + None, + populate_xy_data_collection_info( + self.params.hyperion_params.detector_params, + ), self.params.hyperion_params.detector_params, + self.params.hyperion_params.ispyb_params, ), - self.params.hyperion_params.detector_params, - self.params.hyperion_params.ispyb_params, - ), - ) + ) + ] + if self.is_3d_gridscan(): + scan_data_infos.append( + ScanDataInfo( + data_collection_info=populate_remaining_data_collection_info( + None, + None, + populate_xz_data_collection_info( + self.params.hyperion_params.detector_params + ), + self.params.hyperion_params.detector_params, + self.params.hyperion_params.ispyb_params, + ) + ) + ) + self.ispyb_ids = self.ispyb.begin_deposition( - data_collection_group_info, scan_data_info + data_collection_group_info, scan_data_infos ) set_dcgid_tag(self.ispyb_ids.data_collection_group_id) return super().activity_gated_start(doc) @@ -173,117 +188,35 @@ def populate_info_for_update( self, event_sourced_data_collection_info: DataCollectionInfo, params ) -> Sequence[ScanDataInfo]: params = cast(GridscanInternalParameters, params) - scan_data_infos = [ - self.populate_xy_scan_data_info(params, event_sourced_data_collection_info) - ] - if self.is_3d_gridscan(): - scan_data_infos.append( - self.populate_xz_scan_data_info( - params, event_sourced_data_collection_info - ) - ) - return scan_data_infos - - def populate_xy_scan_data_info( - self, params, event_sourced_data_collection_info: DataCollectionInfo - ): - grid_scan_info = GridScanInfo( - [ - int(params.hyperion_params.ispyb_params.upper_left[0]), - int(params.hyperion_params.ispyb_params.upper_left[1]), - ], - params.experiment_params.y_steps, - params.experiment_params.y_step_size, - ) - - xy_data_collection_info = populate_xy_data_collection_info( - grid_scan_info, - params, - params.hyperion_params.ispyb_params, - params.hyperion_params.detector_params, - ) - - xy_data_collection_info = replace( - xy_data_collection_info, - **{ - k: v - for (k, v) in asdict(event_sourced_data_collection_info).items() - if v - }, - ) - - def comment_constructor(): - return construct_comment_for_gridscan( - params, params.hyperion_params.ispyb_params, grid_scan_info - ) - - xy_data_collection_info = populate_remaining_data_collection_info( - comment_constructor, - self.ispyb_ids.data_collection_group_id, - xy_data_collection_info, - params.hyperion_params.detector_params, - params.hyperion_params.ispyb_params, - ) - - return ScanDataInfo( - data_collection_info=xy_data_collection_info, - data_collection_grid_info=populate_data_collection_grid_info( - params, grid_scan_info, params.hyperion_params.ispyb_params - ), + assert ( + self.ispyb_ids.data_collection_ids + ), "Expect at least one valid data collection to record scan data" + xy_scan_data_info = ScanDataInfo( + data_collection_info=event_sourced_data_collection_info, data_collection_position_info=populate_data_collection_position_info( params.hyperion_params.ispyb_params ), + data_collection_id=self.ispyb_ids.data_collection_ids[0], ) + scan_data_infos = [xy_scan_data_info] - def populate_xz_scan_data_info( - self, params, event_sourced_data_collection_info: DataCollectionInfo - ): - xz_grid_scan_info = GridScanInfo( - [ - int(params.hyperion_params.ispyb_params.upper_left[0]), - int(params.hyperion_params.ispyb_params.upper_left[2]), - ], - params.experiment_params.z_steps, - params.experiment_params.z_step_size, - ) - xz_data_collection_info = populate_xz_data_collection_info( - xz_grid_scan_info, - params, - params.hyperion_params.ispyb_params, - params.hyperion_params.detector_params, - ) - xz_data_collection_info = replace( - xz_data_collection_info, - **{ - k: v - for (k, v) in asdict(event_sourced_data_collection_info).items() - if v - }, - ) - - def xz_comment_constructor(): - return construct_comment_for_gridscan( - params, params.hyperion_params.ispyb_params, xz_grid_scan_info + if self.is_3d_gridscan(): + data_collection_id = ( + self.ispyb_ids.data_collection_ids[1] + if len(self.ispyb_ids.data_collection_ids) > 1 + else None ) + xz_scan_data_info = ScanDataInfo( + data_collection_info=event_sourced_data_collection_info, + data_collection_position_info=populate_data_collection_position_info( + params.hyperion_params.ispyb_params + ), + data_collection_id=data_collection_id, + ) + scan_data_infos.append(xz_scan_data_info) + return scan_data_infos - xz_data_collection_info = populate_remaining_data_collection_info( - xz_comment_constructor, - self.ispyb_ids.data_collection_group_id, - xz_data_collection_info, - params.hyperion_params.detector_params, - params.hyperion_params.ispyb_params, - ) - return ScanDataInfo( - data_collection_info=xz_data_collection_info, - data_collection_grid_info=populate_data_collection_grid_info( - params, xz_grid_scan_info, params.hyperion_params.ispyb_params - ), - data_collection_position_info=populate_data_collection_position_info( - params.hyperion_params.ispyb_params - ), - ) - - def activity_gated_stop(self, doc: RunStop) -> Optional[RunStop]: + def activity_gated_stop(self, doc: RunStop) -> RunStop: if doc.get("run_start") == self._start_of_fgs_uid: self._processing_start_time = time() if doc.get("run_start") == self.uid_to_finalize_on: diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py index 89157fad3..5700d20fc 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py @@ -1,101 +1,64 @@ from __future__ import annotations +import numpy from dodal.devices.oav import utils as oav_utils -from hyperion.external_interaction.callbacks.common.ispyb_mapping import GridScanInfo from hyperion.external_interaction.ispyb.data_model import ( DataCollectionGridInfo, DataCollectionInfo, ) -from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation -def populate_xz_data_collection_info( - grid_scan_info: GridScanInfo, - full_params, - ispyb_params, - detector_params, -) -> DataCollectionInfo: +def populate_xz_data_collection_info(detector_params) -> DataCollectionInfo: assert ( detector_params.omega_start is not None and detector_params.run_number is not None - and ispyb_params is not None - and full_params is not None ), "StoreGridscanInIspyb failed to get parameters" omega_start = detector_params.omega_start + 90 run_number = detector_params.run_number + 1 - xtal_snapshots = ispyb_params.xtal_snapshots_omega_end or [] info = DataCollectionInfo( omega_start=omega_start, data_collection_number=run_number, - n_images=full_params.experiment_params.x_steps * grid_scan_info.y_steps, axis_range=0, axis_end=omega_start, ) - info.xtal_snapshot1, info.xtal_snapshot2, info.xtal_snapshot3 = xtal_snapshots + [ - None - ] * (3 - len(xtal_snapshots)) return info -def populate_xy_data_collection_info( - grid_scan_info: GridScanInfo, full_params, ispyb_params, detector_params -): +def populate_xy_data_collection_info(detector_params): info = DataCollectionInfo( omega_start=detector_params.omega_start, data_collection_number=detector_params.run_number, - n_images=full_params.experiment_params.x_steps * grid_scan_info.y_steps, axis_range=0, axis_end=detector_params.omega_start, ) - snapshots = ispyb_params.xtal_snapshots_omega_start or [] - info.xtal_snapshot1, info.xtal_snapshot2, info.xtal_snapshot3 = snapshots + [ - None - ] * (3 - len(snapshots)) return info -def construct_comment_for_gridscan(full_params, ispyb_params, grid_scan_info) -> str: +def construct_comment_for_gridscan( + ispyb_params, grid_info: DataCollectionGridInfo +) -> str: assert ( - ispyb_params is not None - and full_params is not None - and grid_scan_info is not None + ispyb_params is not None and grid_info is not None ), "StoreGridScanInIspyb failed to get parameters" bottom_right = oav_utils.bottom_right_from_top_left( - grid_scan_info.upper_left, # type: ignore - full_params.experiment_params.x_steps, - grid_scan_info.y_steps, - full_params.experiment_params.x_step_size, - grid_scan_info.y_step_size, + numpy.array( + [grid_info.snapshot_offset_x_pixel, grid_info.snapshot_offset_y_pixel] + ), # type: ignore + grid_info.steps_x, + grid_info.steps_y, + grid_info.dx_in_mm, + grid_info.dy_in_mm, ispyb_params.microns_per_pixel_x, ispyb_params.microns_per_pixel_y, ) return ( "Hyperion: Xray centring - Diffraction grid scan of " - f"{full_params.experiment_params.x_steps} by " - f"{grid_scan_info.y_steps} images in " - f"{(full_params.experiment_params.x_step_size * 1e3):.1f} um by " - f"{(grid_scan_info.y_step_size * 1e3):.1f} um steps. " - f"Top left (px): [{int(grid_scan_info.upper_left[0])},{int(grid_scan_info.upper_left[1])}], " + f"{grid_info.steps_x} by " + f"{grid_info.steps_y} images in " + f"{(grid_info.dx_in_mm * 1e3):.1f} um by " + f"{(grid_info.dy_in_mm * 1e3):.1f} um steps. " + f"Top left (px): [{int(grid_info.snapshot_offset_x_pixel)},{int(grid_info.snapshot_offset_y_pixel)}], " f"bottom right (px): [{bottom_right[0]},{bottom_right[1]}]." ) - - -def populate_data_collection_grid_info(full_params, grid_scan_info, ispyb_params): - assert ispyb_params is not None - assert full_params is not None - dc_grid_info = DataCollectionGridInfo( - dx_in_mm=full_params.experiment_params.x_step_size, - dy_in_mm=grid_scan_info.y_step_size, - steps_x=full_params.experiment_params.x_steps, - steps_y=grid_scan_info.y_steps, - microns_per_pixel_x=ispyb_params.microns_per_pixel_x, - # cast coordinates from numpy int64 to avoid mysql type conversion issues - snapshot_offset_x_pixel=int(grid_scan_info.upper_left[0]), - snapshot_offset_y_pixel=int(grid_scan_info.upper_left[1]), - microns_per_pixel_y=ispyb_params.microns_per_pixel_y, - orientation=Orientation.HORIZONTAL, - snaked=True, - ) - return dc_grid_info diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py b/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py index 485f15cf9..b1d6a030d 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/nexus_callback.py @@ -67,18 +67,22 @@ def activity_gated_descriptor(self, doc: EventDescriptor): def activity_gated_event(self, doc: Event) -> Event | None: assert (event_descriptor := self.descriptors.get(doc["descriptor"])) is not None - if event_descriptor.get("name") == CONST.PLAN.ISPYB_TRANSMISSION_FLUX_READ: + if ( + event_descriptor.get("name") + == CONST.DESCRIPTORS.ISPYB_TRANSMISSION_FLUX_READ + ): data = doc["data"] for nexus_writer in [self.nexus_writer_1, self.nexus_writer_2]: assert nexus_writer, "Nexus callback did not receive start doc" - nexus_writer.beam, nexus_writer.attenuator = ( - create_beam_and_attenuator_parameters( - data["dcm_energy_in_kev"], - data["flux_flux_reading"], - data["attenuator_actual_transmission"], - ) + ( + nexus_writer.beam, + nexus_writer.attenuator, + ) = create_beam_and_attenuator_parameters( + data["dcm_energy_in_kev"], + data["flux_flux_reading"], + data["attenuator_actual_transmission"], ) - if event_descriptor.get("name") == CONST.PLAN.NEXUS_READ: + if event_descriptor.get("name") == CONST.DESCRIPTORS.NEXUS_READ: NEXUS_LOGGER.info(f"Nexus handler received event from read hardware {doc}") for nexus_writer in [self.nexus_writer_1, self.nexus_writer_2]: vds_data_type = vds_type_based_on_bit_depth( diff --git a/src/hyperion/external_interaction/callbacks/zocalo_callback.py b/src/hyperion/external_interaction/callbacks/zocalo_callback.py index a437c4b79..26cf61f7a 100644 --- a/src/hyperion/external_interaction/callbacks/zocalo_callback.py +++ b/src/hyperion/external_interaction/callbacks/zocalo_callback.py @@ -72,7 +72,7 @@ def descriptor(self, doc: EventDescriptor): def event(self, doc: Event) -> Event: event_descriptor = self.descriptors[doc["descriptor"]] - if event_descriptor.get("name") == CONST.PLAN.ZOCALO_HW_READ: + if event_descriptor.get("name") == CONST.DESCRIPTORS.ZOCALO_HW_READ: filename = doc["data"]["eiger_odin_file_writer_id"] for start_info in self.zocalo_info: start_info.filename = filename diff --git a/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py b/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py index eea0cd5e7..6e5dc1073 100644 --- a/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py @@ -9,8 +9,6 @@ "visit_path": "", "microns_per_pixel_x": 0.0, "microns_per_pixel_y": 0.0, - # gets stored as 2x2D coords - (x, y) and (x, z). Values in pixels - "upper_left": [0, 0, 0], "position": [0, 0, 0], "xtal_snapshots_omega_start": ["test_1_y", "test_2_y", "test_3_y"], "xtal_snapshots_omega_end": ["test_1_z", "test_2_z", "test_3_z"], @@ -73,22 +71,10 @@ class RotationIspybParams(IspybParams): ... class GridscanIspybParams(IspybParams): - upper_left: np.ndarray - def dict(self, **kwargs): as_dict = super().dict(**kwargs) - as_dict["upper_left"] = as_dict["upper_left"].tolist() return as_dict - @validator("upper_left", pre=True) - def _parse_upper_left( - cls, upper_left: list[int | float] | np.ndarray, values: Dict[str, Any] - ) -> np.ndarray: - assert len(upper_left) == 3 - if isinstance(upper_left, np.ndarray): - return upper_left - return np.array(upper_left) - class Orientation(Enum): HORIZONTAL = "horizontal" diff --git a/src/hyperion/external_interaction/ispyb/ispyb_store.py b/src/hyperion/external_interaction/ispyb/ispyb_store.py index 64dfca6b0..a7d283443 100755 --- a/src/hyperion/external_interaction/ispyb/ispyb_store.py +++ b/src/hyperion/external_interaction/ispyb/ispyb_store.py @@ -2,7 +2,6 @@ from abc import ABC from dataclasses import asdict -from itertools import zip_longest from typing import TYPE_CHECKING, Optional, Sequence, Tuple import ispyb @@ -13,6 +12,7 @@ from pydantic import BaseModel from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionGridInfo, DataCollectionGroupInfo, DataCollectionInfo, ExperimentType, @@ -51,24 +51,24 @@ def experiment_type(self) -> str: def begin_deposition( self, data_collection_group_info: DataCollectionGroupInfo, - scan_data_info: ScanDataInfo, + scan_data_infos: Sequence[ScanDataInfo], ) -> IspybIds: ispyb_ids = IspybIds() - if scan_data_info.data_collection_info: - ispyb_ids.data_collection_group_id = ( - scan_data_info.data_collection_info.parent_id - ) + if scan_data_infos[0].data_collection_info: + ispyb_ids.data_collection_group_id = scan_data_infos[ + 0 + ].data_collection_info.parent_id return self._begin_or_update_deposition( - ispyb_ids, data_collection_group_info, [scan_data_info] + ispyb_ids, data_collection_group_info, scan_data_infos ) def update_deposition( self, ispyb_ids, - data_collection_group_info: DataCollectionGroupInfo, + data_collection_group_info: Optional[DataCollectionGroupInfo], scan_data_infos: Sequence[ScanDataInfo], - ): + ) -> IspybIds: assert ( ispyb_ids.data_collection_group_id ), "Attempted to store scan data without a collection group" @@ -80,22 +80,30 @@ def update_deposition( ) def _begin_or_update_deposition( - self, ispyb_ids, data_collection_group_info, scan_data_infos - ): + self, + ispyb_ids, + data_collection_group_info: Optional[DataCollectionGroupInfo], + scan_data_infos, + ) -> IspybIds: with ispyb.open(self.ISPYB_CONFIG_PATH) as conn: assert conn is not None, "Failed to connect to ISPyB" - - ispyb_ids.data_collection_group_id = ( - self._store_data_collection_group_table( - conn, data_collection_group_info, ispyb_ids.data_collection_group_id + if data_collection_group_info: + ispyb_ids.data_collection_group_id = ( + self._store_data_collection_group_table( + conn, + data_collection_group_info, + ispyb_ids.data_collection_group_id, + ) ) - ) + else: + assert ( + ispyb_ids.data_collection_group_id + ), "Attempt to update data collection without a data collection group ID" grid_ids = [] - data_collection_ids_out = [] - for scan_data_info, data_collection_id in zip_longest( - scan_data_infos, ispyb_ids.data_collection_ids - ): + data_collection_ids_out = list(ispyb_ids.data_collection_ids) + for scan_data_info in scan_data_infos: + data_collection_id = scan_data_info.data_collection_id if ( scan_data_info.data_collection_info and not scan_data_info.data_collection_info.parent_id @@ -104,10 +112,11 @@ def _begin_or_update_deposition( ispyb_ids.data_collection_group_id ) - data_collection_id, grid_id = self._store_single_scan_data( + new_data_collection_id, grid_id = self._store_single_scan_data( conn, scan_data_info, data_collection_id ) - data_collection_ids_out.append(data_collection_id) + if not data_collection_id: + data_collection_ids_out.append(new_data_collection_id) if grid_id: grid_ids.append(grid_id) ispyb_ids = IspybIds( @@ -235,7 +244,10 @@ def _store_single_scan_data( return data_collection_id, grid_id def _store_grid_info_table( - self, conn: Connector, ispyb_data_collection_id: int, dc_grid_info + self, + conn: Connector, + ispyb_data_collection_id: int, + dc_grid_info: DataCollectionGridInfo, ) -> int: mx_acquisition: MXAcquisition = conn.mx_acquisition params = mx_acquisition.get_dc_grid_params() @@ -251,10 +263,11 @@ def _fill_common_data_collection_params( if data_collection_id: params["id"] = data_collection_id - assert data_collection_info.visit_string - params["visit_id"] = get_session_id_from_visit( - conn, data_collection_info.visit_string - ) + if data_collection_info.visit_string: + # This is only needed for populating the DataCollectionGroup + params["visit_id"] = get_session_id_from_visit( + conn, data_collection_info.visit_string + ) params |= { k: v for k, v in asdict(data_collection_info).items() if k != "visit_string" } diff --git a/src/hyperion/parameters/constants.py b/src/hyperion/parameters/constants.py index 6f4a3ac58..0fed12648 100644 --- a/src/hyperion/parameters/constants.py +++ b/src/hyperion/parameters/constants.py @@ -19,14 +19,11 @@ class SimConstants: @dataclass(frozen=True) class PlanNameConstants: - # For callbacks to use - NEXUS_READ = "nexus_read_plan" - ISPYB_HARDWARE_READ = "ispyb_reading_hardware" - ISPYB_TRANSMISSION_FLUX_READ = "ispyb_update_transmission_flux" - ZOCALO_HW_READ = "zocalo_read_hardware_plan" - # Robot load + # Robot load subplan ROBOT_LOAD = "robot_load" # Gridscan + GRID_DETECT_AND_DO_GRIDSCAN = "grid_detect_and_do_gridscan" + GRID_DETECT_INNER = "grid_detect" GRIDSCAN_OUTER = "run_gridscan_move_and_tidy" GRIDSCAN_AND_MOVE = "run_gridscan_and_move" GRIDSCAN_MAIN = "run_gridscan" @@ -36,6 +33,18 @@ class PlanNameConstants: ROTATION_MAIN = "rotation_scan_main" +@dataclass(frozen=True) +class DocDescriptorNames: + # Robot load event descriptor + ROBOT_LOAD = "robot_load" + # For callbacks to use + OAV_SNAPSHOT_TRIGGERED = "snapshot_to_ispyb" + NEXUS_READ = "nexus_read_plan" + ISPYB_HARDWARE_READ = "ispyb_reading_hardware" + ISPYB_TRANSMISSION_FLUX_READ = "ispyb_update_transmission_flux" + ZOCALO_HW_READ = "zocalo_read_hardware_plan" + + @dataclass(frozen=True) class HardwareConstants: OAV_REFRESH_DELAY = 0.3 @@ -91,6 +100,7 @@ class HyperionConstants: PARAM = ExperimentParamConstants() I03 = I03Constants() CALLBACK_0MQ_PROXY_PORTS = (5577, 5578) + DESCRIPTORS = DocDescriptorNames() PARAMETER_SCHEMA_DIRECTORY = "src/hyperion/parameters/schemas/" ZOCALO_ENV = "dev_artemis" if TEST_MODE else "artemis" diff --git a/src/hyperion/parameters/plan_specific/gridscan_internal_params.py b/src/hyperion/parameters/plan_specific/gridscan_internal_params.py index 3625d8b3f..22ffcff7f 100644 --- a/src/hyperion/parameters/plan_specific/gridscan_internal_params.py +++ b/src/hyperion/parameters/plan_specific/gridscan_internal_params.py @@ -85,7 +85,6 @@ def _preprocess_hyperion_params( all_params["num_triggers"] = all_params["num_images"] all_params["num_images_per_trigger"] = 1 all_params["trigger_mode"] = TriggerMode.FREE_RUN - all_params["upper_left"] = np.array(all_params["upper_left"]) hyperion_param_dict = extract_hyperion_params_from_flat_dict( all_params, cls._hyperion_param_key_definitions() ) diff --git a/src/hyperion/parameters/plan_specific/panda/panda_gridscan_internal_params.py b/src/hyperion/parameters/plan_specific/panda/panda_gridscan_internal_params.py index 212db5299..b75eed7cc 100644 --- a/src/hyperion/parameters/plan_specific/panda/panda_gridscan_internal_params.py +++ b/src/hyperion/parameters/plan_specific/panda/panda_gridscan_internal_params.py @@ -76,7 +76,6 @@ def _preprocess_hyperion_params( all_params["num_triggers"] = all_params["num_images"] all_params["num_images_per_trigger"] = 1 all_params["trigger_mode"] = TriggerMode.FREE_RUN - all_params["upper_left"] = np.array(all_params["upper_left"]) hyperion_param_dict = extract_hyperion_params_from_flat_dict( all_params, cls._hyperion_param_key_definitions() ) diff --git a/src/hyperion/parameters/schemas/ispyb_parameters_schema.json b/src/hyperion/parameters/schemas/ispyb_parameters_schema.json new file mode 100644 index 000000000..e69de29bb diff --git a/tests/system_tests/external_interaction/conftest.py b/tests/system_tests/external_interaction/conftest.py index 758705156..efff3bd56 100644 --- a/tests/system_tests/external_interaction/conftest.py +++ b/tests/system_tests/external_interaction/conftest.py @@ -4,7 +4,6 @@ import dodal.devices.zocalo.zocalo_interaction import ispyb.sqlalchemy -import numpy as np import pytest from ispyb.sqlalchemy import DataCollection, DataCollectionGroup from sqlalchemy import create_engine @@ -124,7 +123,6 @@ def dummy_params(): "tests/test_data/parameter_json_files/system_test_parameter_defaults.json" ) ) - dummy_params.hyperion_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.hyperion_params.ispyb_params.visit_path = ( "/dls/i03/data/2022/cm31105-5/" ) diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index ce010900a..65a315220 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -12,13 +12,13 @@ from dodal.devices.s4_slit_gaps import S4SlitGaps from dodal.devices.synchrotron import Synchrotron, SynchrotronMode from dodal.devices.undulator import Undulator +from ophyd_async.core import set_sim_value from hyperion.experiment_plans.rotation_scan_plan import ( RotationScanComposite, rotation_scan, ) from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( - GridScanInfo, populate_data_collection_group, populate_data_collection_position_info, populate_remaining_data_collection_info, @@ -28,14 +28,15 @@ ) from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( construct_comment_for_gridscan, - populate_data_collection_grid_info, populate_xy_data_collection_info, populate_xz_data_collection_info, ) from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionGridInfo, ExperimentType, ScanDataInfo, ) +from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, StoreInIspyb, @@ -60,21 +61,11 @@ def dummy_data_collection_group_info(dummy_params): @pytest.fixture def dummy_scan_data_info_for_begin(dummy_params): - grid_scan_info = GridScanInfo( - dummy_params.hyperion_params.ispyb_params.upper_left, - dummy_params.experiment_params.y_steps, - dummy_params.experiment_params.y_step_size, - ) info = populate_xy_data_collection_info( - grid_scan_info, - dummy_params, - dummy_params.hyperion_params.ispyb_params, dummy_params.hyperion_params.detector_params, ) info = populate_remaining_data_collection_info( - lambda: construct_comment_for_gridscan( - dummy_params, dummy_params.hyperion_params.ispyb_params, grid_scan_info - ), + None, None, info, dummy_params.hyperion_params.detector_params, @@ -89,15 +80,26 @@ def scan_xy_data_info_for_update( data_collection_group_id, dummy_params, scan_data_info_for_begin ): scan_data_info_for_update = deepcopy(scan_data_info_for_begin) - grid_scan_info = GridScanInfo( - dummy_params.hyperion_params.ispyb_params.upper_left, - dummy_params.experiment_params.y_steps, - dummy_params.experiment_params.y_step_size, - ) scan_data_info_for_update.data_collection_info.parent_id = data_collection_group_id - scan_data_info_for_update.data_collection_grid_info = ( - populate_data_collection_grid_info( - dummy_params, grid_scan_info, dummy_params.hyperion_params.ispyb_params + assert dummy_params.hyperion_params.ispyb_params is not None + assert dummy_params is not None + scan_data_info_for_update.data_collection_grid_info = DataCollectionGridInfo( + dx_in_mm=dummy_params.experiment_params.x_step_size, + dy_in_mm=dummy_params.experiment_params.y_step_size, + steps_x=dummy_params.experiment_params.x_steps, + steps_y=dummy_params.experiment_params.y_steps, + microns_per_pixel_x=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x, + microns_per_pixel_y=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y, + # cast coordinates from numpy int64 to avoid mysql type conversion issues + snapshot_offset_x_pixel=100, + snapshot_offset_y_pixel=100, + orientation=Orientation.HORIZONTAL, + snaked=True, + ) + scan_data_info_for_update.data_collection_info.comments = ( + construct_comment_for_gridscan( + dummy_params.hyperion_params.ispyb_params, + scan_data_info_for_update.data_collection_grid_info, ) ) scan_data_info_for_update.data_collection_position_info = ( @@ -111,26 +113,29 @@ def scan_xy_data_info_for_update( def scan_data_infos_for_update_3d( ispyb_ids, scan_xy_data_info_for_update, dummy_params ): - upper_left = dummy_params.hyperion_params.ispyb_params.upper_left - xz_grid_scan_info = GridScanInfo( - [upper_left[0], upper_left[2]], - dummy_params.experiment_params.z_steps, - dummy_params.experiment_params.z_step_size, - ) xz_data_collection_info = populate_xz_data_collection_info( - xz_grid_scan_info, - dummy_params, - dummy_params.hyperion_params.ispyb_params, - dummy_params.hyperion_params.detector_params, + dummy_params.hyperion_params.detector_params + ) + + assert dummy_params.hyperion_params.ispyb_params is not None + assert dummy_params is not None + data_collection_grid_info = DataCollectionGridInfo( + dx_in_mm=dummy_params.experiment_params.x_step_size, + dy_in_mm=dummy_params.experiment_params.z_step_size, + steps_x=dummy_params.experiment_params.x_steps, + steps_y=dummy_params.experiment_params.z_steps, + microns_per_pixel_x=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x, + microns_per_pixel_y=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y, + # cast coordinates from numpy int64 to avoid mysql type conversion issues + snapshot_offset_x_pixel=100, + snapshot_offset_y_pixel=50, + orientation=Orientation.HORIZONTAL, + snaked=True, ) - - def comment_constructor(): - return construct_comment_for_gridscan( - dummy_params, dummy_params.hyperion_params.ispyb_params, xz_grid_scan_info - ) - xz_data_collection_info = populate_remaining_data_collection_info( - comment_constructor, + construct_comment_for_gridscan( + dummy_params.hyperion_params.ispyb_params, data_collection_grid_info + ), ispyb_ids.data_collection_group_id, xz_data_collection_info, dummy_params.hyperion_params.detector_params, @@ -140,13 +145,7 @@ def comment_constructor(): scan_xz_data_info_for_update = ScanDataInfo( data_collection_info=xz_data_collection_info, - data_collection_grid_info=( - populate_data_collection_grid_info( - dummy_params, - xz_grid_scan_info, - dummy_params.hyperion_params.ispyb_params, - ) - ), + data_collection_grid_info=(data_collection_grid_info), data_collection_position_info=( populate_data_collection_position_info( dummy_params.hyperion_params.ispyb_params @@ -178,12 +177,12 @@ def test_ispyb_deposition_comment_correct_on_failure( dummy_scan_data_info_for_begin, ): ispyb_ids = dummy_ispyb.begin_deposition( - dummy_data_collection_group_info, dummy_scan_data_info_for_begin + dummy_data_collection_group_info, [dummy_scan_data_info_for_begin] ) dummy_ispyb.end_deposition(ispyb_ids, "fail", "could not connect to devices") assert ( fetch_comment(ispyb_ids.data_collection_ids[0]) # type: ignore - == "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images in 100.0 um by 100.0 um steps. Top left (px): [100,100], bottom right (px): [3300,1700]. DataCollection Unsuccessful reason: could not connect to devices" + == "DataCollection Unsuccessful reason: could not connect to devices" ) @@ -196,7 +195,7 @@ def test_ispyb_deposition_comment_correct_for_3D_on_failure( dummy_scan_data_info_for_begin, ): ispyb_ids = dummy_ispyb_3d.begin_deposition( - dummy_data_collection_group_info, dummy_scan_data_info_for_begin + dummy_data_collection_group_info, [dummy_scan_data_info_for_begin] ) scan_data_infos = generate_scan_data_infos( dummy_params, @@ -243,7 +242,7 @@ def test_can_store_2D_ispyb_data_correctly_when_in_error( CONST.SIM.DEV_ISPYB_DATABASE_CFG, experiment_type ) ispyb_ids: IspybIds = ispyb.begin_deposition( - dummy_data_collection_group_info, dummy_scan_data_info_for_begin + dummy_data_collection_group_info, [dummy_scan_data_info_for_begin] ) scan_data_infos = generate_scan_data_infos( dummy_params, dummy_scan_data_info_for_begin, experiment_type, ispyb_ids @@ -293,6 +292,7 @@ def generate_scan_data_infos( xy_scan_data_info = scan_xy_data_info_for_update( ispyb_ids.data_collection_group_id, dummy_params, dummy_scan_data_info_for_begin ) + xy_scan_data_info.data_collection_id = ispyb_ids.data_collection_ids[0] if experiment_type == ExperimentType.GRIDSCAN_3D: scan_data_infos = scan_data_infos_for_update_3d( ispyb_ids, xy_scan_data_info, dummy_params @@ -338,12 +338,14 @@ def test_ispyb_deposition_in_rotation_plan( fake_create_rotation_devices.dcm.energy_in_kev.user_readback.sim_put( # pyright: ignore energy_ev / 1000 ) - fake_create_rotation_devices.undulator.current_gap.sim_put(1.12) - fake_create_rotation_devices.synchrotron.machine_status.synchrotron_mode.sim_put( # pyright: ignore - test_synchrotron_mode.value + fake_create_rotation_devices.undulator.current_gap.sim_put(1.12) # pyright: ignore + set_sim_value( + fake_create_rotation_devices.synchrotron.synchrotron_mode, + test_synchrotron_mode, ) - fake_create_rotation_devices.synchrotron.top_up.start_countdown.sim_put( # pyright: ignore - -1 + set_sim_value( + fake_create_rotation_devices.synchrotron.topup_start_countdown, # pyright: ignore + -1, ) fake_create_rotation_devices.s4_slit_gaps.xgap.user_readback.sim_put( # pyright: ignore test_slit_gap_horiz diff --git a/tests/system_tests/external_interaction/test_zocalo_system.py b/tests/system_tests/external_interaction/test_zocalo_system.py index d4351eb68..d228bf454 100644 --- a/tests/system_tests/external_interaction/test_zocalo_system.py +++ b/tests/system_tests/external_interaction/test_zocalo_system.py @@ -1,3 +1,5 @@ +import re + import bluesky.plan_stubs as bps import bluesky.preprocessors as bpp import numpy as np @@ -9,6 +11,9 @@ from hyperion.external_interaction.callbacks.common.callback_util import ( create_gridscan_callbacks, ) +from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( + ispyb_activation_wrapper, +) from hyperion.parameters.constants import CONST from hyperion.parameters.plan_specific.gridscan_internal_params import ( GridscanInternalParameters, @@ -62,7 +67,6 @@ async def inner(sample_name="", fallback=np.array([0, 0, 0])): dummy_params.hyperion_params.detector_params.prefix = sample_name _, ispyb_callback = create_gridscan_callbacks() ispyb_callback.ispyb_config = dummy_ispyb_3d.ISPYB_CONFIG_PATH - ispyb_callback.active = True RE.subscribe(ispyb_callback) @bpp.set_run_key_decorator("testing123") @@ -84,7 +88,11 @@ def inner_plan(): yield from inner_plan() - RE(trigger_zocalo_after_fast_grid_scan()) + RE( + ispyb_activation_wrapper( + trigger_zocalo_after_fast_grid_scan(), dummy_params + ) + ) centre = await zocalo_device.centres_of_mass.get_value() if centre.size == 0: centre = fallback @@ -125,9 +133,11 @@ async def test_zocalo_adds_nonzero_comment_time( ispyb, zc, _ = await run_zocalo_with_dev_ispyb() comment = fetch_comment(ispyb.ispyb_ids.data_collection_ids[0]) - assert comment[156:178] == "Zocalo processing took" - assert float(comment[179:184]) > 0 - assert float(comment[179:184]) < 180 + match = re.match(r"Zocalo processing took (\d+\.\d+) s", comment) + assert match + time_s = float(match.group(1)) + assert time_s > 0 + assert time_s < 180 @pytest.mark.asyncio diff --git a/tests/test_data/hyperion_parameters.json b/tests/test_data/hyperion_parameters.json index 18a13a106..617c11404 100644 --- a/tests/test_data/hyperion_parameters.json +++ b/tests/test_data/hyperion_parameters.json @@ -24,11 +24,6 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json b/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json index f2fb7fdbc..56491115f 100644 --- a/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json +++ b/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json @@ -22,11 +22,6 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_parameters.json b/tests/test_data/parameter_json_files/good_test_parameters.json index 8dd136268..55f83016a 100644 --- a/tests/test_data/parameter_json_files/good_test_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_parameters.json @@ -20,11 +20,6 @@ "current_energy_ev": 100, "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json index 2fd061783..8cbdfd598 100644 --- a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json @@ -18,11 +18,6 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json index 3be80ce24..cf848a494 100644 --- a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json +++ b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json @@ -19,11 +19,6 @@ "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, "sample_id": "123456", - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/live_test_rotation_params.json b/tests/test_data/parameter_json_files/live_test_rotation_params.json index bccafacbf..13e13c050 100644 --- a/tests/test_data/parameter_json_files/live_test_rotation_params.json +++ b/tests/test_data/parameter_json_files/live_test_rotation_params.json @@ -18,11 +18,6 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json b/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json index a6aacb97b..f8a84c748 100644 --- a/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json +++ b/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json @@ -18,11 +18,6 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/panda_test_parameters.json b/tests/test_data/parameter_json_files/panda_test_parameters.json index 3063b98df..2b321e878 100644 --- a/tests/test_data/parameter_json_files/panda_test_parameters.json +++ b/tests/test_data/parameter_json_files/panda_test_parameters.json @@ -19,11 +19,6 @@ "visit_path": "/dls/i03/data/2023/cm33866-5/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/system_test_parameter_defaults.json b/tests/test_data/parameter_json_files/system_test_parameter_defaults.json index 967b2204f..d59470942 100644 --- a/tests/test_data/parameter_json_files/system_test_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/system_test_parameter_defaults.json @@ -17,11 +17,6 @@ "visit_path": "/tmp/cm31105-4", "microns_per_pixel_x": 1.25, "microns_per_pixel_y": 1.25, - "upper_left": [ - 100, - 100, - 50 - ], "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json b/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json index 3a55a6395..48c03bd87 100644 --- a/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json @@ -18,11 +18,6 @@ "visit_path": "/tmp/cm31105-4", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 0, - 0, - 0 - ], "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_parameter_defaults.json b/tests/test_data/parameter_json_files/test_parameter_defaults.json index 66f35efad..2ee9dd7d6 100644 --- a/tests/test_data/parameter_json_files/test_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/test_parameter_defaults.json @@ -17,11 +17,6 @@ "visit_path": "/tmp/cm31105-4", "microns_per_pixel_x": 1.25, "microns_per_pixel_y": 1.25, - "upper_left": [ - 50, - 100, - 0 - ], "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json b/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json index aea4eac11..e46d69a69 100644 --- a/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json +++ b/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json @@ -17,11 +17,6 @@ "visit_path": "/tmp/cm31105-4", "microns_per_pixel_x": 10.0, "microns_per_pixel_y": 10.0, - "upper_left": [ - 50, - 100, - 0 - ], "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_parameters.json b/tests/test_data/parameter_json_files/test_parameters.json index a8cc26ed9..aa373eb83 100644 --- a/tests/test_data/parameter_json_files/test_parameters.json +++ b/tests/test_data/parameter_json_files/test_parameters.json @@ -18,11 +18,6 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": [ - 10.0, - 20.0, - 30.0 - ], "position": [ 10.0, 20.0, diff --git a/tests/unit_tests/experiment_plans/conftest.py b/tests/unit_tests/experiment_plans/conftest.py index 304b370ec..7926b8caf 100644 --- a/tests/unit_tests/experiment_plans/conftest.py +++ b/tests/unit_tests/experiment_plans/conftest.py @@ -80,7 +80,7 @@ def run_generic_ispyb_handler_setup( } # type: ignore ) ispyb_handler.activity_gated_descriptor( - {"uid": "123abc", "name": CONST.PLAN.ISPYB_HARDWARE_READ} # type: ignore + {"uid": "123abc", "name": CONST.DESCRIPTORS.ISPYB_HARDWARE_READ} # type: ignore ) ispyb_handler.activity_gated_event( make_event_doc( @@ -89,7 +89,7 @@ def run_generic_ispyb_handler_setup( ) ) ispyb_handler.activity_gated_descriptor( - {"uid": "abc123", "name": CONST.PLAN.ISPYB_TRANSMISSION_FLUX_READ} # type: ignore + {"uid": "abc123", "name": CONST.DESCRIPTORS.ISPYB_TRANSMISSION_FLUX_READ} # type: ignore ) ispyb_handler.activity_gated_event( make_event_doc( @@ -148,12 +148,6 @@ def mock_subscriptions(test_fgs_params): ): nexus_callback, ispyb_callback = create_gridscan_callbacks() ispyb_callback.ispyb = MagicMock(spec=StoreInIspyb) - start_doc = { - "subplan_name": CONST.PLAN.GRIDSCAN_OUTER, - "hyperion_internal_parameters": test_fgs_params.json(), - CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, - } - ispyb_callback.activity_gated_start(start_doc) # type: ignore return (nexus_callback, ispyb_callback) diff --git a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py index c34b113c3..bc160abf7 100644 --- a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py @@ -46,6 +46,7 @@ ) from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( GridscanISPyBCallback, + ispyb_activation_wrapper, ) from hyperion.external_interaction.callbacks.xray_centre.nexus_callback import ( GridscanNexusFileCallback, @@ -163,7 +164,12 @@ def test_when_run_gridscan_called_ispyb_deposition_made_and_records_errors( error = AssertionError("Test Exception") mock_set.return_value = FailedStatus(error) - RE(flyscan_xray_centre(fake_fgs_composite, test_new_fgs_params)) + RE( + ispyb_activation_wrapper( + flyscan_xray_centre(fake_fgs_composite, test_new_fgs_params), + test_new_fgs_params, + ) + ) assert exc.value.args[0] is error ispyb_callback.ispyb.end_deposition.assert_called_once_with( @@ -391,13 +397,14 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( ): RE, (_, ispyb_cb) = RE_with_subs - run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) - RE( - run_gridscan_and_move( + def wrapped_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_fgs_params, ) - ) + + RE(ispyb_activation_wrapper(wrapped_gridscan_and_move(), test_fgs_params)) run_gridscan.assert_called_once() move_xyz.assert_called_once() @@ -421,14 +428,15 @@ def test_when_gridscan_finished_then_smargon_stub_offsets_are_set( fake_fgs_composite: FlyScanXRayCentreComposite, ): RE, (nexus_cb, ispyb_cb) = RE_with_subs - run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) - RE( - run_gridscan_and_move( + def wrapped_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_fgs_params, ) - ) + + RE(ispyb_activation_wrapper(wrapped_gridscan_and_move(), test_fgs_params)) assert ( fake_fgs_composite.smargon.stub_offsets.center_at_current_position.proc.get() == 1 @@ -454,16 +462,17 @@ def test_when_gridscan_succeeds_ispyb_comment_appended_to( fake_fgs_composite: FlyScanXRayCentreComposite, ): RE, (nexus_cb, ispyb_cb) = RE_with_subs - run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) - - RE.subscribe(VerbosePlanExecutionLoggingCallback()) - RE( - run_gridscan_and_move( + def _wrapped_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_fgs_params, ) - ) + + RE.subscribe(VerbosePlanExecutionLoggingCallback()) + + RE(ispyb_activation_wrapper(_wrapped_gridscan_and_move(), test_fgs_params)) app_to_comment: MagicMock = ispyb_cb.ispyb.append_to_comment # type:ignore app_to_comment.assert_called() call = app_to_comment.call_args_list[0] @@ -532,14 +541,16 @@ def test_when_gridscan_fails_ispyb_comment_appended_to( fake_fgs_composite: FlyScanXRayCentreComposite, ): RE, (nexus_cb, ispyb_cb) = RE_with_subs - run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) - mock_zocalo_trigger(fake_fgs_composite.zocalo, []) - RE( - run_gridscan_and_move( + + def wrapped_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_fgs_params, ) - ) + + mock_zocalo_trigger(fake_fgs_composite.zocalo, []) + RE(ispyb_activation_wrapper(wrapped_gridscan_and_move(), test_fgs_params)) app_to_comment: MagicMock = ispyb_cb.ispyb.append_to_comment # type:ignore app_to_comment.assert_called() call = app_to_comment.call_args_list[0] @@ -570,14 +581,16 @@ def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_ fake_fgs_composite.smargon.x.user_readback.sim_put(initial_x_y_z[0]) # type: ignore fake_fgs_composite.smargon.y.user_readback.sim_put(initial_x_y_z[1]) # type: ignore fake_fgs_composite.smargon.z.user_readback.sim_put(initial_x_y_z[2]) # type: ignore - run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) - mock_zocalo_trigger(fake_fgs_composite.zocalo, []) - RE( - run_gridscan_and_move( + + def wrapped_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_fgs_params, ) - ) + + mock_zocalo_trigger(fake_fgs_composite.zocalo, []) + RE(ispyb_activation_wrapper(wrapped_gridscan_and_move(), test_fgs_params)) assert np.all(move_xyz.call_args[0][1:] == initial_x_y_z) @patch( @@ -627,16 +640,17 @@ def test_given_setting_stub_offsets_disabled_then_stub_offsets_not_set( return_value=done_status ) test_fgs_params.experiment_params.set_stub_offsets = False - run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) - RE.subscribe(VerbosePlanExecutionLoggingCallback()) - - RE( - run_gridscan_and_move( + def wrapped_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_fgs_params, ) - ) + + RE.subscribe(VerbosePlanExecutionLoggingCallback()) + + RE(ispyb_activation_wrapper(wrapped_gridscan_and_move(), test_fgs_params)) assert ( fake_fgs_composite.smargon.stub_offsets.center_at_current_position.proc.get() == 0 @@ -723,15 +737,23 @@ def test_when_grid_scan_ran_then_eiger_disarmed_before_zocalo_end( ) fake_fgs_composite.xbpm_feedback.pos_stable.sim_put(1) # type: ignore - with patch( - "hyperion.external_interaction.callbacks.xray_centre.nexus_callback.NexusWriter.create_nexus_file", - autospec=True, - ), patch( - "hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger", - lambda _: modified_interactor_mock(mock_parent.run_end), + with ( + patch( + "hyperion.external_interaction.callbacks.xray_centre.nexus_callback.NexusWriter.create_nexus_file", + autospec=True, + ), + patch( + "hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger", + lambda _: modified_interactor_mock(mock_parent.run_end), + ), ): [RE.subscribe(cb) for cb in (nexus_cb, ispyb_cb)] - RE(flyscan_xray_centre(fake_fgs_composite, test_new_fgs_params)) + RE( + ispyb_activation_wrapper( + flyscan_xray_centre(fake_fgs_composite, test_new_fgs_params), + test_new_fgs_params, + ) + ) mock_parent.assert_has_calls([call.disarm(), call.run_end(0), call.run_end(0)]) diff --git a/tests/unit_tests/experiment_plans/test_grid_detect_then_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_grid_detect_then_xray_centre_plan.py index 35409c7a3..881299fc9 100644 --- a/tests/unit_tests/experiment_plans/test_grid_detect_then_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_grid_detect_then_xray_centre_plan.py @@ -9,16 +9,13 @@ from dodal.devices.eiger import EigerDetector from dodal.devices.oav.oav_detector import OAVConfigParams from dodal.devices.oav.oav_parameters import OAVParameters -from numpy.testing import assert_array_equal from hyperion.experiment_plans.grid_detect_then_xray_centre_plan import ( GridDetectThenXRayCentreComposite, detect_grid_and_do_gridscan, grid_detect_then_xray_centre, ) -from hyperion.external_interaction.callbacks.oav_snapshot_callback import ( - OavSnapshotCallback, -) +from hyperion.parameters.constants import CONST from hyperion.parameters.plan_specific.grid_scan_with_edge_detect_params import ( GridScanWithEdgeDetectInternalParameters, ) @@ -39,13 +36,12 @@ def _fake_grid_detection( ): oav = i03.oav(fake_with_ophyd_sim=True) smargon = fake_smargon() - yield from bps.open_run() oav.snapshot.box_width.put(635.00986) # first grid detection: x * y oav.snapshot.num_boxes_x.put(10) oav.snapshot.num_boxes_y.put(4) - yield from bps.create("snapshot_to_ispyb") + yield from bps.create(CONST.DESCRIPTORS.OAV_SNAPSHOT_TRIGGERED) yield from bps.read(oav.snapshot) yield from bps.read(smargon) yield from bps.save() @@ -53,13 +49,11 @@ def _fake_grid_detection( # second grid detection: x * z, so num_boxes_y refers to smargon z oav.snapshot.num_boxes_x.put(10) oav.snapshot.num_boxes_y.put(1) - yield from bps.create("snapshot_to_ispyb") + yield from bps.create(CONST.DESCRIPTORS.OAV_SNAPSHOT_TRIGGERED) yield from bps.read(oav.snapshot) yield from bps.read(smargon) yield from bps.save() - yield from bps.close_run() - @pytest.fixture def grid_detect_devices(aperture_scatterguard, backlight, detector_motion): @@ -103,12 +97,7 @@ def test_full_grid_scan(test_new_fgs_params, test_config_files): "hyperion.experiment_plans.grid_detect_then_xray_centre_plan.flyscan_xray_centre", autospec=True, ) -@patch( - "hyperion.experiment_plans.grid_detect_then_xray_centre_plan.OavSnapshotCallback", - autospec=True, -) def test_detect_grid_and_do_gridscan( - mock_oav_callback_init: MagicMock, mock_flyscan_xray_centre_plan: MagicMock, mock_grid_detection_plan: MagicMock, grid_detect_devices: GridDetectThenXRayCentreComposite, @@ -116,10 +105,6 @@ def test_detect_grid_and_do_gridscan( test_full_grid_scan_params: GridScanWithEdgeDetectInternalParameters, test_config_files: Dict, ): - mock_oav_callback = OavSnapshotCallback() - mock_oav_callback.out_upper_left = [[0, 1], [2, 3]] - mock_oav_callback.snapshot_filenames = [["test"], ["test3"]] - mock_oav_callback_init.return_value = mock_oav_callback mock_grid_detection_plan.side_effect = _fake_grid_detection grid_detect_devices.oav.parameters = OAVConfigParams( test_config_files["zoom_params_file"], test_config_files["display_config"] @@ -145,9 +130,6 @@ def test_detect_grid_and_do_gridscan( # Verify we called the grid detection plan mock_grid_detection_plan.assert_called_once() - # Verify callback to oav snaposhot was called - mock_oav_callback_init.assert_called_once() - # Check backlight was moved OUT assert grid_detect_devices.backlight.pos.get() == Backlight.OUT @@ -168,12 +150,7 @@ def test_detect_grid_and_do_gridscan( "hyperion.experiment_plans.grid_detect_then_xray_centre_plan.flyscan_xray_centre", autospec=True, ) -@patch( - "hyperion.experiment_plans.grid_detect_then_xray_centre_plan.OavSnapshotCallback", - autospec=True, -) def test_when_full_grid_scan_run_then_parameters_sent_to_fgs_as_expected( - mock_oav_callback_init: MagicMock, mock_flyscan_xray_centre_plan: MagicMock, mock_grid_detection_plan: MagicMock, eiger: EigerDetector, @@ -183,10 +160,6 @@ def test_when_full_grid_scan_run_then_parameters_sent_to_fgs_as_expected( test_config_files: Dict, ): oav_params = OAVParameters("xrayCentring", test_config_files["oav_config_json"]) - mock_oav_callback = OavSnapshotCallback() - mock_oav_callback.snapshot_filenames = [["a", "b", "c"], ["d", "e", "f"]] - mock_oav_callback.out_upper_left = [[1, 2], [1, 3]] - mock_oav_callback_init.return_value = mock_oav_callback mock_grid_detection_plan.side_effect = _fake_grid_detection @@ -215,19 +188,6 @@ def test_when_full_grid_scan_run_then_parameters_sent_to_fgs_as_expected( assert isinstance(params, GridscanInternalParameters) - ispyb_params = params.hyperion_params.ispyb_params - assert_array_equal(ispyb_params.upper_left, [1, 2, 3]) - assert ispyb_params.xtal_snapshots_omega_start == [ - "c", - "b", - "a", - ] - assert ispyb_params.xtal_snapshots_omega_end == [ - "f", - "e", - "d", - ] - assert params.hyperion_params.detector_params.num_triggers == 50 assert params.experiment_params.x_axis.full_steps == 10 diff --git a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py index 9267cd550..21d4daeb0 100644 --- a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py +++ b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py @@ -1,5 +1,6 @@ -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import DEFAULT, AsyncMock, MagicMock, patch +import bluesky.preprocessors as bpp import numpy as np import pytest from bluesky.run_engine import RunEngine @@ -22,11 +23,13 @@ from hyperion.external_interaction.callbacks.grid_detection_callback import ( GridDetectionCallback, ) -from hyperion.external_interaction.callbacks.oav_snapshot_callback import ( - OavSnapshotCallback, +from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( + GridscanISPyBCallback, + ispyb_activation_wrapper, ) from ...conftest import RunEngineSimulator +from .conftest import assert_event @pytest.fixture @@ -62,9 +65,10 @@ def fake_devices(RE, smargon: Smargon, backlight: Backlight, test_config_files): oav.zoom_controller.frst.set("7.0x") oav.zoom_controller.fvst.set("9.0x") - with patch("dodal.devices.areadetector.plugins.MJPG.requests"), patch( - "dodal.devices.areadetector.plugins.MJPG.Image" - ) as mock_image_class: + with ( + patch("dodal.devices.areadetector.plugins.MJPG.requests"), + patch("dodal.devices.areadetector.plugins.MJPG.Image") as mock_image_class, + ): mock_image = MagicMock() mock_image_class.open.return_value.__enter__.return_value = mock_image @@ -86,28 +90,20 @@ def test_grid_detection_plan_runs_and_triggers_snapshots( fake_devices, ): params = OAVParameters("loopCentring", test_config_files["oav_config_json"]) - cb = OavSnapshotCallback() - RE.subscribe(cb) composite, image = fake_devices - RE( - grid_detection_plan( + @bpp.run_decorator() + def decorated(): + yield from grid_detection_plan( composite, parameters=params, snapshot_dir="tmp", snapshot_template="test_{angle}", grid_width_microns=161.2, ) - ) - assert image.save.call_count == 6 - - assert len(cb.snapshot_filenames) == 2 - assert len(cb.snapshot_filenames[0]) == 3 - assert cb.snapshot_filenames[0][0] == "tmp/test_0.png" - assert cb.snapshot_filenames[1][2] == "tmp/test_90_grid_overlay.png" - assert len(cb.out_upper_left) == 2 - assert len(cb.out_upper_left[0]) + RE(decorated()) + assert image.save.call_count == 6 @patch("dodal.beamlines.beamline_utils.active_device_is_same_type", lambda a, b: True) @@ -160,12 +156,12 @@ def test_given_when_grid_detect_then_upper_left_and_start_position_as_expected( composite.oav.parameters.beam_centre_j = 4 box_size_y_pixels = box_size_um / composite.oav.parameters.micronsPerYPixel - oav_cb = OavSnapshotCallback() grid_param_cb = GridDetectionCallback(composite.oav.parameters, 0.004, False) - RE.subscribe(oav_cb) RE.subscribe(grid_param_cb) - RE( - grid_detection_plan( + + @bpp.run_decorator() + def decorated(): + yield from grid_detection_plan( composite, parameters=params, snapshot_dir="tmp", @@ -173,11 +169,8 @@ def test_given_when_grid_detect_then_upper_left_and_start_position_as_expected( grid_width_microns=161.2, box_size_um=0.2, ) - ) - # 8, 2 based on tip x, and lowest value in the top array - assert oav_cb.out_upper_left[0] == [8, 2 - box_size_y_pixels / 2] - assert oav_cb.out_upper_left[1] == [8, 2] + RE(decorated()) gridscan_params = grid_param_cb.get_grid_parameters() @@ -207,28 +200,80 @@ def test_when_grid_detection_plan_run_twice_then_values_do_not_persist_in_callba composite, _ = fake_devices for _ in range(2): - cb = OavSnapshotCallback() - RE.subscribe(cb) - RE( - grid_detection_plan( + @bpp.run_decorator() + def decorated(): + yield from grid_detection_plan( composite, parameters=params, snapshot_dir="tmp", snapshot_template="test_{angle}", grid_width_microns=161.2, ) + + RE(decorated()) + + +@patch("dodal.beamlines.beamline_utils.active_device_is_same_type", lambda a, b: True) +@patch("bluesky.plan_stubs.sleep", new=MagicMock()) +def test_when_grid_detection_plan_run_then_ispyb_callback_gets_correct_values( + fake_devices, RE: RunEngine, test_config_files, test_fgs_params +): + params = OAVParameters("loopCentring", test_config_files["oav_config_json"]) + composite, _ = fake_devices + composite.oav.parameters.micronsPerYPixel = 1.25 + composite.oav.parameters.micronsPerXPixel = 1.25 + cb = GridscanISPyBCallback() + RE.subscribe(cb) + + def decorated(): + yield from grid_detection_plan( + composite, + parameters=params, + snapshot_dir="tmp", + snapshot_template="test_{angle}", + grid_width_microns=161.2, + ) + + with patch.multiple(cb, activity_gated_start=DEFAULT, activity_gated_event=DEFAULT): + RE(ispyb_activation_wrapper(decorated(), test_fgs_params)) + + assert_event( + cb.activity_gated_start.mock_calls[0], # pyright:ignore + {"activate_callbacks": ["GridscanISPyBCallback"]}, + ) + assert_event( + cb.activity_gated_event.mock_calls[0], # pyright: ignore + { + "oav_snapshot_top_left_x": 8, + "oav_snapshot_top_left_y": -6, + "oav_snapshot_num_boxes_x": 8, + "oav_snapshot_num_boxes_y": 2, + "oav_snapshot_box_width": 16, + "oav_snapshot_last_path_full_overlay": "tmp/test_0_grid_overlay.png", + "oav_snapshot_last_path_outer": "tmp/test_0_outer_overlay.png", + "oav_snapshot_last_saved_path": "tmp/test_0.png", + }, + ) + assert_event( + cb.activity_gated_event.mock_calls[1], # pyright:ignore + { + "oav_snapshot_top_left_x": 8, + "oav_snapshot_top_left_y": 2, + "oav_snapshot_num_boxes_x": 8, + "oav_snapshot_num_boxes_y": 1, + "oav_snapshot_box_width": 16, + "oav_snapshot_last_path_full_overlay": "tmp/test_90_grid_overlay.png", + "oav_snapshot_last_path_outer": "tmp/test_90_outer_overlay.png", + "oav_snapshot_last_saved_path": "tmp/test_90.png", + }, ) - assert len(cb.snapshot_filenames) == 2 - assert len(cb.out_upper_left) == 2 @patch("dodal.beamlines.beamline_utils.active_device_is_same_type", lambda a, b: True) @patch("bluesky.plan_stubs.sleep", new=MagicMock()) def test_when_grid_detection_plan_run_then_grid_detection_callback_gets_correct_values( - fake_devices, - RE: RunEngine, - test_config_files, + fake_devices, RE: RunEngine, test_config_files, test_fgs_params ): params = OAVParameters("loopCentring", test_config_files["oav_config_json"]) composite, _ = fake_devices @@ -236,15 +281,16 @@ def test_when_grid_detection_plan_run_then_grid_detection_callback_gets_correct_ cb = GridDetectionCallback(composite.oav.parameters, 0.5, True) RE.subscribe(cb) - RE( - grid_detection_plan( + def decorated(): + yield from grid_detection_plan( composite, parameters=params, snapshot_dir="tmp", snapshot_template="test_{angle}", grid_width_microns=161.2, ) - ) + + RE(ispyb_activation_wrapper(decorated(), test_fgs_params)) my_grid_params = cb.get_grid_parameters() diff --git a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py index 77278f3af..3c8e87158 100644 --- a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py @@ -1,6 +1,6 @@ import random import types -from typing import Tuple +from typing import Any, Tuple from unittest.mock import DEFAULT, MagicMock, call, patch import bluesky.preprocessors as bpp @@ -43,6 +43,7 @@ ) from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( GridscanISPyBCallback, + ispyb_activation_wrapper, ) from hyperion.external_interaction.callbacks.xray_centre.nexus_callback import ( GridscanNexusFileCallback, @@ -351,18 +352,24 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( move_xyz: MagicMock, run_gridscan: MagicMock, move_aperture: MagicMock, - RE: RunEngine, + RE_with_subs, mock_subscriptions: Tuple[GridscanNexusFileCallback, GridscanISPyBCallback], fake_fgs_composite: FlyScanXRayCentreComposite, test_panda_fgs_params: PandAGridscanInternalParameters, ): - _, ispyb_cb = mock_subscriptions - run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) - RE( - run_gridscan_and_move( + def wrapped_run_gridscan_and_move(): + run_generic_ispyb_handler_setup( + mock_subscriptions[1], test_panda_fgs_params + ) + yield from run_gridscan_and_move( fake_fgs_composite, test_panda_fgs_params, ) + + RE_with_subs[0]( + ispyb_activation_wrapper( + wrapped_run_gridscan_and_move(), test_panda_fgs_params + ) ) run_gridscan.assert_called_once() move_xyz.assert_called_once() @@ -389,19 +396,24 @@ def test_when_gridscan_finished_then_smargon_stub_offsets_are_set( move_xyz: MagicMock, run_gridscan: MagicMock, aperture_set: MagicMock, - RE: RunEngine, + RE_with_subs: tuple[RunEngine, Any], mock_subscriptions: Tuple[GridscanNexusFileCallback, GridscanISPyBCallback], test_panda_fgs_params: PandAGridscanInternalParameters, fake_fgs_composite: FlyScanXRayCentreComposite, ): - _, ispyb_cb = mock_subscriptions - run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) - - RE( - run_gridscan_and_move( + def wrapped_run_gridscan_and_move(): + run_generic_ispyb_handler_setup( + mock_subscriptions[1], test_panda_fgs_params + ) + yield from run_gridscan_and_move( fake_fgs_composite, test_panda_fgs_params, ) + + RE_with_subs[0]( + ispyb_activation_wrapper( + wrapped_run_gridscan_and_move(), test_panda_fgs_params + ) ) assert ( fake_fgs_composite.smargon.stub_offsets.center_at_current_position.proc.get() @@ -430,22 +442,26 @@ def test_when_gridscan_succeeds_ispyb_comment_appended_to( move_xyz: MagicMock, run_gridscan: MagicMock, aperture_set: MagicMock, - RE: RunEngine, + RE_with_subs: tuple[RunEngine, Any], mock_subscriptions: Tuple[GridscanNexusFileCallback, GridscanISPyBCallback], test_panda_fgs_params: PandAGridscanInternalParameters, fake_fgs_composite: FlyScanXRayCentreComposite, ): - _, ispyb_cb = mock_subscriptions - run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) - - RE.subscribe(ispyb_cb) - RE.subscribe(VerbosePlanExecutionLoggingCallback()) + RE_with_subs[0].subscribe(VerbosePlanExecutionLoggingCallback()) - RE( - run_gridscan_and_move( + def wrapped_run_gridscan_and_move(): + run_generic_ispyb_handler_setup( + mock_subscriptions[1], test_panda_fgs_params + ) + yield from run_gridscan_and_move( fake_fgs_composite, test_panda_fgs_params, ) + + RE_with_subs[0]( + ispyb_activation_wrapper( + wrapped_run_gridscan_and_move(), test_panda_fgs_params + ) ) app_to_comment: MagicMock = mock_subscriptions[ 1 @@ -471,20 +487,26 @@ def test_when_gridscan_fails_ispyb_comment_appended_to( setup_panda_for_flyscan: MagicMock, move_xyz: MagicMock, run_gridscan: MagicMock, - RE: RunEngine, + RE_with_subs: tuple[RunEngine, Any], mock_subscriptions: Tuple[GridscanNexusFileCallback, GridscanISPyBCallback], test_panda_fgs_params: PandAGridscanInternalParameters, fake_fgs_composite: FlyScanXRayCentreComposite, ): - _, ispyb_cb = mock_subscriptions - run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) mock_zocalo_trigger(fake_fgs_composite.zocalo, []) - RE.subscribe(ispyb_cb) - RE( - run_gridscan_and_move( + + def wrapped_run_gridscan_and_move(): + run_generic_ispyb_handler_setup( + mock_subscriptions[1], test_panda_fgs_params + ) + yield from run_gridscan_and_move( fake_fgs_composite, test_panda_fgs_params, ) + + RE_with_subs[0]( + ispyb_activation_wrapper( + wrapped_run_gridscan_and_move(), test_panda_fgs_params + ) ) app_to_comment: MagicMock = mock_subscriptions[ 1 @@ -528,13 +550,19 @@ def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_ fake_fgs_composite.smargon.x.user_readback.sim_put(initial_x_y_z[0]) # type: ignore fake_fgs_composite.smargon.y.user_readback.sim_put(initial_x_y_z[1]) # type: ignore fake_fgs_composite.smargon.z.user_readback.sim_put(initial_x_y_z[2]) # type: ignore - run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) mock_zocalo_trigger(fake_fgs_composite.zocalo, []) - RE( - run_gridscan_and_move( + + def wrapped_run_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_panda_fgs_params, ) + + RE( + ispyb_activation_wrapper( + wrapped_run_gridscan_and_move(), test_panda_fgs_params + ) ) assert np.all(move_xyz.call_args[0][1:] == initial_x_y_z) @@ -600,15 +628,21 @@ def test_given_setting_stub_offsets_disabled_then_stub_offsets_not_set( return_value=done_status ) test_panda_fgs_params.experiment_params.set_stub_offsets = False - run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) RE.subscribe(VerbosePlanExecutionLoggingCallback()) + RE.subscribe(ispyb_cb) - RE( - run_gridscan_and_move( + def wrapped_run_gridscan_and_move(): + run_generic_ispyb_handler_setup(ispyb_cb, test_panda_fgs_params) + yield from run_gridscan_and_move( fake_fgs_composite, test_panda_fgs_params, ) + + RE( + ispyb_activation_wrapper( + wrapped_run_gridscan_and_move(), test_panda_fgs_params + ) ) assert ( fake_fgs_composite.smargon.stub_offsets.center_at_current_position.proc.get() @@ -708,14 +742,24 @@ def test_when_grid_scan_ran_then_eiger_disarmed_before_zocalo_end( ) fake_fgs_composite.xbpm_feedback.pos_stable.sim_put(1) # type: ignore - with patch( - "hyperion.external_interaction.callbacks.xray_centre.nexus_callback.NexusWriter.create_nexus_file", - autospec=True, - ), patch( - "hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger", - lambda _: modified_interactor_mock(mock_parent.run_end), + with ( + patch( + "hyperion.external_interaction.callbacks.xray_centre.nexus_callback.NexusWriter.create_nexus_file", + autospec=True, + ), + patch( + "hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger", + lambda _: modified_interactor_mock(mock_parent.run_end), + ), ): - RE(panda_flyscan_xray_centre(fake_fgs_composite, test_panda_fgs_params)) + RE( + ispyb_activation_wrapper( + panda_flyscan_xray_centre( + fake_fgs_composite, test_panda_fgs_params + ), + test_panda_fgs_params, + ) + ) mock_parent.assert_has_calls([call.disarm(), call.run_end(0), call.run_end(0)]) diff --git a/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py index c7c7ceef3..f69611cc7 100644 --- a/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_pin_centre_then_xray_centre_plan.py @@ -71,9 +71,6 @@ def test_when_pin_centre_xray_centre_called_then_plan_runs_correctly( @patch( "hyperion.experiment_plans.grid_detect_then_xray_centre_plan.GridDetectionCallback", ) -@patch( - "hyperion.experiment_plans.grid_detect_then_xray_centre_plan.OavSnapshotCallback", -) @patch( "hyperion.experiment_plans.pin_centre_then_xray_centre_plan.pin_tip_centre_plan", autospec=True, @@ -85,19 +82,12 @@ def test_when_pin_centre_xray_centre_called_then_plan_runs_correctly( def test_when_pin_centre_xray_centre_called_then_detector_positioned( mock_grid_detect: MagicMock, mock_pin_tip_centre: MagicMock, - mock_oav_callback: MagicMock, mock_grid_callback: MagicMock, test_pin_centre_then_xray_centre_params: PinCentreThenXrayCentreInternalParameters, simple_beamline, test_config_files, sim_run_engine, ): - mock_oav_callback.return_value.out_upper_left = [[1, 3], [3, 4]] - mock_oav_callback.return_value.snapshot_filenames = [ - ["1.png", "2.png", "3.png"], - ["1.png", "2.png", "3.png"], - ["1.png", "2.png", "3.png"], - ] mock_grid_callback.return_value.get_grid_parameters.return_value = GridScanParams( transmission_fraction=0.01, dwell_time_ms=0, diff --git a/tests/unit_tests/external_interaction/callbacks/conftest.py b/tests/unit_tests/external_interaction/callbacks/conftest.py index c1ebef8cb..e56c7b185 100644 --- a/tests/unit_tests/external_interaction/callbacks/conftest.py +++ b/tests/unit_tests/external_interaction/callbacks/conftest.py @@ -49,20 +49,41 @@ class TestData: CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, "hyperion_internal_parameters": dummy_params().json(), } + test_gridscan3d_start_document: RunStart = { # type: ignore + "uid": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", + "time": 1666604299.6149616, + "versions": {"ophyd": "1.6.4.post76+g0895f9f", "bluesky": "1.8.3"}, + "scan_id": 1, + "plan_type": "generator", + "plan_name": "test", + "subplan_name": CONST.PLAN.GRID_DETECT_AND_DO_GRIDSCAN, + "hyperion_internal_parameters": dummy_params().json(), + } test_gridscan2d_start_document = { "uid": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", "time": 1666604299.6149616, "versions": {"ophyd": "1.6.4.post76+g0895f9f", "bluesky": "1.8.3"}, "scan_id": 1, "plan_type": "generator", - "plan_name": CONST.PLAN.GRIDSCAN_OUTER, - "subplan_name": CONST.PLAN.GRIDSCAN_OUTER, + "plan_name": "test", + "subplan_name": CONST.PLAN.GRID_DETECT_AND_DO_GRIDSCAN, "hyperion_internal_parameters": dummy_params_2d().json(), } test_rotation_start_main_document = { "uid": "2093c941-ded1-42c4-ab74-ea99980fbbfd", "subplan_name": CONST.PLAN.ROTATION_MAIN, } + test_gridscan_outer_start_document = { + "uid": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", + "time": 1666604299.6149616, + "versions": {"ophyd": "1.6.4.post76+g0895f9f", "bluesky": "1.8.3"}, + "scan_id": 1, + "plan_type": "generator", + "plan_name": CONST.PLAN.GRIDSCAN_OUTER, + "subplan_name": CONST.PLAN.GRIDSCAN_OUTER, + CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS, + "hyperion_internal_parameters": dummy_params().json(), + } test_rotation_event_document_during_data_collection: Event = { "descriptor": "bd45c2e5-2b85-4280-95d7-a9a15800a78b", "time": 2666604299.928203, @@ -104,26 +125,65 @@ class TestData: "zocalo_environment": "dev_artemis", "scan_points": create_dummy_scan_spec(10, 20, 30), } + test_descriptor_document_oav_snapshot: EventDescriptor = { + "uid": "b5ba4aec-de49-4970-81a4-b4a847391d34", + "run_start": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", + "name": CONST.DESCRIPTORS.OAV_SNAPSHOT_TRIGGERED, + } # type: ignore test_descriptor_document_pre_data_collection: EventDescriptor = { "uid": "bd45c2e5-2b85-4280-95d7-a9a15800a78b", "run_start": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", - "name": CONST.PLAN.ISPYB_HARDWARE_READ, + "name": CONST.DESCRIPTORS.ISPYB_HARDWARE_READ, } # type: ignore test_descriptor_document_during_data_collection: EventDescriptor = { "uid": "bd45c2e5-2b85-4280-95d7-a9a15800a78b", "run_start": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", - "name": CONST.PLAN.ISPYB_TRANSMISSION_FLUX_READ, + "name": CONST.DESCRIPTORS.ISPYB_TRANSMISSION_FLUX_READ, } # type: ignore test_descriptor_document_zocalo_hardware: EventDescriptor = { "uid": "f082901b-7453-4150-8ae5-c5f98bb34406", "run_start": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", - "name": CONST.PLAN.ZOCALO_HW_READ, + "name": CONST.DESCRIPTORS.ZOCALO_HW_READ, } # type: ignore test_descriptor_document_nexus_read: EventDescriptor = { "uid": "aaaaaa", "run_start": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", - "name": CONST.PLAN.NEXUS_READ, + "name": CONST.DESCRIPTORS.NEXUS_READ, } # type: ignore + test_event_document_oav_snapshot_xy: Event = { + "descriptor": "b5ba4aec-de49-4970-81a4-b4a847391d34", + "time": 1666604299.828203, + "timestamps": {}, + "seq_num": 1, + "uid": "29033ecf-e052-43dd-98af-c7cdd62e8174", + "data": { + "oav_snapshot_top_left_x": 50, + "oav_snapshot_top_left_y": 100, + "oav_snapshot_num_boxes_x": 40, + "oav_snapshot_num_boxes_y": 20, + "oav_snapshot_box_width": 0.1 * 1000 / 1.25, # size in pixels + "oav_snapshot_last_path_full_overlay": "test_1_y", + "oav_snapshot_last_path_outer": "test_2_y", + "oav_snapshot_last_saved_path": "test_3_y", + }, + } + test_event_document_oav_snapshot_xz: Event = { + "descriptor": "b5ba4aec-de49-4970-81a4-b4a847391d34", + "time": 1666604299.828203, + "timestamps": {}, + "seq_num": 1, + "uid": "29033ecf-e052-43dd-98af-c7cdd62e8174", + "data": { + "oav_snapshot_top_left_x": 50, + "oav_snapshot_top_left_y": 0, + "oav_snapshot_num_boxes_x": 40, + "oav_snapshot_num_boxes_y": 10, + "oav_snapshot_box_width": 0.1 * 1000 / 1.25, # size in pixels + "oav_snapshot_last_path_full_overlay": "test_1_z", + "oav_snapshot_last_path_outer": "test_2_z", + "oav_snapshot_last_saved_path": "test_3_z", + }, + } test_event_document_pre_data_collection: Event = { "descriptor": "bd45c2e5-2b85-4280-95d7-a9a15800a78b", "time": 1666604299.828203, diff --git a/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py index 335ef58bc..47e57a4a3 100644 --- a/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py @@ -114,7 +114,7 @@ def test_given_plan_reads_barcode_then_data_put_in_ispyb( @bpp.run_decorator(md=metadata) def my_plan(): - yield from bps.create(name=CONST.PLAN.ROBOT_LOAD) + yield from bps.create(name=CONST.DESCRIPTORS.ROBOT_LOAD) yield from bps.read(robot.barcode) yield from bps.save() diff --git a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py index c6bb5ca8f..bbf79e6ce 100644 --- a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py @@ -82,7 +82,7 @@ def test_activity_gated_start(mock_ispyb_conn, test_rotation_start_outer_documen "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", new=MagicMock(return_value=EXPECTED_START_TIME), ) -def test_activity_gated_event( +def test_hardware_read_events( mock_ispyb_conn, dummy_rotation_params, test_rotation_start_outer_document ): callback = RotationISPyBCallback() @@ -99,12 +99,6 @@ def test_activity_gated_event( TestData.test_descriptor_document_pre_data_collection ) callback.activity_gated_event(TestData.test_event_document_pre_data_collection) - callback.activity_gated_descriptor( - TestData.test_descriptor_document_during_data_collection - ) - callback.activity_gated_event( - TestData.test_rotation_event_document_during_data_collection - ) assert_upsert_call_with( mx.upsert_data_collection_group.mock_calls[0], mx.get_data_collection_group_params(), @@ -119,16 +113,13 @@ def test_activity_gated_event( assert_upsert_call_with( mx.upsert_data_collection.mock_calls[0], mx.get_data_collection_params(), - EXPECTED_DATA_COLLECTION - | { + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, "id": TEST_DATA_COLLECTION_IDS[0], "slitgaphorizontal": 0.1234, "slitgapvertical": 0.2345, "synchrotronmode": "User", "undulatorgap1": 1.234, - "wavelength": 1.1164718451643736, - "transmission": 98, - "flux": 9.81, }, ) assert_upsert_call_with( @@ -143,6 +134,46 @@ def test_activity_gated_event( ) +@patch( + "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", + new=MagicMock(return_value=EXPECTED_START_TIME), +) +def test_flux_read_events( + mock_ispyb_conn, dummy_rotation_params, test_rotation_start_outer_document +): + callback = RotationISPyBCallback() + callback.activity_gated_start(test_rotation_start_outer_document) # pyright: ignore + callback.activity_gated_start( + TestData.test_rotation_start_main_document # pyright: ignore + ) + mx = mx_acquisition_from_conn(mock_ispyb_conn) + callback.activity_gated_descriptor( + TestData.test_descriptor_document_pre_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_pre_data_collection) + mx.upsert_data_collection_group.reset_mock() + mx.upsert_data_collection.reset_mock() + callback.activity_gated_descriptor( + TestData.test_descriptor_document_during_data_collection + ) + callback.activity_gated_event( + TestData.test_rotation_event_document_during_data_collection + ) + + mx.upsert_data_collection_group.assert_not_called() + assert_upsert_call_with( + mx.upsert_data_collection.mock_calls[0], + mx.get_data_collection_params(), + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_IDS[0], + "wavelength": 1.1164718451643736, + "transmission": 98, + "flux": 9.81, + }, + ) + + @patch( "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", new=MagicMock(return_value=EXPECTED_START_TIME), diff --git a/tests/unit_tests/external_interaction/callbacks/test_plan_reactive_callback.py b/tests/unit_tests/external_interaction/callbacks/test_plan_reactive_callback.py index 007627e1b..0f3dca5c5 100644 --- a/tests/unit_tests/external_interaction/callbacks/test_plan_reactive_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/test_plan_reactive_callback.py @@ -1,5 +1,7 @@ -from unittest.mock import MagicMock +from unittest.mock import DEFAULT, MagicMock, patch +import bluesky.plan_stubs as bps +import bluesky.preprocessors as bpp import pytest from bluesky.run_engine import RunEngine from event_model.documents import Event, EventDescriptor, RunStart, RunStop @@ -44,14 +46,6 @@ def test_activates_on_appropriate_start_doc(mocked_test_callback): assert mocked_test_callback.active is True -def test_deactivates_on_inappropriate_start_doc(mocked_test_callback): - assert mocked_test_callback.active is False - mocked_test_callback.start({"activate_callbacks": ["MockReactiveCallback"]}) - assert mocked_test_callback.active is True - mocked_test_callback.start({"activate_callbacks": ["TestNotCallback"]}) - assert mocked_test_callback.active is False - - def test_deactivates_on_appropriate_stop_doc_uid(mocked_test_callback): assert mocked_test_callback.active is False mocked_test_callback.start( @@ -73,7 +67,7 @@ def test_doesnt_deactivate_on_inappropriate_stop_doc_uid(mocked_test_callback): def test_activates_on_metadata( - RE_with_mock_callback: tuple[RunEngine, MockReactiveCallback] + RE_with_mock_callback: tuple[RunEngine, MockReactiveCallback], ): RE, callback = RE_with_mock_callback RE(get_test_plan("MockReactiveCallback")[0]()) @@ -84,7 +78,7 @@ def test_activates_on_metadata( def test_deactivates_after_closing( - RE_with_mock_callback: tuple[RunEngine, MockReactiveCallback] + RE_with_mock_callback: tuple[RunEngine, MockReactiveCallback], ): RE, callback = RE_with_mock_callback assert callback.active is False @@ -93,7 +87,7 @@ def test_deactivates_after_closing( def test_doesnt_activate_on_wrong_metadata( - RE_with_mock_callback: tuple[RunEngine, MockReactiveCallback] + RE_with_mock_callback: tuple[RunEngine, MockReactiveCallback], ): RE, callback = RE_with_mock_callback RE(get_test_plan("TestNotCallback")[0]()) @@ -159,3 +153,68 @@ def test_emit_called_correctly(): receiving_cb.activity_gated_event.assert_called_once_with(event_doc) test_cb.stop(stop_doc) receiving_cb.activity_gated_stop.assert_called_once_with(stop_doc) + + +class OuterCallback(PlanReactiveCallback): + pass + + +class InnerCallback(PlanReactiveCallback): + pass + + +def test_activate_callbacks_doesnt_deactivate_unlisted_callbacks(RE: RunEngine): + @bpp.set_run_key_decorator("inner_plan") + @bpp.run_decorator(md={"activate_callbacks": ["InnerCallback"]}) + def inner_plan(): + yield from bps.null() + + @bpp.set_run_key_decorator("outer_plan") + @bpp.run_decorator(md={"activate_callbacks": ["OuterCallback"]}) + def outer_plan(): + yield from inner_plan() + + outer_callback = OuterCallback(MagicMock()) + inner_callback = InnerCallback(MagicMock()) + + RE.subscribe(outer_callback) + RE.subscribe(inner_callback) + + with patch.multiple( + outer_callback, activity_gated_start=DEFAULT, activity_gated_stop=DEFAULT + ): + with patch.multiple( + inner_callback, activity_gated_start=DEFAULT, activity_gated_stop=DEFAULT + ): + root_mock = MagicMock() + # fmt: off + root_mock.attach_mock(outer_callback.activity_gated_start, "outer_start") # pyright: ignore + root_mock.attach_mock(outer_callback.activity_gated_stop, "outer_stop") # pyright: ignore + root_mock.attach_mock(inner_callback.activity_gated_start, "inner_start") # pyright: ignore + root_mock.attach_mock(inner_callback.activity_gated_stop, "inner_stop") # pyright: ignore + # fmt: on + RE(outer_plan()) + + assert [call[0] for call in root_mock.mock_calls] == [ + "outer_start", + "outer_start", + "inner_start", + "outer_stop", + "inner_stop", + "outer_stop", + ] + + assert ( + root_mock.mock_calls[0].args[0]["uid"] + != root_mock.mock_calls[1].args[0]["uid"] + ) + assert root_mock.mock_calls[1].args[0] == root_mock.mock_calls[2].args[0] + assert root_mock.mock_calls[3].args[0] == root_mock.mock_calls[4].args[0] + assert ( + root_mock.mock_calls[0].args[0]["uid"] + == root_mock.mock_calls[5].args[0]["run_start"] + ) + assert ( + root_mock.mock_calls[2].args[0]["uid"] + == root_mock.mock_calls[4].args[0]["run_start"] + ) diff --git a/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py b/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py index 4a954de47..48585d4b6 100644 --- a/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py +++ b/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py @@ -352,7 +352,7 @@ def after_main_do(callbacks: list[RotationISPyBCallback]): RE(fake_rotation_scan(params, [ispyb_cb], after_open_do, after_main_do)) begin_deposition_scan_data: ScanDataInfo = ( - rotation_ispyb.return_value.begin_deposition.call_args.args[1] + rotation_ispyb.return_value.begin_deposition.call_args.args[1][0] ) if same_dcgid: assert begin_deposition_scan_data.data_collection_info.parent_id is not None diff --git a/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py b/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py index da85b67a8..6d2bfbd39 100644 --- a/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py +++ b/tests/unit_tests/external_interaction/callbacks/test_zocalo_handler.py @@ -103,7 +103,8 @@ def test_execution_of_do_fgs_triggers_zocalo_calls( ispyb_store.return_value.begin_deposition.return_value = mock_ids ispyb_store.return_value.update_deposition.return_value = mock_ids - ispyb_cb.start(td.test_start_document) # type: ignore + ispyb_cb.start(td.test_gridscan3d_start_document) # type: ignore + ispyb_cb.start(td.test_gridscan_outer_start_document) # type: ignore ispyb_cb.start(td.test_do_fgs_start_document) # type: ignore ispyb_cb.descriptor(td.test_descriptor_document_pre_data_collection) # type: ignore ispyb_cb.event(td.test_event_document_pre_data_collection) diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py index ae284f08b..375592822 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py @@ -15,7 +15,7 @@ ) from ..conftest import TestData -EXPECTED_DATA_COLLECTION_3D = { +EXPECTED_DATA_COLLECTION_3D_XY = { "visitid": TEST_SESSION_ID, "parentid": TEST_DATA_COLLECTION_GROUP_ID, "sampleid": TEST_SAMPLE_ID, @@ -27,9 +27,6 @@ "focal_spot_size_at_sampley": 0.0, "beamsize_at_samplex": 0.1, "beamsize_at_sampley": 0.1, - "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " - "bottom right (px): [3250,1700].", "data_collection_number": 0, "detector_distance": 100.0, "exp_time": 0.1, @@ -44,14 +41,19 @@ "wavelength": None, "xbeam": 150.0, "ybeam": 160.0, - "xtal_snapshot1": "test_1_y", - "xtal_snapshot2": "test_2_y", - "xtal_snapshot3": "test_3_y", "synchrotron_mode": None, "undulator_gap1": None, "starttime": EXPECTED_START_TIME, "filetemplate": "file_name_0_master.h5", - "nimages": 40 * 20, +} + +EXPECTED_DATA_COLLECTION_3D_XZ = EXPECTED_DATA_COLLECTION_3D_XY | { + "omegastart": 90, + "axis_range": 0, + "axisend": 90, + "axisstart": 90, + "data_collection_number": 1, + "filetemplate": "file_name_1_master.h5", } EXPECTED_DATA_COLLECTION_2D = { @@ -66,9 +68,6 @@ "focal_spot_size_at_sampley": 0.0, "beamsize_at_samplex": 0.1, "beamsize_at_sampley": 0.1, - "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " - "bottom right (px): [450,300].", "data_collection_number": 0, "detector_distance": 100.0, "exp_time": 0.1, @@ -83,14 +82,10 @@ "wavelength": None, "xbeam": 150.0, "ybeam": 160.0, - "xtal_snapshot1": "test_1_y", - "xtal_snapshot2": "test_2_y", - "xtal_snapshot3": "test_3_y", "synchrotron_mode": None, "undulator_gap1": None, "starttime": EXPECTED_START_TIME, "filetemplate": "file_name_0_master.h5", - "nimages": 40 * 20, } @@ -98,228 +93,337 @@ "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", new=MagicMock(return_value=EXPECTED_START_TIME), ) -def test_activity_gated_start_2d(mock_ispyb_conn): - callback = GridscanISPyBCallback() - callback.activity_gated_start( - TestData.test_gridscan2d_start_document # pyright: ignore - ) - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "parentid": TEST_SESSION_ID, - "experimenttype": "mesh", - "sampleid": TEST_SAMPLE_ID, - }, - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - EXPECTED_DATA_COLLECTION_2D, - ) - mx_acq.upsert_data_collection.update_dc_position.assert_not_called() - mx_acq.upsert_data_collection.upsert_dc_grid.assert_not_called() +class TestXrayCentreISPyBCallback: + def test_activity_gated_start_2d(self, mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan2d_start_document # pyright: ignore + ) + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + assert_upsert_call_with( + mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore + mx_acq.get_data_collection_group_params(), + { + "parentid": TEST_SESSION_ID, + "experimenttype": "mesh", + "sampleid": TEST_SAMPLE_ID, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + EXPECTED_DATA_COLLECTION_2D, + ) + mx_acq.upsert_data_collection.update_dc_position.assert_not_called() + mx_acq.upsert_data_collection.upsert_dc_grid.assert_not_called() + def test_hardware_read_event_2d(self, mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan2d_start_document # pyright: ignore + ) + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + mx_acq.upsert_data_collection_group.reset_mock() + mx_acq.upsert_data_collection.reset_mock() + callback.activity_gated_descriptor( + TestData.test_descriptor_document_pre_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_pre_data_collection) -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -def test_activity_gated_event_2d(mock_ispyb_conn): - callback = GridscanISPyBCallback() - callback.activity_gated_start( - TestData.test_gridscan2d_start_document # pyright: ignore - ) - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - mx_acq.upsert_data_collection_group.reset_mock() - mx_acq.upsert_data_collection.reset_mock() - callback.activity_gated_descriptor( - TestData.test_descriptor_document_pre_data_collection - ) - callback.activity_gated_event(TestData.test_event_document_pre_data_collection) - callback.activity_gated_descriptor( - TestData.test_descriptor_document_during_data_collection - ) - callback.activity_gated_event(TestData.test_event_document_during_data_collection) + assert_upsert_call_with( + mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore + mx_acq.get_data_collection_group_params(), + { + "id": TEST_DATA_COLLECTION_GROUP_ID, + "parentid": TEST_SESSION_ID, + "experimenttype": "mesh", + "sampleid": TEST_SAMPLE_ID, + "sample_barcode": "BARCODE", # deferred + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_IDS[0], + "slitgaphorizontal": 0.1234, + "slitgapvertical": 0.2345, + "synchrotronmode": "User", + "undulatorgap1": 1.234, + }, + ) + assert_upsert_call_with( + mx_acq.update_dc_position.mock_calls[0], + mx_acq.get_dc_position_params(), + { + "id": TEST_DATA_COLLECTION_IDS[0], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, + }, + ) + mx_acq.upsert_dc_grid.assert_not_called() - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "mesh", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": "BARCODE", # deferred - }, - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - EXPECTED_DATA_COLLECTION_2D - | { - "id": TEST_DATA_COLLECTION_IDS[0], - "slitgaphorizontal": 0.1234, - "slitgapvertical": 0.2345, - "synchrotronmode": "User", - "undulatorgap1": 1.234, - "wavelength": 1.1164718451643736, - "transmission": 100, - "flux": 10, - }, - ) - assert_upsert_call_with( - mx_acq.update_dc_position.mock_calls[0], - mx_acq.get_dc_position_params(), - { - "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": 0, - "pos_y": 0, - "pos_z": 0, - }, - ) - assert_upsert_call_with( - mx_acq.upsert_dc_grid.mock_calls[0], - mx_acq.get_dc_grid_params(), - { - "parentid": TEST_DATA_COLLECTION_IDS[0], - "dxinmm": 0.1, - "dyinmm": 0.1, - "stepsx": 40, - "stepsy": 20, - "micronsperpixelx": 10, - "micronsperpixely": 10, - "snapshotoffsetxpixel": 50, - "snapshotoffsetypixel": 100, - "orientation": "horizontal", - "snaked": True, - }, - ) + def test_flux_read_event_2d(self, mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan2d_start_document # pyright: ignore + ) + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + callback.activity_gated_descriptor( + TestData.test_descriptor_document_pre_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_pre_data_collection) + mx_acq.upsert_data_collection_group.reset_mock() + mx_acq.upsert_data_collection.reset_mock() + callback.activity_gated_descriptor( + TestData.test_descriptor_document_during_data_collection + ) + callback.activity_gated_event( + TestData.test_event_document_during_data_collection + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_IDS[0], + "wavelength": 1.1164718451643736, + "transmission": 100, + "flux": 10, + }, + ) + assert_upsert_call_with( + mx_acq.update_dc_position.mock_calls[0], + mx_acq.get_dc_position_params(), + { + "id": TEST_DATA_COLLECTION_IDS[0], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, + }, + ) + mx_acq.upsert_dc_grid.assert_not_called() -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -def test_activity_gated_start_3d(mock_ispyb_conn): - callback = GridscanISPyBCallback() - callback.activity_gated_start(TestData.test_start_document) # pyright: ignore - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "parentid": TEST_SESSION_ID, - "experimenttype": "Mesh3D", - "sampleid": TEST_SAMPLE_ID, - }, - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - EXPECTED_DATA_COLLECTION_3D, - ) - mx_acq.upsert_data_collection.update_dc_position.assert_not_called() - mx_acq.upsert_data_collection.upsert_dc_grid.assert_not_called() + def test_activity_gated_start_3d(self, mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan3d_start_document + ) # pyright: ignore + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + assert_upsert_call_with( + mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore + mx_acq.get_data_collection_group_params(), + { + "parentid": TEST_SESSION_ID, + "experimenttype": "Mesh3D", + "sampleid": TEST_SAMPLE_ID, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + EXPECTED_DATA_COLLECTION_3D_XY, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[1], + mx_acq.get_data_collection_params(), + EXPECTED_DATA_COLLECTION_3D_XZ, + ) + mx_acq.upsert_data_collection.update_dc_position.assert_not_called() + mx_acq.upsert_data_collection.upsert_dc_grid.assert_not_called() + def test_hardware_read_event_3d(self, mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan3d_start_document + ) # pyright: ignore + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + mx_acq.upsert_data_collection_group.reset_mock() + mx_acq.upsert_data_collection.reset_mock() + callback.activity_gated_descriptor( + TestData.test_descriptor_document_pre_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_pre_data_collection) + assert_upsert_call_with( + mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore + mx_acq.get_data_collection_group_params(), + { + "id": TEST_DATA_COLLECTION_GROUP_ID, + "parentid": TEST_SESSION_ID, + "experimenttype": "Mesh3D", + "sampleid": TEST_SAMPLE_ID, + "sample_barcode": "BARCODE", # deferred + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_IDS[0], + "slitgaphorizontal": 0.1234, + "slitgapvertical": 0.2345, + "synchrotronmode": "User", + "undulatorgap1": 1.234, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[1], + mx_acq.get_data_collection_params(), + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_IDS[1], + "slitgaphorizontal": 0.1234, + "slitgapvertical": 0.2345, + "synchrotronmode": "User", + "undulatorgap1": 1.234, + }, + ) -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -def test_activity_gated_event_3d(mock_ispyb_conn): - callback = GridscanISPyBCallback() - callback.activity_gated_start(TestData.test_start_document) # pyright: ignore - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - mx_acq.upsert_data_collection_group.reset_mock() - mx_acq.upsert_data_collection.reset_mock() - callback.activity_gated_descriptor( - TestData.test_descriptor_document_pre_data_collection - ) - callback.activity_gated_event(TestData.test_event_document_pre_data_collection) - callback.activity_gated_descriptor( - TestData.test_descriptor_document_during_data_collection - ) - callback.activity_gated_event(TestData.test_event_document_during_data_collection) + def test_flux_read_events_3d(self, mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan3d_start_document + ) # pyright: ignore + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + callback.activity_gated_descriptor( + TestData.test_descriptor_document_pre_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_pre_data_collection) + mx_acq.upsert_data_collection_group.reset_mock() + mx_acq.upsert_data_collection.reset_mock() + + callback.activity_gated_descriptor( + TestData.test_descriptor_document_during_data_collection + ) + callback.activity_gated_event( + TestData.test_event_document_during_data_collection + ) + + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_IDS[0], + "wavelength": 1.1164718451643736, + "transmission": 100, + "flux": 10, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[1], + mx_acq.get_data_collection_params(), + { + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_IDS[1], + "wavelength": 1.1164718451643736, + "transmission": 100, + "flux": 10, + }, + ) + assert_upsert_call_with( + mx_acq.update_dc_position.mock_calls[0], + mx_acq.get_dc_position_params(), + { + "id": TEST_DATA_COLLECTION_IDS[0], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, + }, + ) + assert_upsert_call_with( + mx_acq.update_dc_position.mock_calls[1], + mx_acq.get_dc_position_params(), + { + "id": TEST_DATA_COLLECTION_IDS[1], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, + }, + ) + mx_acq.upsert_dc_grid.assert_not_called() + + def test_activity_gated_event_oav_snapshot_triggered(self, mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan3d_start_document + ) # pyright: ignore + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + mx_acq.upsert_data_collection_group.reset_mock() + mx_acq.upsert_data_collection.reset_mock() + + callback.activity_gated_descriptor( + TestData.test_descriptor_document_oav_snapshot + ) + callback.activity_gated_event(TestData.test_event_document_oav_snapshot_xy) + callback.activity_gated_event(TestData.test_event_document_oav_snapshot_xz) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "Mesh3D", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": "BARCODE", # deferred - }, - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - EXPECTED_DATA_COLLECTION_3D - | { - "id": TEST_DATA_COLLECTION_IDS[0], - "slitgaphorizontal": 0.1234, - "slitgapvertical": 0.2345, - "synchrotronmode": "User", - "undulatorgap1": 1.234, - "wavelength": 1.1164718451643736, - "transmission": 100, - "flux": 10, - }, - ) - assert_upsert_call_with( - mx_acq.update_dc_position.mock_calls[0], - mx_acq.get_dc_position_params(), - { - "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": 0, - "pos_y": 0, - "pos_z": 0, - }, - ) - assert_upsert_call_with( - mx_acq.upsert_dc_grid.mock_calls[0], - mx_acq.get_dc_grid_params(), - { - "parentid": TEST_DATA_COLLECTION_IDS[0], - "dxinmm": 0.1, - "dyinmm": 0.1, - "stepsx": 40, - "stepsy": 20, - "micronsperpixelx": 1.25, - "micronsperpixely": 1.25, - "snapshotoffsetxpixel": 50, - "snapshotoffsetypixel": 100, - "orientation": "horizontal", - "snaked": True, - }, - ) - assert_upsert_call_with( - mx_acq.update_dc_position.mock_calls[1], - mx_acq.get_dc_position_params(), - { - "id": TEST_DATA_COLLECTION_IDS[1], - "pos_x": 0, - "pos_y": 0, - "pos_z": 0, - }, - ) - assert_upsert_call_with( - mx_acq.upsert_dc_grid.mock_calls[1], - mx_acq.get_dc_grid_params(), - { - "parentid": TEST_DATA_COLLECTION_IDS[1], - "dxinmm": 0.1, - "dyinmm": 0.1, - "stepsx": 40, - "stepsy": 10, - "micronsperpixelx": 1.25, - "micronsperpixely": 1.25, - "snapshotoffsetxpixel": 50, - "snapshotoffsetypixel": 0, - "orientation": "horizontal", - "snaked": True, - }, - ) + mx_acq.upsert_data_collection_group.assert_not_called() + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + { + "id": TEST_DATA_COLLECTION_IDS[0], + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "nimages": 40 * 20, + "xtal_snapshot1": "test_1_y", + "xtal_snapshot2": "test_2_y", + "xtal_snapshot3": "test_3_y", + "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " + "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " + "bottom right (px): [3250,1700].", + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[1], + mx_acq.get_data_collection_params(), + { + "id": TEST_DATA_COLLECTION_IDS[1], + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "nimages": 40 * 10, + "xtal_snapshot1": "test_1_z", + "xtal_snapshot2": "test_2_z", + "xtal_snapshot3": "test_3_z", + "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 10 " + "images in 100.0 um by 100.0 um steps. Top left (px): [50,0], " + "bottom right (px): [3250,800].", + }, + ) + assert_upsert_call_with( + mx_acq.upsert_dc_grid.mock_calls[0], + mx_acq.get_dc_grid_params(), + { + "parentid": TEST_DATA_COLLECTION_IDS[0], + "dxinmm": 0.1, + "dyinmm": 0.1, + "stepsx": 40, + "stepsy": 20, + "micronsperpixelx": 1.25, + "micronsperpixely": 1.25, + "snapshotoffsetxpixel": 50, + "snapshotoffsetypixel": 100, + "orientation": "horizontal", + "snaked": True, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_dc_grid.mock_calls[1], + mx_acq.get_dc_grid_params(), + { + "parentid": TEST_DATA_COLLECTION_IDS[1], + "dxinmm": 0.1, + "dyinmm": 0.1, + "stepsx": 40, + "stepsy": 10, + "micronsperpixelx": 1.25, + "micronsperpixely": 1.25, + "snapshotoffsetxpixel": 50, + "snapshotoffsetypixel": 0, + "orientation": "horizontal", + "snaked": True, + }, + ) diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_handler.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_handler.py index 1ba4c80c4..e8d8713ed 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_handler.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_handler.py @@ -53,7 +53,7 @@ def test_fgs_failing_results_in_bad_run_status_in_ispyb( self, ): ispyb_handler = GridscanISPyBCallback() - ispyb_handler.activity_gated_start(td.test_start_document) + ispyb_handler.activity_gated_start(td.test_gridscan3d_start_document) ispyb_handler.activity_gated_descriptor( td.test_descriptor_document_pre_data_collection ) @@ -80,7 +80,7 @@ def test_fgs_raising_no_exception_results_in_good_run_status_in_ispyb( self, ): ispyb_handler = GridscanISPyBCallback() - ispyb_handler.activity_gated_start(td.test_start_document) + ispyb_handler.activity_gated_start(td.test_gridscan3d_start_document) ispyb_handler.activity_gated_descriptor( td.test_descriptor_document_pre_data_collection ) @@ -114,7 +114,7 @@ def test_given_ispyb_callback_started_writing_to_ispyb_when_messages_logged_then gelf_handler.emit = MagicMock() ispyb_handler = GridscanISPyBCallback() - ispyb_handler.activity_gated_start(td.test_start_document) + ispyb_handler.activity_gated_start(td.test_gridscan3d_start_document) ispyb_handler.activity_gated_descriptor( td.test_descriptor_document_pre_data_collection ) @@ -141,7 +141,7 @@ def test_given_ispyb_callback_finished_writing_to_ispyb_when_messages_logged_the gelf_handler.emit = MagicMock() ispyb_handler = GridscanISPyBCallback() - ispyb_handler.activity_gated_start(td.test_start_document) + ispyb_handler.activity_gated_start(td.test_gridscan3d_start_document) ispyb_handler.activity_gated_descriptor( td.test_descriptor_document_pre_data_collection ) @@ -167,7 +167,9 @@ def test_given_fgs_plan_finished_when_zocalo_results_event_then_expected_comment ): ispyb_handler = GridscanISPyBCallback() - ispyb_handler.activity_gated_start(td.test_start_document) # type:ignore + ispyb_handler.activity_gated_start( + td.test_gridscan3d_start_document + ) # type:ignore ispyb_handler.activity_gated_start(td.test_do_fgs_start_document) # type:ignore ispyb_handler.activity_gated_stop(td.test_do_fgs_gridscan_stop_document) diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py index d26191940..8867c2a5d 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py @@ -1,13 +1,12 @@ from unittest.mock import MagicMock, patch -import numpy as np import pytest -from hyperion.external_interaction.callbacks.common.ispyb_mapping import GridScanInfo from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( construct_comment_for_gridscan, - populate_xy_data_collection_info, ) +from hyperion.external_interaction.ispyb.data_model import DataCollectionGridInfo +from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation from hyperion.parameters.plan_specific.gridscan_internal_params import ( GridscanInternalParameters, ) @@ -22,45 +21,22 @@ def dummy_params(): dummy_params = GridscanInternalParameters(**default_raw_params()) dummy_params.hyperion_params.ispyb_params.sample_id = TEST_SAMPLE_ID - dummy_params.hyperion_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x = 1.25 dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y = 1.25 dummy_params.hyperion_params.detector_params.run_number = 0 return dummy_params -def test_given_x_and_y_steps_different_from_total_images_when_grid_scan_stored_then_num_images_correct( - dummy_params: GridscanInternalParameters, -): - expected_number_of_steps = 200 * 3 - dummy_params.experiment_params.x_steps = 200 - dummy_params.experiment_params.y_steps = 3 - grid_scan_info = GridScanInfo( - dummy_params.hyperion_params.ispyb_params.upper_left, - 3, - dummy_params.experiment_params.y_step_size, - ) - actual = populate_xy_data_collection_info( - grid_scan_info, - dummy_params, - dummy_params.hyperion_params.ispyb_params, - dummy_params.hyperion_params.detector_params, - ) - - assert actual.n_images == expected_number_of_steps - - @patch("ispyb.open", autospec=True) def test_ispyb_deposition_rounds_position_to_int( mock_ispyb_conn: MagicMock, dummy_params, ): - dummy_params.hyperion_params.ispyb_params.upper_left = np.array([0.01, 100, 50]) - assert construct_comment_for_gridscan( - dummy_params, dummy_params.hyperion_params.ispyb_params, - GridScanInfo(dummy_params.hyperion_params.ispyb_params.upper_left, 20, 0.1), + DataCollectionGridInfo( + 0.1, 0.1, 40, 20, 1.25, 1.25, 0.01, 100, Orientation.HORIZONTAL, True # type: ignore + ), ) == ( "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images " "in 100.0 um by 100.0 um steps. Top left (px): [0,100], bottom right (px): [3200,1700]." @@ -89,22 +65,12 @@ def test_ispyb_deposition_rounds_box_size_int( raw, rounded, ): - dummy_params.experiment_params.x_steps = 0 - dummy_params.experiment_params.x_step_size = raw - grid_scan_info = GridScanInfo( - [ - 0, - 0, - 0, - ], - 0, - raw, + data_collection_grid_info = DataCollectionGridInfo( + raw, raw, 0, 0, 1.25, 1.25, 0, 0, Orientation.HORIZONTAL, True ) - bottom_right_from_top_left.return_value = grid_scan_info.upper_left + bottom_right_from_top_left.return_value = [0, 0] - assert construct_comment_for_gridscan( - dummy_params, MagicMock(), grid_scan_info - ) == ( + assert construct_comment_for_gridscan(MagicMock(), data_collection_grid_info) == ( "Hyperion: Xray centring - Diffraction grid scan of 0 by 0 images in " f"{rounded} um by {rounded} um steps. Top left (px): [0,0], bottom right (px): [0,0]." ) diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_nexus_handler.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_nexus_handler.py index 6078125ee..ddb4a8550 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_nexus_handler.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_nexus_handler.py @@ -37,7 +37,7 @@ def test_writers_dont_create_on_init_but_do_on_nexus_read_event( assert nexus_handler.nexus_writer_1 is None assert nexus_handler.nexus_writer_2 is None - nexus_handler.activity_gated_start(TestData.test_start_document) + nexus_handler.activity_gated_start(TestData.test_gridscan_outer_start_document) nexus_handler.activity_gated_descriptor( TestData.test_descriptor_document_nexus_read ) diff --git a/tests/unit_tests/external_interaction/ispyb/conftest.py b/tests/unit_tests/external_interaction/ispyb/conftest.py index 531a38195..142dfecc0 100644 --- a/tests/unit_tests/external_interaction/ispyb/conftest.py +++ b/tests/unit_tests/external_interaction/ispyb/conftest.py @@ -1,12 +1,12 @@ from copy import deepcopy -import numpy as np import pytest from hyperion.external_interaction.ispyb.data_model import ( DataCollectionGridInfo, DataCollectionPositionInfo, ExperimentType, + ScanDataInfo, ) from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation from hyperion.external_interaction.ispyb.ispyb_store import StoreInIspyb @@ -17,6 +17,7 @@ from ..conftest import ( TEST_DATA_COLLECTION_GROUP_ID, + TEST_DATA_COLLECTION_IDS, TEST_SAMPLE_ID, default_raw_params, ) @@ -26,7 +27,6 @@ def dummy_params(): dummy_params = GridscanInternalParameters(**default_raw_params()) dummy_params.hyperion_params.ispyb_params.sample_id = TEST_SAMPLE_ID - dummy_params.hyperion_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x = 1.25 dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y = 1.25 dummy_params.hyperion_params.detector_params.run_number = 0 @@ -51,7 +51,9 @@ def dummy_2d_gridscan_ispyb(): @pytest.fixture -def scan_xy_data_info_for_update(scan_data_info_for_begin): +def scan_xy_data_info_for_update( + scan_data_info_for_begin: ScanDataInfo, +) -> ScanDataInfo: scan_data_info_for_update = deepcopy(scan_data_info_for_begin) scan_data_info_for_update.data_collection_info.parent_id = ( TEST_DATA_COLLECTION_GROUP_ID @@ -73,4 +75,5 @@ def scan_xy_data_info_for_update(scan_data_info_for_begin): scan_data_info_for_update.data_collection_position_info = ( DataCollectionPositionInfo(0, 0, 0) ) + scan_data_info_for_update.data_collection_id = TEST_DATA_COLLECTION_IDS[0] return scan_data_info_for_update diff --git a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py b/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py index 328b0b18f..0495ad9ba 100644 --- a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py +++ b/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py @@ -134,7 +134,7 @@ def scan_data_infos_for_update(): undulator_gap1=1.0, start_time=EXPECTED_START_TIME, ), - data_collection_id=None, + data_collection_id=TEST_DATA_COLLECTION_IDS[0], data_collection_position_info=DataCollectionPositionInfo( pos_x=0, pos_y=0, pos_z=0 ), @@ -243,7 +243,7 @@ def test_ispyb_deposition_comment_for_3D_correct( mock_mx_aquisition = mx_acquisition_from_conn(mock_ispyb_conn) mock_upsert_dc = mock_mx_aquisition.upsert_data_collection ispyb_ids = dummy_3d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) dummy_3d_gridscan_ispyb.update_deposition( ispyb_ids, dummy_collection_group_info, scan_data_infos_for_update @@ -271,11 +271,12 @@ def test_store_3d_grid_scan( assert dummy_3d_gridscan_ispyb.experiment_type == "Mesh3D" ispyb_ids = dummy_3d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) assert ispyb_ids == IspybIds( data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, + data_collection_id=TEST_DATA_COLLECTION_IDS[0], ) assert dummy_3d_gridscan_ispyb.update_deposition( @@ -298,7 +299,7 @@ def test_begin_deposition( scan_data_info_for_begin, ): assert dummy_3d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) == IspybIds( data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, @@ -376,7 +377,7 @@ def test_update_deposition( scan_data_infos_for_update, ): ispyb_ids = dummy_3d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) mx_acq.upsert_data_collection_group.assert_called_once() @@ -577,7 +578,7 @@ def test_end_deposition_happy_path( scan_data_infos_for_update, ): ispyb_ids = dummy_3d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) ispyb_ids = dummy_3d_gridscan_ispyb.update_deposition( ispyb_ids, dummy_collection_group_info, scan_data_infos_for_update @@ -633,7 +634,7 @@ def test_param_keys( scan_xy_data_info_for_update, ): ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) assert dummy_2d_gridscan_ispyb.update_deposition( ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] @@ -655,7 +656,7 @@ def _test_when_grid_scan_stored_then_data_present_in_upserts( ): setup_mock_return_values(ispyb_conn) ispyb_ids = dummy_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) dummy_ispyb.update_deposition( ispyb_ids, dummy_collection_group_info, [scan_data_info_for_update] @@ -743,7 +744,7 @@ def test_fail_result_run_results_in_bad_run_status( mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] @@ -772,7 +773,7 @@ def test_no_exception_during_run_results_in_good_run_status( mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin + dummy_collection_group_info, [scan_data_info_for_begin] ) ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] diff --git a/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py b/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py index 28d887322..e9bf84c1e 100644 --- a/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py +++ b/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py @@ -199,7 +199,7 @@ def test_begin_deposition( assert scan_data_info_for_begin.data_collection_info.parent_id is None assert dummy_rotation_ispyb.begin_deposition( - dummy_rotation_data_collection_group_info, scan_data_info_for_begin + dummy_rotation_data_collection_group_info, [scan_data_info_for_begin] ) == IspybIds( data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, @@ -242,7 +242,7 @@ def test_begin_deposition_with_group_id_updates_but_doesnt_insert( ) assert dummy_rotation_ispyb.begin_deposition( - dummy_rotation_data_collection_group_info, scan_data_info_for_begin + dummy_rotation_data_collection_group_info, [scan_data_info_for_begin] ) == IspybIds( data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, @@ -283,7 +283,7 @@ def test_begin_deposition_with_alternate_experiment_type( dummy_rotation_data_collection_group_info.experiment_type = "Characterization" assert dummy_rotation_ispyb_with_experiment_type.begin_deposition( dummy_rotation_data_collection_group_info, - scan_data_info_for_begin, + [scan_data_info_for_begin], ) == IspybIds( data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, @@ -312,7 +312,7 @@ def test_update_deposition( scan_data_info_for_update, ): ispyb_ids = dummy_rotation_ispyb.begin_deposition( - dummy_rotation_data_collection_group_info, scan_data_info_for_begin + dummy_rotation_data_collection_group_info, [scan_data_info_for_begin] ) mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) mx_acq.upsert_data_collection_group.reset_mock() @@ -383,7 +383,7 @@ def test_update_deposition_with_group_id_updates( TEST_DATA_COLLECTION_GROUP_ID ) ispyb_ids = dummy_rotation_ispyb.begin_deposition( - dummy_rotation_data_collection_group_info, scan_data_info_for_begin + dummy_rotation_data_collection_group_info, [scan_data_info_for_begin] ) mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) mx_acq.upsert_data_collection_group.reset_mock() @@ -454,7 +454,7 @@ def test_end_deposition_happy_path( scan_data_info_for_update, ): ispyb_ids = dummy_rotation_ispyb.begin_deposition( - dummy_rotation_data_collection_group_info, scan_data_info_for_begin + dummy_rotation_data_collection_group_info, [scan_data_info_for_begin] ) scan_data_info_for_update.data_collection_info.parent_id = ( ispyb_ids.data_collection_group_id @@ -518,7 +518,7 @@ def test_store_rotation_scan_uses_supplied_dcgid( store_in_ispyb = StoreInIspyb(CONST.SIM.ISPYB_CONFIG, ExperimentType.ROTATION) scan_data_info_for_begin.data_collection_info.parent_id = dcgid ispyb_ids = store_in_ispyb.begin_deposition( - dummy_rotation_data_collection_group_info, scan_data_info_for_begin + dummy_rotation_data_collection_group_info, [scan_data_info_for_begin] ) assert ispyb_ids.data_collection_group_id == dcgid mx = mx_acquisition_from_conn(mock_ispyb_conn) diff --git a/tests/unit_tests/parameters/plan_specific/test_fgs_internal_parameters.py b/tests/unit_tests/parameters/plan_specific/test_fgs_internal_parameters.py index eaaa6d3a3..d10fedf1a 100644 --- a/tests/unit_tests/parameters/plan_specific/test_fgs_internal_parameters.py +++ b/tests/unit_tests/parameters/plan_specific/test_fgs_internal_parameters.py @@ -21,7 +21,6 @@ def test_FGS_parameters_load_from_file(): ispyb_params = internal_parameters.hyperion_params.ispyb_params np.testing.assert_array_equal(ispyb_params.position, np.array([10, 20, 30])) - np.testing.assert_array_equal(ispyb_params.upper_left, np.array([10, 20, 30])) detector_params = internal_parameters.hyperion_params.detector_params diff --git a/tests/unit_tests/parameters/plan_specific/test_grid_scan_with_edge_detect_internal_parameters.py b/tests/unit_tests/parameters/plan_specific/test_grid_scan_with_edge_detect_internal_parameters.py index 9fa3de707..37775d946 100644 --- a/tests/unit_tests/parameters/plan_specific/test_grid_scan_with_edge_detect_internal_parameters.py +++ b/tests/unit_tests/parameters/plan_specific/test_grid_scan_with_edge_detect_internal_parameters.py @@ -22,7 +22,6 @@ def test_grid_scan_with_edge_detect_parameters_load_from_file(): ispyb_params = internal_parameters.hyperion_params.ispyb_params np.testing.assert_array_equal(ispyb_params.position, np.array([10, 20, 30])) - np.testing.assert_array_equal(ispyb_params.upper_left, np.array([0, 0, 0])) detector_params = internal_parameters.hyperion_params.detector_params diff --git a/tests/unit_tests/parameters/test_internal_parameters.py b/tests/unit_tests/parameters/test_internal_parameters.py index 533dc56a7..3ce60e297 100644 --- a/tests/unit_tests/parameters/test_internal_parameters.py +++ b/tests/unit_tests/parameters/test_internal_parameters.py @@ -223,8 +223,6 @@ def test_param_fields_match_components_they_should_use( g_calculated_ispyb_param_keys == base_ispyb_annotation_keys + g_ispyb_annotation_keys ) - assert "upper_left" in g_ispyb_annotation_keys - assert "upper_left" not in r_ispyb_annotation_keys def test_internal_params_eq():