Fix broken system tests

src/hyperion/device_setup_plans/read_hardware_for_setup.py

@@ -8,6 +8,7 @@
 from dodal.devices.flux import Flux
 from dodal.devices.robot import BartRobot
 from dodal.devices.s4_slit_gaps import S4SlitGaps
+from dodal.devices.smargon import Smargon
 from dodal.devices.synchrotron import Synchrotron
 from dodal.devices.undulator import Undulator
 
@@ -21,6 +22,7 @@ def read_hardware_for_ispyb_pre_collection(
     s4_slit_gaps: S4SlitGaps,
     aperture_scatterguard: ApertureScatterguard,
     robot: BartRobot,
+    smargon: Smargon,
 ):
     LOGGER.info("Reading status of beamline for ispyb deposition, pre collection.")
     yield from bps.create(
@@ -31,6 +33,9 @@ def read_hardware_for_ispyb_pre_collection(
     yield from bps.read(s4_slit_gaps.xgap)
     yield from bps.read(s4_slit_gaps.ygap)
     yield from bps.read(aperture_scatterguard)
+    yield from bps.read(smargon.x)
+    yield from bps.read(smargon.y)
+    yield from bps.read(smargon.z)
     yield from bps.save()
 
 

src/hyperion/experiment_plans/flyscan_xray_centre_plan.py

@@ -237,6 +237,7 @@ def run_gridscan(
             fgs_composite.s4_slit_gaps,
             fgs_composite.aperture_scatterguard,
             fgs_composite.robot,
+            fgs_composite.smargon,
         )
         yield from read_hardware_for_ispyb_during_collection(
             fgs_composite.attenuator, fgs_composite.flux, fgs_composite.dcm

src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py

@@ -100,6 +100,7 @@ def run_gridscan(
             fgs_composite.s4_slit_gaps,
             fgs_composite.aperture_scatterguard,
             fgs_composite.robot,
+            fgs_composite.smargon,
         )
         yield from read_hardware_for_ispyb_during_collection(
             fgs_composite.attenuator, fgs_composite.flux, fgs_composite.dcm

src/hyperion/experiment_plans/rotation_scan_plan.py

@@ -212,6 +212,7 @@ def _rotation_scan_plan(
             composite.s4_slit_gaps,
             composite.aperture_scatterguard,
             composite.robot,
+            composite.smargon,
         )
         yield from read_hardware_for_ispyb_during_collection(
             composite.attenuator, composite.flux, composite.dcm

src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py

@@ -8,7 +8,6 @@
 from hyperion.external_interaction.ispyb.data_model import (
     DataCollectionGroupInfo,
     DataCollectionInfo,
-    DataCollectionPositionInfo,
 )
 from hyperion.external_interaction.ispyb.ispyb_dataclass import IspybParams
 from hyperion.external_interaction.ispyb.ispyb_store import (
@@ -32,16 +31,6 @@ def populate_data_collection_group(params: DiffractionExperimentWithSample):
     return dcg_info
 
 
-def populate_data_collection_position_info(ispyb_params):
-    # explicit cast to float because numpy int64, grrr...
-    dc_pos_info = DataCollectionPositionInfo(
-        float(ispyb_params.position[0]),
-        float(ispyb_params.position[1]),
-        float(ispyb_params.position[2]),
-    )
-    return dc_pos_info
-
-
 def populate_remaining_data_collection_info(
     comment,
     data_collection_group_id,

src/hyperion/external_interaction/callbacks/ispyb_callback_base.py

@@ -18,6 +18,7 @@
 from hyperion.external_interaction.ispyb.data_model import (
     DataCollectionGridInfo,
     DataCollectionInfo,
+    DataCollectionPositionInfo,
     ScanDataInfo,
 )
 from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation
@@ -123,8 +124,13 @@ def _handle_ispyb_hardware_read(self, doc) -> Sequence[ScanDataInfo]:
             slitgap_horizontal=doc["data"]["s4_slit_gaps_xgap"],
             slitgap_vertical=doc["data"]["s4_slit_gaps_ygap"],
         )
+        hwscan_position_info = DataCollectionPositionInfo(
+            pos_x=doc["data"]["smargon_x"],
+            pos_y=doc["data"]["smargon_y"],
+            pos_z=doc["data"]["smargon_z"],
+        )
         scan_data_infos = self.populate_info_for_update(
-            hwscan_data_collection_info, self.params
+            hwscan_data_collection_info, hwscan_position_info, self.params
         )
         ISPYB_LOGGER.info("Updating ispyb data collection after hardware read.")
         return scan_data_infos
@@ -193,7 +199,7 @@ def _handle_ispyb_transmission_flux_read(self, doc) -> Sequence[ScanDataInfo]:
                 self.params.detector_params.detector_distance,
             )
         scan_data_infos = self.populate_info_for_update(
-            hwscan_data_collection_info, self.params
+            hwscan_data_collection_info, None, self.params
         )
         ISPYB_LOGGER.info("Updating ispyb data collection after flux read.")
         return scan_data_infos
@@ -202,6 +208,7 @@ def _handle_ispyb_transmission_flux_read(self, doc) -> Sequence[ScanDataInfo]:
     def populate_info_for_update(
         self,
         event_sourced_data_collection_info: DataCollectionInfo,
+        event_sourced_position_info: Optional[DataCollectionPositionInfo],
         params: DiffractionExperimentWithSample,
     ) -> Sequence[ScanDataInfo]:
         pass

src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
-from typing import TYPE_CHECKING, Any, Callable, cast
+from typing import TYPE_CHECKING, Any, Callable, Optional
 
 from hyperion.external_interaction.callbacks.common.ispyb_mapping import (
     populate_data_collection_group,
-    populate_data_collection_position_info,
     populate_remaining_data_collection_info,
 )
 from hyperion.external_interaction.callbacks.ispyb_callback_base import (
@@ -16,6 +15,7 @@
 )
 from hyperion.external_interaction.ispyb.data_model import (
     DataCollectionInfo,
+    DataCollectionPositionInfo,
     ScanDataInfo,
 )
 from hyperion.external_interaction.ispyb.ispyb_store import (
@@ -102,19 +102,19 @@ def activity_gated_start(self, doc: RunStart):
         return super().activity_gated_start(doc)
 
     def populate_info_for_update(
-        self, event_sourced_data_collection_info: DataCollectionInfo, params
+        self,
+        event_sourced_data_collection_info: DataCollectionInfo,
+        event_sourced_position_info: Optional[DataCollectionPositionInfo],
+        params,
     ) -> Sequence[ScanDataInfo]:
         assert (
             self.ispyb_ids.data_collection_ids
         ), "Expect an existing DataCollection to update"
-        params = cast(RotationScan, params)
         return [
             ScanDataInfo(
                 data_collection_info=event_sourced_data_collection_info,
-                data_collection_position_info=populate_data_collection_position_info(
-                    params.ispyb_params
-                ),
                 data_collection_id=self.ispyb_ids.data_collection_ids[0],
+                data_collection_position_info=event_sourced_position_info,
             )
         ]
 

src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py

@@ -2,7 +2,7 @@
 
 from collections.abc import Sequence
 from time import time
-from typing import TYPE_CHECKING, Any, Callable, List
+from typing import TYPE_CHECKING, Any, Callable, List, Optional
 
 import numpy as np
 from blueapi.core import MsgGenerator
@@ -24,6 +24,7 @@
 from hyperion.external_interaction.exceptions import ISPyBDepositionNotMade
 from hyperion.external_interaction.ispyb.data_model import (
     DataCollectionInfo,
+    DataCollectionPositionInfo,
     ScanDataInfo,
 )
 from hyperion.external_interaction.ispyb.ispyb_store import (
@@ -165,6 +166,7 @@ def activity_gated_event(self, doc: Event):
     def populate_info_for_update(
         self,
         event_sourced_data_collection_info: DataCollectionInfo,
+        event_sourced_position_info: Optional[DataCollectionPositionInfo],
         params: ThreeDGridScan | GridScanWithEdgeDetect,
     ) -> Sequence[ScanDataInfo]:
         assert (

src/hyperion/parameters/components.py

@@ -119,11 +119,11 @@ def __hash__(self) -> int:
 
     @validator("parameter_model_version")
     def _validate_version(cls, version: ParameterVersion):
-        assert (
-            version >= ParameterVersion(major=PARAMETER_VERSION.major)
+        assert version >= ParameterVersion(
+            major=PARAMETER_VERSION.major
         ), f"Parameter version too old! This version of hyperion uses {PARAMETER_VERSION}"
-        assert (
-            version <= ParameterVersion(major=PARAMETER_VERSION.major + 1)
+        assert version <= ParameterVersion(
+            major=PARAMETER_VERSION.major + 1
         ), f"Parameter version too new! This version of hyperion uses {PARAMETER_VERSION}"
         return version
 
@@ -251,7 +251,6 @@ class Config:
         arbitrary_types_allowed = True
         extra = Extra.forbid
 
-    position: list[float] = None
     xtal_snapshots_omega_start: list[str] | None = None
     xtal_snapshots_omega_end: list[str] | None = None
     xtal_snapshots: list[str] | None = None

src/hyperion/parameters/rotation.py

@@ -2,7 +2,6 @@
 
 import os
 
-import numpy as np
 from dodal.devices.detector import DetectorParams
 from dodal.devices.detector.det_dist_to_beam_converter import (
     DetectorDistanceToBeamXYConverter,
@@ -79,7 +78,6 @@ def detector_params(self):
     def ispyb_params(self):  # pyright: ignore
         return RotationIspybParams(
             visit_path=str(self.visit_directory),
-            position=np.array(self.ispyb_extras.position),
             comment=self.comment,
             xtal_snapshots_omega_start=self.ispyb_extras.xtal_snapshots_omega_start,
             xtal_snapshots_omega_end=self.ispyb_extras.xtal_snapshots_omega_end,

tests/system_tests/experiment_plans/test_fgs_plan.py

@@ -137,36 +137,29 @@ def test_read_hardware_for_ispyb_pre_collection(
     RE: RunEngine,
     fxc_composite: FlyScanXRayCentreComposite,
 ):
-    undulator = fxc_composite.undulator
-    synchrotron = fxc_composite.synchrotron
-    slit_gaps = fxc_composite.s4_slit_gaps
-    attenuator = fxc_composite.attenuator
-    flux = fxc_composite.flux
-    dcm = fxc_composite.dcm
-    aperture_scatterguard = fxc_composite.aperture_scatterguard
-    robot = fxc_composite.robot
-
     @bpp.run_decorator()
-    def read_run(u, s, g, r, a, f, dcm, ap_sg):
+    def read_run(u, s, g, r, a, f, dcm, ap_sg, sm):
         yield from read_hardware_for_ispyb_pre_collection(
             undulator=u,
             synchrotron=s,
             s4_slit_gaps=g,
             aperture_scatterguard=ap_sg,
             robot=r,
+            smargon=sm,
         )
         yield from read_hardware_for_ispyb_during_collection(a, f, dcm)
 
     RE(
         read_run(
-            undulator,
-            synchrotron,
-            slit_gaps,
-            robot,
-            attenuator,
-            flux,
-            dcm,
-            aperture_scatterguard,
+            fxc_composite.undulator,
+            fxc_composite.synchrotron,
+            fxc_composite.s4_slit_gaps,
+            fxc_composite.robot,
+            fxc_composite.attenuator,
+            fxc_composite.flux,
+            fxc_composite.dcm,
+            fxc_composite.aperture_scatterguard,
+            fxc_composite.smargon,
         )
     )
 

tests/system_tests/experiment_plans/test_plan_system.py

@@ -27,20 +27,24 @@ async def test_getting_data_for_ispyb():
     aperture_scatterguard = ApertureScatterguard(
         prefix=f"{CONST.SIM.BEAMLINE}-AL-APSG-04:", name="ao_sg"
     )
+    smargon = i03.smargon(fake_with_ophyd_sim=True)
 
     undulator.wait_for_connection()
     await synchrotron.connect()
     slit_gaps.wait_for_connection()
     attenuator.wait_for_connection()
     flux.wait_for_connection()
     aperture_scatterguard.wait_for_connection()
+    smargon.wait_for_connection()
     robot = i03.robot(fake_with_ophyd_sim=True)
 
     RE = RunEngine()
 
     @bpp.run_decorator()
-    def standalone_read_hardware(und, syn, slits, robot, att, flux, ap_sg):
-        yield from read_hardware_for_ispyb_pre_collection(und, syn, slits, robot, ap_sg)
+    def standalone_read_hardware(und, syn, slits, robot, att, flux, ap_sg, sm):
+        yield from read_hardware_for_ispyb_pre_collection(
+            und, syn, slits, robot, ap_sg, smargon=sm
+        )
         yield from read_hardware_for_ispyb_during_collection(att, flux, dcm)
 
     RE(
@@ -52,5 +56,6 @@ def standalone_read_hardware(und, syn, slits, robot, att, flux, ap_sg):
             attenuator,
             flux,
             aperture_scatterguard,
+            smargon,
         )
     )

tests/system_tests/external_interaction/callbacks/test_external_callbacks.py

@@ -5,6 +5,7 @@
 import signal
 import subprocess
 import threading
+from genericpath import isfile
 from time import sleep
 from unittest.mock import MagicMock, patch
 
@@ -17,7 +18,6 @@
 from bluesky.callbacks.zmq import Publisher
 from bluesky.run_engine import RunEngine
 from dodal.devices.zocalo import ZocaloResults
-from genericpath import isfile
 from zmq.utils.monitor import recv_monitor_message
 
 from hyperion.experiment_plans.flyscan_xray_centre_plan import (

tests/system_tests/external_interaction/test_ispyb_dev_connection.py

@@ -845,7 +845,7 @@ def test_ispyb_deposition_in_rotation_plan(
     position_id = fetch_datacollection_attribute(
         dcid, DATA_COLLECTION_COLUMN_MAP["positionid"]
     )
-    expected_values = {"posX": 10.0, "posY": 20.0, "posZ": 30.0}
+    expected_values = {"posX": 1.0, "posY": 2.0, "posZ": 3.0}
     compare_actual_and_expected(
         position_id, expected_values, fetch_datacollection_position_attribute
     )

tests/system_tests/external_interaction/test_nexgen.py

@@ -156,7 +156,7 @@ def _fake_rotation_scan(
     @bpp.run_decorator(  # attach experiment metadata to the start document
         md={
             "subplan_name": CONST.PLAN.ROTATION_OUTER,
-            "hyperion_internal_parameters": parameters.json(),
+            "hyperion_parameters": parameters.json(),
             "activate_callbacks": "RotationNexusFileCallback",
         }
     )

tests/test_data/nexus_files/rotation/ins_8_5_expected_output.txt

@@ -42,8 +42,8 @@
  rotation axis = "OMEGA"
 
  Sweep-1    :
-   from image      1 : Omega=    0.000 ..    0.100  Kappa=    0.000 ..    0.000  Chi=    0.000 ..    0.000  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
-   to   image   3600 : Omega=  359.900 ..  360.000  Kappa=    0.000 ..    0.000  Chi=    0.000 ..    0.000  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
+   from image      1 : Omega=    0.000 ..    0.100  Kappa=    0.000 ..    0.000  Chi=   23.850 ..   23.850  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
+   to   image   3600 : Omega=  359.900 ..  360.000  Kappa=    0.000 ..    0.000  Chi=   23.850 ..   23.850  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
      Image number 1/3600
 
 
@@ -56,7 +56,7 @@
  Phi-angle                  [degree] = 0.00000
  Omega-angle (start, end)   [degree] = 0.00000 0.10000
  Oscillation-angle in Omega [degree] = 0.10000
- Chi-angle                  [degree] = 0.00000
+ Chi-angle                  [degree] = 23.85000
  Pixel size in X                [mm] = 0.075000
  Pixel size in Y                [mm] = 0.075000
  Number of pixels in X               = 4148

tests/test_data/nexus_files/rotation_unicode_metafile/ins_8_5_expected_output.txt

@@ -42,8 +42,8 @@
  rotation axis = "OMEGA"
 
  Sweep-1    :
-   from image      1 : Omega=    0.000 ..    0.100  Kappa=    0.000 ..    0.000  Chi=    0.000 ..    0.000  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
-   to   image   3600 : Omega=  359.900 ..  360.000  Kappa=    0.000 ..    0.000  Chi=    0.000 ..    0.000  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
+   from image      1 : Omega=    0.000 ..    0.100  Kappa=    0.000 ..    0.000  Chi=   23.850 ..   23.850  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
+   to   image   3600 : Omega=  359.900 ..  360.000  Kappa=    0.000 ..    0.000  Chi=   23.850 ..   23.850  Phi=    0.000 ..    0.000  2-Theta=    0.000 ..    0.000
      Image number 1/3600
 
 
@@ -56,7 +56,7 @@
  Phi-angle                  [degree] = 0.00000
  Omega-angle (start, end)   [degree] = 0.00000 0.10000
  Oscillation-angle in Omega [degree] = 0.10000
- Chi-angle                  [degree] = 0.00000
+ Chi-angle                  [degree] = 23.85000
  Pixel size in X                [mm] = 0.075000
  Pixel size in Y                [mm] = 0.075000
  Number of pixels in X               = 4148

tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json

@@ -26,11 +26,6 @@
     "y_start_um": 2.0,
     "z_start_um": 3.0,
     "ispyb_extras": {
-        "position": [
-            10.0,
-            20.0,
-            30.0
-        ],
         "xtal_snapshots_omega_start": [
             "test_1_y",
             "test_2_y",