diff --git a/.github/actions/verify-nexus/Dockerfile b/.github/actions/verify-nexus/Dockerfile new file mode 100644 index 000000000..02d402f5e --- /dev/null +++ b/.github/actions/verify-nexus/Dockerfile @@ -0,0 +1,3 @@ +FROM ghcr.io/githubgphl/imginfo:main +COPY entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/.github/actions/verify-nexus/action.yml b/.github/actions/verify-nexus/action.yml new file mode 100644 index 000000000..0a9e66765 --- /dev/null +++ b/.github/actions/verify-nexus/action.yml @@ -0,0 +1,18 @@ +name: "Verify nexus" +description: "Verify nexus files against imginfo" +inputs: + filename: + description: "nexus file to verify" + required: true +outputs: + imginfo_stdout: + description: "imginfo output" + imginfo_stderr: + description: "imginfo error output" + imginfo_exit_code: + description: "imginfo exit code" +runs: + using: "docker" + image: "Dockerfile" + args: + - ${{ inputs.filename }} diff --git a/.github/actions/verify-nexus/entrypoint.sh b/.github/actions/verify-nexus/entrypoint.sh new file mode 100755 index 000000000..7c9524d55 --- /dev/null +++ b/.github/actions/verify-nexus/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh -l +echo "Contents of /github/workspace:" +ls /github/workspace +echo "current dir: $(pwd)" +echo "Running imginfo on $1" +/imginfo/imginfo $1 >> imginfo_out_file 2>> imginfo_err_file +{ echo "imginfo_output<> "$GITHUB_OUTPUT" +{ echo "imginfo_output<> "$GITHUB_OUTPUT" +echo "imginfo_exit_code=$?" >> "$GITHUB_OUTPUT" +echo "------------- IMGINFO STDOUT -------------" +cat imginfo_out_file +echo "------------- IMGINFO STDERR -------------" +cat imginfo_err_file +echo "------------------------------------------" +if [ -s imginfo_err_file ]; then + echo "ERRORS IN IMGINFO PROCESSING!" + exit 1 +fi +echo "VALIDATED SUCCESSFULLY!" \ No newline at end of file diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index f3d093600..5901759a0 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -56,3 +56,25 @@ jobs: with: name: ${{ matrix.python }}/${{ matrix.os }} files: cov.xml + + - name: Prepare test data for reference nexus + run: hyperion-populate-test-and-meta-files + + - name: Run imginfo on reference nexus + uses: ./.github/actions/verify-nexus + id: verify_reference_nexus + with: + filename: "tests/test_data/nexus_files/rotation/ins_8_5.nxs" + + # ugly hack because we get double free error on exit + - name: Generate test nexus files + run: hyperion-generate-test-nexus + - name: Report test nexus files + run: echo "filename=$(cat OUTPUT_FILENAME)" >> $GITHUB_OUTPUT + id: generated_nexus + + - name: Run imginfo on generated nexus + uses: ./.github/actions/verify-nexus + id: verify_generated_nexus + with: + filename: ${{ steps.generated_nexus.outputs.filename }} diff --git a/.github/workflows/test_data/setup.cfg b/.github/workflows/test_data/setup.cfg index 327f5227f..0e5333af0 100644 --- a/.github/workflows/test_data/setup.cfg +++ b/.github/workflows/test_data/setup.cfg @@ -44,6 +44,7 @@ install_requires = console_scripts = hyperion = hyperion.__main__:main hyperion-callbacks = hyperion.external_interaction.callbacks.__main__:main + hyperion-generate-test-nexus = hyperion.utils.validation:generate_test_nexus [options.extras_require] dev = diff --git a/setup.cfg b/setup.cfg index dcad6929a..499a2dec7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,9 +51,13 @@ install_requires = console_scripts = hyperion = hyperion.__main__:main hyperion-callbacks = hyperion.external_interaction.callbacks.__main__:main + hyperion-generate-test-nexus = hyperion.utils.validation:generate_test_nexus + hyperion-populate-test-and-meta-files = hyperion.utils.validation:copy_test_meta_data_files + [options.extras_require] dev = + ophyd-async GitPython black pytest-cov diff --git a/src/hyperion/utils/validation.py b/src/hyperion/utils/validation.py new file mode 100644 index 000000000..1c0206692 --- /dev/null +++ b/src/hyperion/utils/validation.py @@ -0,0 +1,185 @@ +import gzip +import json +import os +import shutil +from pathlib import Path +from unittest.mock import MagicMock, patch + +import bluesky.preprocessors as bpp +from bluesky.run_engine import RunEngine +from dodal.beamlines import i03 +from ophyd.status import Status +from ophyd_async.core import set_mock_value + +from hyperion.device_setup_plans.read_hardware_for_setup import ( + read_hardware_for_ispyb_during_collection, + read_hardware_for_nexus_writer, +) +from hyperion.experiment_plans.rotation_scan_plan import RotationScanComposite +from hyperion.external_interaction.callbacks.rotation.nexus_callback import ( + RotationNexusFileCallback, +) +from hyperion.parameters.constants import CONST +from hyperion.parameters.rotation import RotationScan + +TEST_DATA_DIRECTORY = Path("tests/test_data/nexus_files/rotation") +TEST_METAFILE = "ins_8_5_meta.h5.gz" +FAKE_DATAFILE = "../fake_data.h5" +FILENAME_STUB = "test_rotation_nexus" + + +def test_params(filename_stub, dir): + def get_params(filename): + with open(filename) as f: + return json.loads(f.read()) + + params = RotationScan( + **get_params( + "tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json" + ) + ) + params.file_name = filename_stub + params.scan_width_deg = 360 + params.demand_energy_ev = 12700 + params.storage_directory = str(dir) + params.x_start_um = 0 + params.y_start_um = 0 + params.z_start_um = 0 + params.exposure_time_s = 0.004 + return params + + +def fake_rotation_scan( + parameters: RotationScan, + subscription: RotationNexusFileCallback, + rotation_devices: RotationScanComposite, +): + @bpp.subs_decorator(subscription) + @bpp.set_run_key_decorator("rotation_scan_with_cleanup_and_subs") + @bpp.run_decorator( # attach experiment metadata to the start document + md={ + "subplan_name": CONST.PLAN.ROTATION_OUTER, + "hyperion_parameters": parameters.json(), + "activate_callbacks": "RotationNexusFileCallback", + } + ) + def plan(): + yield from read_hardware_for_ispyb_during_collection( + rotation_devices.attenuator, rotation_devices.flux, rotation_devices.dcm + ) + yield from read_hardware_for_nexus_writer(rotation_devices.eiger) + + return plan() + + +def fake_create_rotation_devices(): + eiger = i03.eiger(fake_with_ophyd_sim=True) + smargon = i03.smargon(fake_with_ophyd_sim=True) + zebra = i03.zebra(fake_with_ophyd_sim=True) + detector_motion = i03.detector_motion(fake_with_ophyd_sim=True) + backlight = i03.backlight(fake_with_ophyd_sim=True) + attenuator = i03.attenuator(fake_with_ophyd_sim=True) + flux = i03.flux(fake_with_ophyd_sim=True) + undulator = i03.undulator(fake_with_ophyd_sim=True) + aperture_scatterguard = i03.aperture_scatterguard(fake_with_ophyd_sim=True) + synchrotron = i03.synchrotron(fake_with_ophyd_sim=True) + s4_slit_gaps = i03.s4_slit_gaps(fake_with_ophyd_sim=True) + dcm = i03.dcm(fake_with_ophyd_sim=True) + robot = i03.robot(fake_with_ophyd_sim=True) + mock_omega_sets = MagicMock(return_value=Status(done=True, success=True)) + mock_omega_velocity_sets = MagicMock(return_value=Status(done=True, success=True)) + + smargon.omega.velocity.set = mock_omega_velocity_sets + smargon.omega.set = mock_omega_sets + + smargon.omega.max_velocity.sim_put(131) # type: ignore + + set_mock_value(dcm.energy_in_kev.user_readback, 12700) + + return RotationScanComposite( + attenuator=attenuator, + backlight=backlight, + dcm=dcm, + detector_motion=detector_motion, + eiger=eiger, + flux=flux, + smargon=smargon, + undulator=undulator, + aperture_scatterguard=aperture_scatterguard, + synchrotron=synchrotron, + s4_slit_gaps=s4_slit_gaps, + zebra=zebra, + robot=robot, + ) + + +def sim_rotation_scan_to_create_nexus( + test_params: RotationScan, + fake_create_rotation_devices: RotationScanComposite, + filename_stub, + RE, +): + run_number = test_params.detector_params.run_number + nexus_filename = f"{filename_stub}_{run_number}.nxs" + + fake_create_rotation_devices.eiger.bit_depth.sim_put(32) # type: ignore + + with patch( + "hyperion.external_interaction.nexus.write_nexus.get_start_and_predicted_end_time", + return_value=("test_time", "test_time"), + ): + RE( + fake_rotation_scan( + test_params, RotationNexusFileCallback(), fake_create_rotation_devices + ) + ) + + nexus_path = Path(test_params.storage_directory) / nexus_filename + assert os.path.isfile(nexus_path) + return filename_stub, run_number + + +def extract_metafile(input_filename, output_filename): + with gzip.open(input_filename) as metafile_fo: + with open(output_filename, "wb") as output_fo: + output_fo.write(metafile_fo.read()) + + +def _generate_fake_nexus(filename, dir=os.getcwd()): + RE = RunEngine({}) + params = test_params(filename, dir) + run_number = params.detector_params.run_number + filename_stub, run_number = sim_rotation_scan_to_create_nexus( + params, fake_create_rotation_devices(), filename, RE + ) + return filename_stub, run_number + + +def generate_test_nexus(): + filename_stub, run_number = _generate_fake_nexus(FILENAME_STUB) + # ugly hack because we get double free error on exit + with open("OUTPUT_FILENAME", "x") as f: + f.write(f"{filename_stub}_{run_number}.nxs") + + extract_metafile( + str(TEST_DATA_DIRECTORY / TEST_METAFILE), + f"{FILENAME_STUB}_{run_number}_meta.h5", + ) + + new_hyp_data = [f"{FILENAME_STUB}_{run_number}_00000{n}.h5" for n in [1, 2, 3, 4]] + [shutil.copy(TEST_DATA_DIRECTORY / FAKE_DATAFILE, d) for d in new_hyp_data] + + exit(0) + + +def copy_test_meta_data_files(): + extract_metafile( + str(TEST_DATA_DIRECTORY / TEST_METAFILE), + f"{TEST_DATA_DIRECTORY}/ins_8_5_meta.h5", + ) + new_data = [f"{TEST_DATA_DIRECTORY}/ins_8_5_00000{n}.h5" for n in [1, 2, 3, 4]] + [shutil.copy(TEST_DATA_DIRECTORY / FAKE_DATAFILE, d) for d in new_data] + + +if __name__ == "__main__": + generate_test_nexus() diff --git a/tests/test_data/nexus_files/fake_data.h5 b/tests/test_data/nexus_files/fake_data.h5 new file mode 100644 index 000000000..afd7b3ac4 Binary files /dev/null and b/tests/test_data/nexus_files/fake_data.h5 differ diff --git a/tests/test_data/nexus_files/rotation/ins_8_5_meta.h5.gz b/tests/test_data/nexus_files/rotation/ins_8_5_meta.h5.gz index abef1eb2e..3ec379dbc 100644 Binary files a/tests/test_data/nexus_files/rotation/ins_8_5_meta.h5.gz and b/tests/test_data/nexus_files/rotation/ins_8_5_meta.h5.gz differ diff --git a/tests/unit_tests/external_interaction/nexus/__init__.py b/tests/unit_tests/external_interaction/nexus/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit_tests/external_interaction/nexus/test_compare_nexus_to_gda_exhaustively.py b/tests/unit_tests/external_interaction/nexus/test_compare_nexus_to_gda_exhaustively.py new file mode 100644 index 000000000..c1914532d --- /dev/null +++ b/tests/unit_tests/external_interaction/nexus/test_compare_nexus_to_gda_exhaustively.py @@ -0,0 +1,271 @@ +import shutil +from pathlib import Path +from unittest.mock import MagicMock + +import h5py +import pytest +from h5py import Dataset, Datatype, File, Group +from numpy import dtype + +from hyperion.utils.validation import _generate_fake_nexus + +from ....conftest import extract_metafile + +TEST_DATA_DIRECTORY = Path("tests/test_data/nexus_files/rotation") +TEST_EXAMPLE_NEXUS_FILE = Path("ins_8_5.nxs") +TEST_NEXUS_FILENAME = "rotation_scan_test_nexus" +TEST_METAFILE = "ins_8_5_meta.h5.gz" +FAKE_DATAFILE = "../fake_data.h5" + +h5item = Group | Dataset | File | Datatype + + +def get_groups(dataset: h5py.File) -> set: + e = set() + + def add_layer(s: set, d: h5item): + if isinstance(d, h5py.Group): + for k in d: + s.add(d.name) + add_layer(s, d[k]) + + add_layer(e, dataset) + return e + + +def has_equiv_in(item: str, groups: set, exception_table: dict[str, set[str]]): + if item not in exception_table.keys(): + return False + # one of the items in exception_table[item] must be in the tested groups + return exception_table[item] & groups != set() + + +def test_has_equiv_in(): + test_table = {"a": {"b", "c"}} + assert not has_equiv_in("a", {"x", "y", "z"}, test_table) + assert has_equiv_in("a", {"x", "y", "c"}, test_table) + + +FilesAndgroups = tuple[h5py.File, set[str], h5py.File, set[str]] + + +@pytest.fixture +def files_and_groups(tmpdir): + filename, run_number = _generate_fake_nexus(TEST_NEXUS_FILENAME, tmpdir) + extract_metafile( + str(TEST_DATA_DIRECTORY / TEST_METAFILE), + f"{tmpdir}/{filename}_{run_number}_meta.h5", + ) + extract_metafile( + str(TEST_DATA_DIRECTORY / TEST_METAFILE), + f"{tmpdir}/ins_8_5_meta.h5", + ) + new_hyperion_master = tmpdir / f"{filename}_{run_number}.nxs" + new_gda_master = tmpdir / TEST_EXAMPLE_NEXUS_FILE + new_gda_data = [tmpdir / f"ins_8_5_00000{n}.h5" for n in [1, 2, 3, 4]] + new_hyp_data = [ + tmpdir / f"{filename}_{run_number}_00000{n}.h5" for n in [1, 2, 3, 4] + ] + shutil.copy(TEST_DATA_DIRECTORY / TEST_EXAMPLE_NEXUS_FILE, new_gda_master) + [shutil.copy(TEST_DATA_DIRECTORY / FAKE_DATAFILE, d) for d in new_gda_data] + [shutil.copy(TEST_DATA_DIRECTORY / FAKE_DATAFILE, d) for d in new_hyp_data] + + with ( + h5py.File(new_gda_master, "r") as example_nexus, + h5py.File(new_hyperion_master, "r") as hyperion_nexus, + ): + yield ( + example_nexus, + get_groups(example_nexus), + hyperion_nexus, + get_groups(hyperion_nexus), + ) + + +GROUPS_EQUIVALENTS_TABLE = { + "/entry/instrument/source": {"/entry/source"}, + "/entry/instrument/detector_z": {"/entry/instrument/detector/detector_z"}, + "/entry/instrument/transformations": {"/entry/instrument/detector/transformations"}, +} +GROUPS_EXCEPTIONS = {"/entry/instrument/attenuator"} + + +def test_hyperion_rotation_nexus_groups_against_gda( + files_and_groups: FilesAndgroups, +): + _, gda_groups, _, hyperion_groups = files_and_groups + for item in gda_groups: + assert ( + item in hyperion_groups + or has_equiv_in(item, hyperion_groups, GROUPS_EQUIVALENTS_TABLE) + or item in GROUPS_EXCEPTIONS + ) + + +DATATYPE_EXCEPTION_TABLE = { + "/entry/instrument/detector/bit_depth_image": ( + dtype("int64"), + "gda item bit_depth_image not present", + ), + "/entry/instrument/detector/depends_on": (dtype("S48"), dtype("S1024")), + "/entry/instrument/detector/description": (dtype("S9"), dtype("S1024")), + "/entry/instrument/detector/detector_readout_time": ( + dtype("int64"), + "gda item detector_readout_time not present", + ), + "/entry/instrument/detector/distance": ( + dtype(" int: copyfile(inputfile, tmpfile) with h5py.File(tmpfile, "r+") as metafile: del metafile["flatfield"] + metafile.create_dataset("flatfield", (4362, 4148), "