Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Removing functions from processing notebooks #251

Merged
merged 19 commits into from
Mar 13, 2024
Merged

Removing functions from processing notebooks #251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
eb703cb
More sensible default dstol
jadball Mar 6, 2024
2512974
Merge branch 'scanbooks' of github.com:jadball/ImageD11 into scanbooks
jadball Mar 6, 2024
c184acd
First attempt at de-functioning notebooks
jadball Mar 7, 2024
b2c1685
nb_utils bugfixes
jadball Mar 7, 2024
9ae55b6
Fix plot title bug
jadball Mar 7, 2024
5433c80
Segment notebook defunctionized
jadball Mar 12, 2024
b5df8b9
Index notebook defunctionized
jadball Mar 12, 2024
2a697e8
Index minor phase notebook defunctionized
jadball Mar 12, 2024
f2cb146
Sinogram map notebook defunctionized
jadball Mar 12, 2024
0bd8474
Sinogram map minor phase notebook defunctionized
jadball Mar 12, 2024
dfb6bd1
PBP indexing notebook defunctionized
jadball Mar 12, 2024
59b26ff
Later plotting notebooks defunctionized
jadball Mar 12, 2024
c4833e9
Typo fixing
jadball Mar 12, 2024
cfbfbbd
Add spline, mask, bg to dataset
jadball Mar 12, 2024
aa318ea
Add mask support to frelon peaksearch
jadball Mar 12, 2024
c2f50af
Defunctionize box-beam notebooks
jadball Mar 12, 2024
19ba9f1
Correct hardcoded ImageD11 code path in MLEM runner
jadball Mar 12, 2024
132e1b2
Hopefully Python 2 compat
jadball Mar 12, 2024
9a532e5
Small cleaup of nb_utils
jadball Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 66 additions & 20 deletions ImageD11/nbGui/3DXRD/0_3DXRD_segment_frelon.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,13 +152,27 @@
},
"outputs": [],
"source": [
"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA\n",
"\n",
"### USER: specify your experimental directory\n",
"\n",
"rawdata_path = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/RAW_DATA\"\n",
"rawdata_path = \"/data/visitor/ihma439/id11/20231211/RAW_DATA\"\n",
"\n",
"!ls -lrt {rawdata_path}\n",
"\n",
"processed_data_root_dir = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226\" # USER: modify this to change the destination folder if desired"
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA\n",
"\n",
"### USER: specify your experimental directory\n",
"\n",
"rawdata_path = \"/data/visitor/ihma439/id11/20231211/RAW_DATA\"\n",
"\n",
"!ls -lrt {rawdata_path}\n",
"\n",
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\""
]
},
{
Expand Down Expand Up @@ -211,9 +225,24 @@
},
"outputs": [],
"source": [
"# USER: specify path to background file\n",
"# USER: specify path to background and mask file\n",
"\n",
"bg_file = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/ff_bkg.edf\""
"bg_file = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/ff_bkg.edf\"\n",
"maskfile = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/mask.edf'"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e853b7db-1d0b-4504-a6a6-e6b38c5ff32d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ds.splinefile = spline_file\n",
"ds.maskfile = maskfile\n",
"ds.bgfile = bg_file"
]
},
{
Expand All @@ -227,7 +256,8 @@
"source": [
"#Define the initial parameters\n",
"start_worker_args = {\n",
" \"bgfile\":bg_file,\n",
" \"bgfile\":ds.bgfile,\n",
" \"maskfile\":ds.maskfile,\n",
" \"threshold\":50,\n",
" \"smoothsigma\":1.0,\n",
" \"bgc\":0.9,\n",
Expand Down Expand Up @@ -274,7 +304,8 @@
"\n",
"\n",
"def update(threshold, smoothsigma, bgc, minpx, m_offset_thresh, m_ratio_thresh):\n",
" image_worker = worker(bg_file,\n",
" image_worker = worker(ds.bgfile,\n",
" ds.maskfile,\n",
" threshold,\n",
" smoothsigma,\n",
" bgc,\n",
Expand Down Expand Up @@ -310,7 +341,8 @@
"outputs": [],
"source": [
"end_worker_args = {\n",
" \"bgfile\":bg_file,\n",
" \"bgfile\":ds.bgfile,\n",
" \"maskfile\":ds.maskfile,\n",
" \"threshold\":thresh_slider.value,\n",
" \"smoothsigma\":smsig_slider.value,\n",
" \"bgc\":bgc_slider.value,\n",
Expand Down Expand Up @@ -345,7 +377,7 @@
"\n",
"nthreads = len(os.sched_getaffinity(os.getpid()))\n",
"\n",
"cf_2d, cf_3d = process(ds, bg_file, nthreads-1, end_worker_args)"
"cf_2d, cf_3d = process(ds, nthreads-1, end_worker_args)"
]
},
{
Expand Down Expand Up @@ -421,7 +453,7 @@
},
"outputs": [],
"source": [
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226/Fe_tdxrd_refined.par'"
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/SCRIPTS/James/3DXRD/Fe_tdxrd_refined.par'"
]
},
{
Expand Down Expand Up @@ -453,6 +485,19 @@
"ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7085183c-1991-49b3-af09-abe119542166",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ds.parfile = parfile\n",
"ds.save()"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -472,7 +517,7 @@
"metadata": {},
"outputs": [],
"source": [
"# Now that we're happy with our indexing parameters, we can run the below cell to do this in bulk for many samples/datasets\n",
"# Now that weparfile happy with our indexing parameters, we can run the below cell to do this in bulk for many samples/datasets\n",
"# by default this will do all samples in sample_list, all datasets with a prefix of dset_prefix\n",
"# you can add samples and datasets to skip in skips_dict\n",
"\n",
Expand All @@ -489,12 +534,6 @@
"# manual override:\n",
"# samples_dict = {\"FeAu_0p5_tR_nscope\": [\"top_100um\", \"top_200um\"]}\n",
"\n",
"mask_path = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/mask.edf'\n",
"\n",
"spline_file = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/frelon36.spline'\n",
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/fitted.par'\n",
"bg_file = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/ff_bkg.edf\"\n",
"\n",
"worker_args = end_worker_args\n",
"\n",
"nthreads = len(os.sched_getaffinity(os.getpid()))\n",
Expand All @@ -518,13 +557,17 @@
" ds.import_all(scans=[\"1.1\"])\n",
" print(f\"I have a DataSet {ds.dset} in sample {ds.sample}\")\n",
" ds.save()\n",
" \n",
" ds.splinefile = spline_file\n",
" ds.maskfile = maskfile\n",
" ds.bgfile = bg_file\n",
"\n",
" print(\"Peaksearching\")\n",
" cf_2d, cf_3d = process(ds, bg_file, nthreads-1, worker_args)\n",
" cf_2d, cf_3d = process(ds, nthreads-1, worker_args)\n",
" \n",
" print(\"Spatially correcting peaks\")\n",
" cf_2d = utils.apply_spatial(cf_2d, spline_file, nthreads)\n",
" cf_3d = utils.apply_spatial(cf_3d, spline_file, nthreads)\n",
" cf_2d = utils.apply_spatial_lut(cf_2d, spline_file)\n",
" cf_3d = utils.apply_spatial_lut(cf_3d, spline_file)\n",
" \n",
" print(\"Saving peaks to file\")\n",
" cf_2d.parameters.loadparameters(parfile)\n",
Expand All @@ -534,7 +577,10 @@
" \n",
" cf_3d.parameters.loadparameters(parfile)\n",
" cf_3d.updateGeometry()\n",
" ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile)"
" ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile)\n",
" \n",
" ds.parfile = parfile\n",
" ds.save()"
]
},
{
Expand Down
50 changes: 25 additions & 25 deletions ImageD11/nbGui/3DXRD/1_3DXRD_refine_parameters.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,17 @@
},
"outputs": [],
"source": [
"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA\n",
"\n",
"### USER: specify your experimental directory\n",
"\n",
"rawdata_path = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/RAW_DATA\"\n",
"rawdata_path = \"/data/visitor/ihma439/id11/20231211/RAW_DATA\"\n",
"\n",
"!ls -lrt {rawdata_path}\n",
"\n",
"processed_data_root_dir = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226\" # USER: modify this to change the destination folder if desired"
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\""
]
},
{
Expand All @@ -106,11 +110,7 @@
"# USER: pick a sample and a dataset you want to segment\n",
"\n",
"sample = \"FeAu_0p5_tR\"\n",
"dataset = \"ff1\"\n",
"\n",
"# USER: specify path to detector mask\n",
"\n",
"mask_path = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/mask.edf'"
"dataset = \"ff1\""
]
},
{
Expand All @@ -123,12 +123,7 @@
"source": [
"# desination of H5 files\n",
"\n",
"dset_path = os.path.join(processed_data_root_dir, sample, f\"{sample}_{dataset}\", f\"{sample}_{dataset}_dataset.h5\")\n",
"\n",
"# USER: specify the path to the parameter file\n",
"\n",
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226/Fe_tdxrd_refined.par'\n",
"spline_file = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/frelon36.spline'"
"dset_path = os.path.join(processed_data_root_dir, sample, f\"{sample}_{dataset}\", f\"{sample}_{dataset}_dataset.h5\")"
]
},
{
Expand Down Expand Up @@ -159,7 +154,7 @@
"\n",
"cf_3d = ImageD11.columnfile.colfile_from_hdf(ds.col3dfile)\n",
"\n",
"cf_3d.parameters.loadparameters(parfile)\n",
"cf_3d.parameters.loadparameters(ds.parfile)\n",
"cf_3d.updateGeometry()\n",
"\n",
"\n",
Expand Down Expand Up @@ -203,7 +198,7 @@
"# this indicates the fractional intensity cutoff we will select\n",
"# if the blue line does not look elbow-shaped in the logscale plot, try changing the \"doplot\" parameter (the y scale of the logscale plot) until it does\n",
"\n",
"cf_strong = utils.selectpeaks(cf_3d, frac=0.985, dsmax=0.92, doplot=0.8, dstol=0.01)\n",
"cf_strong = utils.selectpeaks(cf_3d, frac=0.985, dsmax=1.04, doplot=0.8, dstol=0.01)\n",
"print(f\"Got {cf_strong.nrows} strong peaks for indexing\")\n",
"cf_strong.writefile(f'{sample}_{dataset}_3d_peaks_strong.flt')"
]
Expand Down Expand Up @@ -321,23 +316,28 @@
"outputs": [],
"source": [
"# now we are indexing!\n",
"# indexing will select all rings with a multiplicity below max_multiplity to search\n",
"max_multiplicity = 13\n",
"# the minimum number of peaks on a ring for a ring to be indexed on\n",
"min_counts_on_ring = 0\n",
"# we have to choose which rings we want to generate orientations on\n",
"# generally we want two or three low-multiplicity rings that are isolated from other phases\n",
"# take a look at the ring assignment output from a few cells above, and choose two or three\n",
"rings_for_gen = [0, 1]\n",
"\n",
"# now we want to decide which rings to score our found orientations against\n",
"# generally we can just exclude dodgy rings (close to other phases, only a few peaks in etc)\n",
"rings_for_scoring = [0, 1, 2, 3]\n",
"\n",
"# the sequence of hkl tolerances the indexer will iterate through\n",
"hkl_tols_seq = [0.01, 0.02, 0.03, 0.04]\n",
"# the sequence of minpks fractions the indexer will iterate through\n",
"fracs = [0.9, 0.75]\n",
"# the tolerance in g-vector angle\n",
"cosine_tol = np.cos(np.radians(90.25))\n",
"cosine_tol = np.cos(np.radians(90 - 0.25))\n",
"# the max number of UBIs we can find per pair of rings\n",
"max_grains = 1000\n",
"\n",
"grains, indexer = utils.do_index(cf=cf_strong,\n",
" dstol=indexer.ds_tol,\n",
" max_mult=max_multiplicity,\n",
" min_ring_count=min_counts_on_ring,\n",
" forgen=rings_for_gen,\n",
" foridx=rings_for_scoring,\n",
" hkl_tols=hkl_tols_seq,\n",
" fracs=fracs,\n",
" cosine_tol=cosine_tol,\n",
Expand Down Expand Up @@ -412,9 +412,9 @@
"for inc, makemap_tol in enumerate(makemap_hkl_tol_seq):\n",
" print(f\"Running makemap {inc+1}/{len(makemap_hkl_tol_seq)}\")\n",
" if inc == 0: # ubi into map\n",
" makemap_output = !makemap.py -p {parfile} -u {tmp_ubi_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort\n",
" makemap_output = !makemap.py -p {ds.parfile} -u {tmp_ubi_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort\n",
" else: # map into map\n",
" makemap_output = !makemap.py -p {parfile} -u {tmp_map_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort"
" makemap_output = !makemap.py -p {ds.parfile} -u {tmp_map_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort"
]
},
{
Expand Down Expand Up @@ -449,7 +449,7 @@
},
"outputs": [],
"source": [
"refine_output = !refine_em.py {new_flt_path} {tmp_map_path} {parfile} --omega_slop={omega_slop}"
"refine_output = !refine_em.py {new_flt_path} {tmp_map_path} {ds.parfile} --omega_slop={omega_slop}"
]
},
{
Expand Down
Loading
Loading