Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Geometry overhaul (+tests), update all notebooks #276

Merged
merged 21 commits into from
Apr 11, 2024
Merged

Geometry overhaul (+tests), update all notebooks #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
9beb91c
Update notebook 0
jadball Mar 28, 2024
a06b9b9
Comment out manual override
jadball Mar 28, 2024
367fd50
Update notebook 1
jadball Mar 28, 2024
e3d1ae1
Optimise imports
jadball Mar 28, 2024
e74a87b
Optimise imports
jadball Mar 28, 2024
6ca5304
Dates
jadball Mar 28, 2024
44c98fc
Update notebook 2
jadball Mar 28, 2024
6910671
Update minor phase notebook 1
jadball Mar 28, 2024
92fedfc
Optimise imports
jadball Mar 28, 2024
f80c825
Merge branch 'master' of github.com:jadball/ImageD11
jadball Mar 28, 2024
cc83cf2
checkpoint
jadball Mar 28, 2024
d123dbe
Update minor phase notebook 2
jadball Mar 28, 2024
9f198fe
Add optional label array to build_slice_arrays, suppress printing in …
jadball Mar 28, 2024
577e037
Fix math error in geometry
jadball Mar 28, 2024
7d9a43e
IO bugfixes
jadball Mar 28, 2024
677449a
Some attempts at adjusting to new classes
jadball Mar 28, 2024
a4496e1
Geometry overhaul still in progress
jadball Apr 10, 2024
0c72108
Updated PBP indexer to use new geometry
jadball Apr 11, 2024
7ad0028
Add geometry tests and update box-beam notebooks
jadball Apr 11, 2024
5072b19
Fix find/replace typo
jadball Apr 11, 2024
c1b9dbb
Re-add sinogram correction for half-scans
jadball Apr 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 7 additions & 15 deletions ImageD11/nbGui/3DXRD/0_3DXRD_segment_frelon.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@
"\n",
"from frelon_peaksearch import worker, process\n",
"\n",
"from ImageD11.blobcorrector import correct_cf_with_spline\n",
"\n",
"# from utils import apply_spatial"
]
},
Expand Down Expand Up @@ -162,17 +164,7 @@
"\n",
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA\n",
"\n",
"### USER: specify your experimental directory\n",
"\n",
"rawdata_path = \"/data/visitor/ihma439/id11/20231211/RAW_DATA\"\n",
"\n",
"!ls -lrt {rawdata_path}\n",
"\n",
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\""
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA"
]
},
{
Expand Down Expand Up @@ -429,7 +421,7 @@
},
"outputs": [],
"source": [
"cf_2d = utils.apply_spatial_lut(cf_2d, spline_file)"
"cf_2d = correct_cf_with_spline(cf_2d, spline_file)"
]
},
{
Expand All @@ -441,7 +433,7 @@
},
"outputs": [],
"source": [
"cf_3d = utils.apply_spatial_lut(cf_3d, spline_file)"
"cf_3d = correct_cf_with_spline(cf_3d, spline_file)"
]
},
{
Expand Down Expand Up @@ -566,8 +558,8 @@
" cf_2d, cf_3d = process(ds, nthreads-1, worker_args)\n",
" \n",
" print(\"Spatially correcting peaks\")\n",
" cf_2d = utils.apply_spatial_lut(cf_2d, spline_file)\n",
" cf_3d = utils.apply_spatial_lut(cf_3d, spline_file)\n",
" cf_2d = correct_cf_with_spline(cf_2d, spline_file)\n",
" cf_3d = correct_cf_with_spline(cf_3d, spline_file)\n",
" \n",
" print(\"Saving peaks to file\")\n",
" cf_2d.parameters.loadparameters(parfile)\n",
Expand Down
102 changes: 43 additions & 59 deletions ImageD11/nbGui/3DXRD/2_3DXRD_index.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@
"import ImageD11.columnfile\n",
"from ImageD11.sinograms import properties, dataset\n",
"\n",
"from ImageD11.blobcorrector import eiger_spatial"
"from ImageD11.blobcorrector import eiger_spatial\n",
"from ImageD11.peakselect import select_ring_peaks_by_intensity"
]
},
{
Expand Down Expand Up @@ -144,13 +145,10 @@
"source": [
"# load 3d columnfile from disk\n",
"\n",
"cf_3d = ImageD11.columnfile.colfile_from_hdf(ds.col3dfile)\n",
"cf_3d = ds.get_cf_3d_from_disk()\n",
"\n",
"cf_3d.parameters.loadparameters(ds.parfile)\n",
"cf_3d.updateGeometry()\n",
"\n",
"if \"index\" not in cf_3d.titles:\n",
" cf_3d.addcolumn(np.arange(cf_3d.nrows), \"index\")"
"cf_3d.updateGeometry()"
]
},
{
Expand Down Expand Up @@ -194,7 +192,7 @@
"cf_strong_dsmax = 1.03\n",
"cf_strong_dstol = 0.01\n",
"\n",
"cf_strong = utils.selectpeaks(cf_3d, frac=cf_strong_frac, dsmax=cf_strong_dsmax, doplot=0.8, dstol=cf_strong_dstol)\n",
"cf_strong = select_ring_peaks_by_intensity(cf_3d, frac=cf_strong_frac, dsmax=cf_strong_dsmax, doplot=0.8, dstol=cf_strong_dstol)\n",
"print(f\"Got {cf_strong.nrows} strong peaks for indexing\")\n",
"cf_strong.writefile(f'{sample}_{dataset}_3d_peaks_strong.flt')"
]
Expand All @@ -213,7 +211,7 @@
"cf_strong_allrings_frac = 0.95\n",
"cf_strong_allrings_dstol = 0.01\n",
"\n",
"cf_strong_allrings = utils.selectpeaks(cf_3d, frac=cf_strong_allrings_frac, dsmax=cf_3d.ds.max(), doplot=0.8, dstol=cf_strong_allrings_dstol)\n",
"cf_strong_allrings = select_ring_peaks_by_intensity(cf_3d, frac=cf_strong_allrings_frac, dsmax=cf_3d.ds.max(), doplot=0.8, dstol=cf_strong_allrings_dstol)\n",
"print(f\"Got {cf_strong_allrings.nrows} strong peaks for makemap\")\n",
"cf_strong_allrings_path = f'{sample}_{dataset}_3d_peaks_strong_all_rings.flt'\n",
"cf_strong_allrings.writefile(cf_strong_allrings_path)"
Expand Down Expand Up @@ -361,13 +359,34 @@
" g.gid = i"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# let's look at the pole figures\n",
"# for this we need to set a reference unit cell for the grains\n",
"\n",
"cf_pars = cf_strong.parameters.get_parameters()\n",
"spacegroup = 229 # spacegroup for BCC iron\n",
"cf_pars[\"cell_lattice_[P,A,B,C,I,F,R]\"] = spacegroup\n",
"\n",
"ref_ucell = ImageD11.unitcell.unitcell_from_parameters(cf_pars)\n",
"\n",
"for g in grains:\n",
" g.ref_unitcell = ref_ucell"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# plot pole figures?"
"utils.plot_all_ipfs(grains)"
]
},
{
Expand Down Expand Up @@ -507,7 +526,7 @@
"outputs": [],
"source": [
"# find the spike\n",
"absolute_minpks = 45"
"absolute_minpks = 37"
]
},
{
Expand Down Expand Up @@ -563,36 +582,11 @@
},
"outputs": [],
"source": [
"for grain in grains_filtered:\n",
" grain.gid = int(grain.name.split(\":\")[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# now we will assign all our 3D peaks to our grains"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"peak_assignment_hkl_tol = 0.05\n",
"# now we will assign all our 3D peaks to our remaining grains\n",
"\n",
"utils.assign_peaks_to_grains(grains_filtered, cf_3d, tol)\n",
"peak_assignment_hkl_tol = 0.05\n",
"\n",
"print(\"Storing peak data in grains\")\n",
"# iterate through all the grains\n",
"for g in tqdm(grains_filtered):\n",
" # store this grain's peak indices so we know which 3D peaks we used for indexing\n",
" g.peaks_3d = cf_3d.index[cf_3d.grain_id == g.gid]"
"utils.assign_peaks_to_grains(grains_filtered, cf_3d, peak_assignment_hkl_tol)"
]
},
{
Expand All @@ -603,11 +597,9 @@
},
"outputs": [],
"source": [
"fig, ax = plt.subplots()\n",
"ax.hist([np.mean(grain.unitcell[0:3]) for grain in grains_filtered], bins=25)\n",
"plt.show()\n",
"# re-save our now indexed peaks\n",
"\n",
"print(np.mean([np.mean(grain.unitcell[0:3]) for grain in grains_filtered]))"
"ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile, name=\"peaks\")"
]
},
{
Expand All @@ -620,7 +612,7 @@
"source": [
"# save grain data\n",
"\n",
"utils.save_3dxrd_grains(grains_filtered, ds)"
"ds.save_grains_to_disk(grains_filtered)"
]
},
{
Expand Down Expand Up @@ -691,21 +683,18 @@
" print(f\"Found existing grains file for {dataset} in {sample}, skipping\")\n",
" continue\n",
"\n",
"\n",
" print(\"Loading 3D peaks\")\n",
" cf_3d = ImageD11.columnfile.colfile_from_hdf(ds.col3dfile)\n",
" cf_3d = ds.get_cf_3d_from_disk()\n",
" cf_3d.parameters.loadparameters(ds.parfile)\n",
" cf_3d.updateGeometry()\n",
" if \"index\" not in cf_3d.titles:\n",
" cf_3d.addcolumn(np.arange(cf_3d.nrows), \"index\")\n",
"\n",
" print(\"Filtering 3D peaks\")\n",
" cf_strong = utils.selectpeaks(cf_3d, frac=cf_strong_frac, dsmax=cf_strong_dsmax, doplot=None, dstol=cf_strong_dstol)\n",
" cf_strong = select_ring_peaks_by_intensity(cf_3d, frac=cf_strong_frac, dsmax=cf_strong_dsmax, doplot=None, dstol=cf_strong_dstol)\n",
" print(f\"Got {cf_strong.nrows} strong peaks for indexing\")\n",
" cf_strong_path = f'{sample}_{dataset}_3d_peaks_strong.flt'\n",
" cf_strong.writefile(cf_strong_path)\n",
"\n",
" cf_strong_allrings = utils.selectpeaks(cf_3d, frac=cf_strong_allrings_frac, dsmax=cf_3d.ds.max(), doplot=None, dstol=cf_strong_allrings_dstol)\n",
" cf_strong_allrings = select_ring_peaks_by_intensity(cf_3d, frac=cf_strong_allrings_frac, dsmax=cf_3d.ds.max(), doplot=None, dstol=cf_strong_allrings_dstol)\n",
" print(f\"Got {cf_strong_allrings.nrows} strong peaks for makemap\")\n",
" cf_strong_allrings_path = f'{sample}_{dataset}_3d_peaks_strong_all_rings.flt'\n",
" cf_strong_allrings.writefile(cf_strong_allrings_path)\n",
Expand Down Expand Up @@ -752,19 +741,12 @@
" \n",
" grains_filtered = [grain for grain in grains2 if float(grain.npks) > absolute_minpks]\n",
"\n",
" for grain in grains_filtered:\n",
" grain.gid = int(grain.name.split(\":\")[0])\n",
"\n",
" utils.assign_peaks_to_grains(grains_filtered, cf_3d, tol=peak_assignment_hkl_tol)\n",
"\n",
" print(\"Storing peak data in grains\")\n",
" # iterate through all the grains\n",
" for g in tqdm(grains_filtered):\n",
" # store this grain's peak indices so we know which 3D peaks we used for indexing\n",
" g.peaks_3d = cf_3d.index[cf_3d.grain_id == g.gid]\n",
" ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile, name=\"peaks\")\n",
"\n",
" print(\"Saving grains\")\n",
" utils.save_3dxrd_grains(grains_filtered, ds)\n",
" ds.save_grains_to_disk(grains_filtered)\n",
"\n",
" if os.path.exists(cf_strong_path):\n",
" os.remove(cf_strong_path)\n",
Expand All @@ -782,7 +764,9 @@
" os.remove(new_flt_path)\n",
"\n",
" if os.path.exists(unindexed_flt_path):\n",
" os.remove(unindexed_flt_path)"
" os.remove(unindexed_flt_path)\n",
"\n",
"print(\"Done!\")"
]
},
{
Expand Down
Loading
Loading