Skip to content

Commit

Permalink
add new files
Browse files Browse the repository at this point in the history
  • Loading branch information
@joezuntz
joezuntz committed Nov 8, 2024
1 parent 3c085bc commit 59c8087
Show file tree
Hide file tree
Showing 3 changed files with 462 additions and 0 deletions.
Binary file added data/DESY1-R-model.hdf5
View file
Binary file not shown.
358 changes: 358 additions & 0 deletions notebooks/cluster_counts/generate-20deg2-sample.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,358 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"id": "c0a6788b-58c9-4ab0-9be0-95951f8e7184",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import sys\n",
"import os\n",
"import ceci\n",
"# get current directory\n",
"import os\n",
"notebook_dir = os.path.abspath(\"\")\n",
"txpipe_dir = os.path.join(notebook_dir, \"../..\")\n",
"sys.path.append(txpipe_dir)\n",
"import txpipe"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "903b173a-6629-4e04-8bef-db8a2ad6a15c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# These six pixels come to about 23 deg^2\n",
"pixels = [8786, 8787, 8788, 8914, 8915, 9043]\n",
"outdir = \"./data/cosmodc2/inputs-20deg2\"\n",
"extra_cols=\"redshift_true\"\n",
"response_model=\"{txpipe_dir}/data/DESY1-R-model.hdf5\"\n",
"\n",
"# prepare to run\n",
"os.makedirs(outdir, exist_ok=True)\n",
"\n",
"# configure the stage\n",
"stage = txpipe.TXCosmoDC2Mock.make_stage(\n",
" response_model=response_model,\n",
" shear_catalog=f\"{outdir}/shear_catalog.hdf5\",\n",
" photometry_catalog=f\"{outdir}/photometry_catalog.hdf5\",\n",
" healpixels=pixels,\n",
" extra_cols=extra_cols\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "5bf229ec-b92d-4275-92a2-b2a8e9a62dc7",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loading from catalog cosmoDC2\n",
"Rank 0 loaded: length = 103303103.\n",
"Loading chunk 1/18\n",
"Process 0 read chunk 0 - 2409001 of 103303103\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/pscratch/sd/z/zuntz/TXPipe/txpipe/ingest/mocks.py:1172: RuntimeWarning: invalid value encountered in log10\n",
" mag_obs = 25 - 2.5 * np.log10(n_obs / factor / t_b / n_visit)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Removing 59 objects with identically zero shear in both terms\n",
"Detected 1451813 out of 2409001 objects (60.3%)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/pscratch/sd/z/zuntz/TXPipe/txpipe/ingest/mocks.py:490: RuntimeWarning: divide by zero encountered in log10\n",
" log10_snr = np.log10(snr)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"- Rank 0 writing output to 0-1451813\n",
"Loading chunk 2/18\n",
"Process 0 read chunk 2409001 - 4800628 of 103303103\n",
"Removing 59 objects with identically zero shear in both terms\n",
"Detected 1433760 out of 2391627 objects (59.9%)\n",
"- Rank 0 writing output to 1451813-2885573\n",
"Loading chunk 3/18\n",
"Process 0 read chunk 4800628 - 7181747 of 103303103\n",
"Removing 66 objects with identically zero shear in both terms\n",
"Detected 1422957 out of 2381119 objects (59.8%)\n",
"- Rank 0 writing output to 2885573-4308530\n",
"Loading chunk 4/18\n",
"Process 0 read chunk 7181747 - 9575270 of 103303103\n",
"Removing 64 objects with identically zero shear in both terms\n",
"Detected 1435245 out of 2393523 objects (60.0%)\n",
"- Rank 0 writing output to 4308530-5743775\n",
"Loading chunk 5/18\n",
"Process 0 read chunk 9575270 - 11985392 of 103303103\n",
"Removing 61 objects with identically zero shear in both terms\n",
"Detected 1452481 out of 2410122 objects (60.3%)\n",
"- Rank 0 writing output to 5743775-7196256\n",
"Loading chunk 6/18\n",
"Process 0 read chunk 11985392 - 14358906 of 103303103\n",
"Removing 62 objects with identically zero shear in both terms\n",
"Detected 1415772 out of 2373514 objects (59.6%)\n",
"- Rank 0 writing output to 7196256-8612028\n",
"Loading chunk 7/18\n",
"Process 0 read chunk 14358906 - 20872507 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 2086773 out of 6513601 objects (32.0%)\n",
"- Rank 0 writing output to 8612028-10698801\n",
"Loading chunk 8/18\n",
"Process 0 read chunk 20872507 - 27294236 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 2005527 out of 6421729 objects (31.2%)\n",
"- Rank 0 writing output to 10698801-12704328\n",
"Loading chunk 9/18\n",
"Process 0 read chunk 27294236 - 33728144 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 2020810 out of 6433908 objects (31.4%)\n",
"- Rank 0 writing output to 12704328-14725138\n",
"Loading chunk 10/18\n",
"Process 0 read chunk 33728144 - 40196143 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 2045933 out of 6467999 objects (31.6%)\n",
"- Rank 0 writing output to 14725138-16771071\n",
"Loading chunk 11/18\n",
"Process 0 read chunk 40196143 - 46626501 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 2016340 out of 6430358 objects (31.4%)\n",
"- Rank 0 writing output to 16771071-18787411\n",
"Loading chunk 12/18\n",
"Process 0 read chunk 46626501 - 53042728 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 2007384 out of 6416227 objects (31.3%)\n",
"- Rank 0 writing output to 18787411-20794795\n",
"Loading chunk 13/18\n",
"Process 0 read chunk 53042728 - 61375656 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 754346 out of 8332928 objects (9.1%)\n",
"- Rank 0 writing output to 20794795-21549141\n",
"Loading chunk 14/18\n",
"Process 0 read chunk 61375656 - 69797162 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 790467 out of 8421506 objects (9.4%)\n",
"- Rank 0 writing output to 21549141-22339608\n",
"Loading chunk 15/18\n",
"Process 0 read chunk 69797162 - 78145829 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 762637 out of 8348667 objects (9.1%)\n",
"- Rank 0 writing output to 22339608-23102245\n",
"Loading chunk 16/18\n",
"Process 0 read chunk 78145829 - 86535724 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 773673 out of 8389895 objects (9.2%)\n",
"- Rank 0 writing output to 23102245-23875918\n",
"Loading chunk 17/18\n",
"Process 0 read chunk 86535724 - 94910785 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 767506 out of 8375061 objects (9.2%)\n",
"- Rank 0 writing output to 23875918-24643424\n",
"Loading chunk 18/18\n",
"Process 0 read chunk 94910785 - 103303103 of 103303103\n",
"Removing 0 objects with identically zero shear in both terms\n",
"Detected 775297 out of 8392318 objects (9.2%)\n",
"- Rank 0 writing output to 24643424-25418721\n",
"Resizing all outupts to size 25418721\n",
"Resizing dec\n",
"Resizing extendedness\n",
"Resizing id\n",
"Resizing mag_g\n",
"Resizing mag_g_err\n",
"Resizing mag_i\n",
"Resizing mag_i_err\n",
"Resizing mag_r\n",
"Resizing mag_r_err\n",
"Resizing mag_u\n",
"Resizing mag_u_err\n",
"Resizing mag_y\n",
"Resizing mag_y_err\n",
"Resizing mag_z\n",
"Resizing mag_z_err\n",
"Resizing ra\n",
"Resizing redshift_true\n",
"Resizing snr_g\n",
"Resizing snr_i\n",
"Resizing snr_r\n",
"Resizing snr_u\n",
"Resizing snr_y\n",
"Resizing snr_z\n",
"Resizing 00/T\n",
"Resizing 00/T_err\n",
"Resizing 00/dec\n",
"Resizing 00/flags\n",
"Resizing 00/g1\n",
"Resizing 00/g2\n",
"Resizing 00/id\n",
"Resizing 00/mag_err_i\n",
"Resizing 00/mag_err_r\n",
"Resizing 00/mag_err_z\n",
"Resizing 00/mag_i\n",
"Resizing 00/mag_r\n",
"Resizing 00/mag_z\n",
"Resizing 00/mcal_psf_T_mean\n",
"Resizing 00/mcal_psf_g1\n",
"Resizing 00/mcal_psf_g2\n",
"Resizing 00/psf_g1\n",
"Resizing 00/psf_g2\n",
"Resizing 00/ra\n",
"Resizing 00/redshift_true\n",
"Resizing 00/s2n\n",
"Resizing 00/true_g1\n",
"Resizing 00/true_g2\n",
"Resizing 00/weight\n",
"Resizing 1m/T\n",
"Resizing 1m/T_err\n",
"Resizing 1m/dec\n",
"Resizing 1m/flags\n",
"Resizing 1m/g1\n",
"Resizing 1m/g2\n",
"Resizing 1m/id\n",
"Resizing 1m/mag_err_i\n",
"Resizing 1m/mag_err_r\n",
"Resizing 1m/mag_err_z\n",
"Resizing 1m/mag_i\n",
"Resizing 1m/mag_r\n",
"Resizing 1m/mag_z\n",
"Resizing 1m/mcal_psf_T_mean\n",
"Resizing 1m/mcal_psf_g1\n",
"Resizing 1m/mcal_psf_g2\n",
"Resizing 1m/psf_g1\n",
"Resizing 1m/psf_g2\n",
"Resizing 1m/ra\n",
"Resizing 1m/s2n\n",
"Resizing 1m/weight\n",
"Resizing 1p/T\n",
"Resizing 1p/T_err\n",
"Resizing 1p/dec\n",
"Resizing 1p/flags\n",
"Resizing 1p/g1\n",
"Resizing 1p/g2\n",
"Resizing 1p/id\n",
"Resizing 1p/mag_err_i\n",
"Resizing 1p/mag_err_r\n",
"Resizing 1p/mag_err_z\n",
"Resizing 1p/mag_i\n",
"Resizing 1p/mag_r\n",
"Resizing 1p/mag_z\n",
"Resizing 1p/mcal_psf_T_mean\n",
"Resizing 1p/mcal_psf_g1\n",
"Resizing 1p/mcal_psf_g2\n",
"Resizing 1p/psf_g1\n",
"Resizing 1p/psf_g2\n",
"Resizing 1p/ra\n",
"Resizing 1p/s2n\n",
"Resizing 1p/weight\n",
"Resizing 2m/T\n",
"Resizing 2m/T_err\n",
"Resizing 2m/dec\n",
"Resizing 2m/flags\n",
"Resizing 2m/g1\n",
"Resizing 2m/g2\n",
"Resizing 2m/id\n",
"Resizing 2m/mag_err_i\n",
"Resizing 2m/mag_err_r\n",
"Resizing 2m/mag_err_z\n",
"Resizing 2m/mag_i\n",
"Resizing 2m/mag_r\n",
"Resizing 2m/mag_z\n",
"Resizing 2m/mcal_psf_T_mean\n",
"Resizing 2m/mcal_psf_g1\n",
"Resizing 2m/mcal_psf_g2\n",
"Resizing 2m/psf_g1\n",
"Resizing 2m/psf_g2\n",
"Resizing 2m/ra\n",
"Resizing 2m/s2n\n",
"Resizing 2m/weight\n",
"Resizing 2p/T\n",
"Resizing 2p/T_err\n",
"Resizing 2p/dec\n",
"Resizing 2p/flags\n",
"Resizing 2p/g1\n",
"Resizing 2p/g2\n",
"Resizing 2p/id\n",
"Resizing 2p/mag_err_i\n",
"Resizing 2p/mag_err_r\n",
"Resizing 2p/mag_err_z\n",
"Resizing 2p/mag_i\n",
"Resizing 2p/mag_r\n",
"Resizing 2p/mag_z\n",
"Resizing 2p/mcal_psf_T_mean\n",
"Resizing 2p/mcal_psf_g1\n",
"Resizing 2p/mcal_psf_g2\n",
"Resizing 2p/psf_g1\n",
"Resizing 2p/psf_g2\n",
"Resizing 2p/ra\n",
"Resizing 2p/s2n\n",
"Resizing 2p/weight\n"
]
}
],
"source": [
"# run the stage\n",
"stage.run()\n",
"\n",
"# move the stage outputs to their final locations\n",
"stage.finalize()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "93596750-611a-4401-81d5-8d8a9fd84c77",
"metadata": {
"tags": []
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit 59c8087

Please sign in to comment.