Skip to content

Commit

Permalink
make file_postprocessing not need diff files if SAVE_DIFF_RED_AVG=False
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsdillon
jsdillon committed Dec 4, 2024
1 parent fb00086 commit 8417d7b
Showing 1 changed file with 29 additions and 20 deletions.
49 changes: 29 additions & 20 deletions notebooks/file_postprocessing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@
"SAVE_DLY_FILT_RED_AVG = os.environ.get(\"SAVE_DLY_FILT_RED_AVG\", \"TRUE\").upper() == \"TRUE\"\n",
"SAVE_CROSS_POLS = os.environ.get(\"SAVE_CROSS_POLS\", \"FALSE\").upper() == \"TRUE\"\n",
"# TODO: in theory, if some of these are turned off, some memory and/or processing could be saved \n",
"# by doing only a subset of what this notebook does. That's left for fuure work.\n",
"# by doing only a subset of what this notebook does. That's left for future work.\n",
"\n",
"add_to_history = 'Produced by file_prostprocessing notebook with the following environment:\\n' + '=' * 65 + '\\n' + os.popen('conda env export').read() + '=' * 65\n",
"for setting in ['SAVE_RESULTS', 'SAVE_DIFF_RED_AVG', 'SAVE_ABS_CAL_RED_AVG', 'SAVE_DLY_FILT_RED_AVG', 'SAVE_CROSS_POLS']:\n",
Expand Down Expand Up @@ -204,8 +204,9 @@
"hd = io.HERADataFastReader(SUM_FILE)\n",
"data, flags, nsamples = hd.read(pols=pols)\n",
"\n",
"hd_diff = io.HERADataFastReader(DIFF_FILE)\n",
"diff_data, diff_flags, diff_nsamples = hd_diff.read(pols=pols)"
"if SAVE_DIFF_RED_AVG:\n",
" hd_diff = io.HERADataFastReader(DIFF_FILE)\n",
" diff_data, diff_flags, diff_nsamples = hd_diff.read(pols=pols)"
]
},
{
Expand Down Expand Up @@ -309,9 +310,10 @@
"source": [
"# perform redundant averaging\n",
"red_avg_smooth_cal_sum_data, red_avg_flags, red_avg_nsamples = red_average(reds, data, nsamples, smooth_gains, flags=flags, cal_flags=cal_flags)\n",
"red_avg_smooth_cal_diff_data, _, _ = red_average(reds, diff_data, diff_nsamples, smooth_gains, flags=diff_flags, cal_flags=cal_flags)\n",
"red_avg_abs_cal_sum_data, _, _ = red_average(reds, data, nsamples, abs_gains, flags=flags, cal_flags=cal_flags)\n",
"red_avg_abs_cal_diff_data, _, _ = red_average(reds, diff_data, nsamples, abs_gains, flags=flags, cal_flags=cal_flags)"
"if SAVE_DIFF_RED_AVG:\n",
" red_avg_smooth_cal_diff_data, _, _ = red_average(reds, diff_data, diff_nsamples, smooth_gains, flags=diff_flags, cal_flags=cal_flags)\n",
" red_avg_abs_cal_diff_data, _, _ = red_average(reds, diff_data, nsamples, abs_gains, flags=flags, cal_flags=cal_flags)"
]
},
{
Expand Down Expand Up @@ -471,7 +473,9 @@
"source": [
"# save memory\n",
"ALL_CAL_FLAGS = np.all([np.all(~cal_flags[ant]) for ant in cal_flags])\n",
"del hd, data, flags, nsamples, hd_diff, diff_data, diff_flags, diff_nsamples\n",
"del hd, data, flags, nsamples\n",
"if SAVE_DIFF_RED_AVG:\n",
" del hd_diff, diff_data, diff_flags, diff_nsamples\n",
"del hc_smooth, smooth_gains, cal_flags, hc_abs, abs_gains, abs_cal_flags\n",
"hd = io.HERADataFastReader(SUM_FILE)"
]
Expand Down Expand Up @@ -537,9 +541,10 @@
"source": [
"# set up datacontainers for delay-filtered data products\n",
"dly_filt_red_avg_smooth_cal_sum_data = copy.deepcopy(red_avg_smooth_cal_sum_data)\n",
"dly_filt_red_avg_smooth_cal_diff_data = copy.deepcopy(red_avg_smooth_cal_diff_data)\n",
"dly_filt_red_avg_abs_cal_sum_data = copy.deepcopy(red_avg_abs_cal_sum_data)\n",
"dly_filt_red_avg_abs_cal_diff_data = copy.deepcopy(red_avg_abs_cal_diff_data)\n",
"if SAVE_DIFF_RED_AVG:\n",
" dly_filt_red_avg_smooth_cal_diff_data = copy.deepcopy(red_avg_smooth_cal_diff_data)\n",
" dly_filt_red_avg_abs_cal_diff_data = copy.deepcopy(red_avg_abs_cal_diff_data)\n",
"\n",
"auto_bls = [bl for bl in dly_filt_red_avg_smooth_cal_sum_data if utils.split_bl(bl)[0] == utils.split_bl(bl)[1]]\n",
"gap_flag_warned = set()\n",
Expand All @@ -566,8 +571,8 @@
" filter_centers, filter_half_widths = vis_clean.gen_filter_properties(ax='freq', min_dly=DLY_FILT_MIN_DLY, horizon=DLY_FILT_HORIZON,\n",
" standoff=DLY_FILT_STANDOFF, bl_len=bl_len)\n",
" \n",
" for dc in [dly_filt_red_avg_smooth_cal_sum_data, dly_filt_red_avg_smooth_cal_diff_data, \n",
" dly_filt_red_avg_abs_cal_sum_data, dly_filt_red_avg_abs_cal_diff_data]:\n",
" for dc in [dly_filt_red_avg_smooth_cal_sum_data, dly_filt_red_avg_abs_cal_sum_data] + \\\n",
" ([dly_filt_red_avg_smooth_cal_diff_data, dly_filt_red_avg_abs_cal_diff_data] if SAVE_DIFF_RED_AVG else []):\n",
" if np.all(dc[bl] == 0.0) and np.all(red_avg_flags[bl]):\n",
" # don't bother delay filtering all 0s.\n",
" continue\n",
Expand Down Expand Up @@ -643,11 +648,14 @@
" ax.semilogy(delays*1e9, np.mean(np.abs(dfft_df[~integration_flags, :]), axis=0), ls='--', label=f'Delay-Filtered Sum Data')\n",
"\n",
" # FFT diffs\n",
" dfft_ddf, _ = vis_clean.fft_data(np.where(red_avg_flags[bl], 0, red_avg_smooth_cal_diff_data[bl])[:, band], \n",
" delta_bin=(hd.freqs[1] - hd.freqs[0]), window='bh')\n",
" ax.semilogy(delays*1e9, np.mean(np.abs(dfft_ddf[~integration_flags, :]), axis=0), ls=':', label=f'Diff Data')\n",
"\n",
" mins[bl] = np.median(np.abs(dfft_ddf[~integration_flags, :])) / 5\n",
" if SAVE_DIFF_RED_AVG:\n",
" dfft_ddf, _ = vis_clean.fft_data(np.where(red_avg_flags[bl], 0, red_avg_smooth_cal_diff_data[bl])[:, band], \n",
" delta_bin=(hd.freqs[1] - hd.freqs[0]), window='bh')\n",
" ax.semilogy(delays*1e9, np.mean(np.abs(dfft_ddf[~integration_flags, :]), axis=0), ls=':', label=f'Diff Data')\n",
" \n",
" mins[bl] = np.median(np.abs(dfft_ddf[~integration_flags, :])) / 5\n",
" else:\n",
" mins[bl] = np.median(np.abs(dfft_df[~integration_flags, :])) / 5\n",
" try:\n",
" maxes[bl] = np.max(np.abs(dfft_ip[~integration_flags, :])) * 2\n",
" except:\n",
Expand Down Expand Up @@ -714,7 +722,7 @@
"source": [
"if SAVE_RESULTS:\n",
" hd_out = io.HERAData(SUM_FILE)\n",
" hd_out.read(bls=red_avg_flags.keys())\n",
" hd_out.read(bls=list(set([bl[0:2] for bl in red_avg_flags])), polarizations=list(red_avg_flags.pols()))\n",
" hd_out.empty_arrays()\n",
" hd_out.history += add_to_history\n",
" hd_out.update(flags=red_avg_flags, nsamples=red_avg_nsamples)\n",
Expand All @@ -733,13 +741,14 @@
"\n",
" # write delay-filtered file(s)\n",
" _write(red_avg_smooth_cal_sum_data, SUM_SMOOTH_CAL_RED_AVG_FILE)\n",
" _write(red_avg_smooth_cal_diff_data, DIFF_SMOOTH_CAL_RED_AVG_FILE, SAVE_DIFF_RED_AVG)\n",
" _write(red_avg_abs_cal_sum_data, SUM_ABS_CAL_RED_AVG_FILE, SAVE_ABS_CAL_RED_AVG)\n",
" _write(red_avg_abs_cal_diff_data, DIFF_ABS_CAL_RED_AVG_FILE, SAVE_ABS_CAL_RED_AVG, SAVE_DIFF_RED_AVG)\n",
" _write(dly_filt_red_avg_smooth_cal_sum_data, SUM_SMOOTH_CAL_RED_AVG_DLY_FILT_FILE, SAVE_DLY_FILT_RED_AVG)\n",
" _write(dly_filt_red_avg_smooth_cal_diff_data, DIFF_SMOOTH_CAL_RED_AVG_DLY_FILT_FILE, SAVE_DLY_FILT_RED_AVG, SAVE_DIFF_RED_AVG) \n",
" _write(dly_filt_red_avg_abs_cal_sum_data, SUM_ABS_CAL_RED_AVG_DLY_FILT_FILE, SAVE_DLY_FILT_RED_AVG, SAVE_ABS_CAL_RED_AVG)\n",
" _write(dly_filt_red_avg_abs_cal_diff_data, DIFF_ABS_CAL_RED_AVG_DLY_FILT_FILE, SAVE_DLY_FILT_RED_AVG, SAVE_ABS_CAL_RED_AVG, SAVE_DIFF_RED_AVG)"
" if SAVE_DIFF_RED_AVG:\n",
" _write(red_avg_smooth_cal_diff_data, DIFF_SMOOTH_CAL_RED_AVG_FILE, SAVE_DIFF_RED_AVG)\n",
" _write(red_avg_abs_cal_diff_data, DIFF_ABS_CAL_RED_AVG_FILE, SAVE_ABS_CAL_RED_AVG, SAVE_DIFF_RED_AVG)\n",
" _write(dly_filt_red_avg_smooth_cal_diff_data, DIFF_SMOOTH_CAL_RED_AVG_DLY_FILT_FILE, SAVE_DLY_FILT_RED_AVG, SAVE_DIFF_RED_AVG) \n",
" _write(dly_filt_red_avg_abs_cal_diff_data, DIFF_ABS_CAL_RED_AVG_DLY_FILT_FILE, SAVE_DLY_FILT_RED_AVG, SAVE_ABS_CAL_RED_AVG, SAVE_DIFF_RED_AVG)"
]
},
{
Expand Down

0 comments on commit 8417d7b

Please sign in to comment.