Skip to content

Commit

Permalink
implement n_hours_per_met_file (#102)
Browse files Browse the repository at this point in the history 
* num hours per met file

* only call find_met_files twice if met_subgrid is enabled

* implement met_hours using n_hours_per_met_file

* call unique on request and output

* add n_hours_per_met_file param to stilt_cli

* n_hours_per_met_file guidance

---------

Co-authored-by: John-C-Lin <[email protected]>
  • Loading branch information
@jmineau @John-C-Lin
jmineau and John-C-Lin authored Oct 3, 2024
1 parent 5b31c43 commit 0b81580
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 31 deletions.
1 change: 1 addition & 0 deletions docs/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ str(receptors)
| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `met_path` | Absolute path to ARL compatible meteorological data files |
| `met_file_format` | String detailing file naming convention for meteorological data files using a mixture of datetime and regex syntax. The formatting string accepts `grep` compatible regular expressions (`.\*.arl`), `strftime` compatible datetime strings (`%Y%m%d%H`) or any combination of the two. Datetime syntax is expanded to all unique combinations required for the receptor and simulation duration and the intersection between the requested files and files available in `met_path` is determined with `grep`, allowing partial matching and compatible regular expressions to be used to identify the relevant data. Matching does not require the full format to be specified - e.g. `\*.arl`, `%Y`, `%Y%m%d`, `%Y%m%d_d0.*.arl` would all match with a file named `20180130_d01.arl`. |
| `n_hours_per_met_file` | Number of hours per meteorological data file. To determine the number of hours in an ARL compatible meteorological data file, refer to the README, including the file naming convention, provided by the data source. For example, the [NOAA HRRR README](https://www.ready.noaa.gov/data/archives/hrrr/README.TXT) specifies a "6 hour data file beginning with 00z - 05z in the first file of the day". Defaults to 6 |
| `met_subgrid_buffer` | Percent to extend footprint area for meteorological subdomain when using `met_subgrid_enable`. Defaults to 0.1 (10%) |
| `met_subgrid_enable` | Enables extraction of spatial subdomains from files in `met_path` using HYSPLIT's `xtrct_grid` binary prior to executing simulations. If enabled, will create files in `<output_wd>/met/`. This can substantially accelerate simulation speed at the cost of increased disk usage. Defaults to disabled |
| `met_subgrid_levels` | If set, extracts the defined number of vertical levels from the meteorological data files to further accelerate simulations. Defaults to `NA`, which includes all vertical levels available |
Expand Down
2 changes: 2 additions & 0 deletions r/run_stilt.r
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ yres <- xres
# Meteorological data input
met_path <- '<path_to_arl_meteorological_data>'
met_file_format <- '%Y%m%d.%Hz.hrrra'
n_hours_per_met_file <- 6
met_subgrid_buffer <- 0.2
met_subgrid_enable <- F
met_subgrid_levels <- NA
Expand Down Expand Up @@ -239,6 +240,7 @@ stilt_apply(FUN = simulation_step,
met_subgrid_levels = met_subgrid_levels,
mgmin = mgmin,
n_hours = n_hours,
n_hours_per_met_file = n_hours_per_met_file,
n_met_min = n_met_min,
ncycl = ncycl,
ndump = ndump,
Expand Down
55 changes: 37 additions & 18 deletions r/src/find_met_files.r
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,61 @@
#' find_met_files searches for meteorological data files
#' @author Ben Fasoli
#' @author Ben Fasoli & James Mineau
#'
#' Searches for available meteorological files matching the given strftime
#' compatible file naming convention
#'
#' @param t_start time of simulation start
#' @param n_hours number of hours to run each simulation; negative indicates
#' backward in time
#' @param n_hours_per_met_file number of hours of meteorological data in each
#' met file
#' @param met_file_format grep compatible file naming convention to identify
#' meteorological data files necessary for the timing of the simulation
#' indicated by \code{t_start} and \code{n_hours}
#' @param n_hours number of hours to run each simulation; negative indicates
#' backward in time
#' @param met_path directory to find meteorological data
#'
#' @import dplyr
#' @export

find_met_files <- function(t_start, met_file_format, n_hours, met_path) {
find_met_files <- function(t_start, n_hours, n_hours_per_met_file,
met_file_format, met_path) {
require(dplyr)


ts <- as.POSIXct(t_start, tz = 'UTC')
is_backward <- n_hours < 0

# TODO: implement n_hours_per_met_file to better determine file names at
# varying time resolutions
request <- as.POSIXct(t_start, tz='UTC') %>%
c(. + c(1, -1, n_hours, is_backward * (n_hours - 5)) * 3600) %>%
range() %>%
(function(x) seq(x[1], x[2], by = 'hour')) %>%
strftime(tz = 'UTC', format = met_file_format)

met_bracket <- n_hours_per_met_file - 1 # ts can be in the middle of a met file

# Generate the hours to search for
if (is_backward) {
met_hours <- seq(
ts - as.difftime(abs(n_hours) + met_bracket, units = 'hours'),
ts,
by = 3600
)
} else {
met_hours <- seq(
ts - as.difftime(met_bracket, units = 'hours'),
ts + as.difftime(n_hours, units = 'hours'),
by = 3600
)
}

# Format the request and remove duplicates
request <- met_hours %>%
strftime(tz = 'UTC', format = met_file_format) %>%
unique()

# Find the available files
available <- dir(met_path, full.names = T, recursive = T)
available <- available[!grepl('.lock', available)]


# Find the files that match the request
idx <- do.call(c, lapply(request, function(pattern) {
grep(pattern = pattern, x = available)
}))

if (any(idx < 1))
return()
unique(available[idx])

unique(available[idx]) # Available files that match the request
}
29 changes: 16 additions & 13 deletions r/src/simulation_step.r
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ simulation_step <- function(before_footprint = list(function() {output}),
mgmin = 10,
mhrs = 9999,
n_hours = -24,
n_hours_per_met_file = 6,
n_met_min = 1,
ncycl = 0,
ndump = 0,
Expand Down Expand Up @@ -257,33 +258,35 @@ simulation_step <- function(before_footprint = list(function() {output}),
link_files(exe, rundir)

# Find necessary met files
met_files <- find_met_files(r_run_time, met_file_format, n_hours, met_path)
met_files <- find_met_files(r_run_time, n_hours, n_hours_per_met_file,
met_file_format, met_path)
if (length(met_files) < n_met_min) {
msg <- paste('Insufficient number of meteorological files found. Check',
'specifications in run_stilt.r')
warning(msg)
cat(msg, '\n', file = file.path(rundir, 'stilt.log'), append = T)
return()
}

if (met_subgrid_enable) {
met_path <- file.path(output_wd, 'met')
calc_met_subgrids(met_files, met_path, exe,
projection, xmn, xmx, ymn, ymx,
levels = met_subgrid_levels,
met_subgrid_buffer = met_subgrid_buffer)

# Find necessary met files for subgrids
met_files <- find_met_files(r_run_time, n_hours, n_hours_per_met_file,
met_file_format, met_path)
if (length(met_files) < n_met_min) {
msg <- paste('Insufficient number of meteorological files found. Check',
'specifications in run_stilt.r')
warning(msg)
cat(msg, '\n', file = file.path(rundir, 'stilt.log'), append = T)
return()
}
}

# Find necessary met files
met_files <- find_met_files(r_run_time, met_file_format, n_hours, met_path)
if (length(met_files) < n_met_min) {
msg <- paste('Insufficient number of meteorological files found. Check',
'specifications in run_stilt.r')
warning(msg)
cat(msg, '\n', file = file.path(rundir, 'stilt.log'), append = T)
return()
}


# Execute particle trajectory simulation, and read results into data frame
output$receptor <- list(run_time = r_run_time,
lati = r_lati,
Expand Down
1 change: 1 addition & 0 deletions r/stilt_cli.r
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ stilt_args <- list(
met_subgrid_levels = as.numeric(args$met_subgrid_levels),
mgmin = as.numeric(args$mgmin),
n_hours = as.numeric(args$n_hours),
n_hours_per_met_file = as.numeric(args$n_hours_per_met_file),
n_met_min = as.numeric(args$n_met_min),
ncycl = as.numeric(args$ncycl),
ndump = as.numeric(args$ndump),
Expand Down

0 comments on commit 0b81580

Please sign in to comment.